diff --git a/.gitignore b/.gitignore index 92e132338..61652c446 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .gradle -/local.properties +local.properties .DS_Store build /captures diff --git a/example/build.gradle b/example/build.gradle index f5286b24b..b29f6a623 100644 --- a/example/build.gradle +++ b/example/build.gradle @@ -2,8 +2,6 @@ import java.util.Properties plugins { id("com.android.application") - // make sure this line comes *after* you apply the Android plugin - id("com.getkeepsafe.dexcount") } // local.properties is not checked in, so it is where machine-specific settings such as your mobile diff --git a/example/src/main/java/com/launchdarkly/example/MainActivity.java b/example/src/main/java/com/launchdarkly/example/MainActivity.java index 9b6043240..512be492c 100644 --- a/example/src/main/java/com/launchdarkly/example/MainActivity.java +++ b/example/src/main/java/com/launchdarkly/example/MainActivity.java @@ -26,10 +26,6 @@ import java.util.Date; import java.util.Locale; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import timber.log.Timber; @@ -45,6 +41,9 @@ public class MainActivity extends AppCompatActivity { private static final String DEFAULT_USER_KEY = "user key"; + /** How long startup blocks waiting for the first flags to arrive. */ + private static final int INIT_WAIT_SECONDS = 10; + private LDClient ldClient; private LDStatusListener ldStatusListener; private LDAllFlagsListener allFlagsListener; @@ -142,15 +141,12 @@ public void onCreate(Bundle savedInstanceState) { .set("email", "fake@example.com") .build(); - Future initFuture = LDClient.init(this.getApplication(), ldConfig, context); - try { - ldClient = initFuture.get(10, TimeUnit.SECONDS); - updateStatusString(ldClient.getConnectionInformation()); - ldClient.registerStatusListener(ldStatusListener); - ldClient.registerAllFlagsListener(allFlagsListener); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - Timber.e(e, "Exception when awaiting LaunchDarkly Client initialization"); - } + // Returns the client either way: if the flags have not arrived within the wait, it is usable + // with whatever it has cached. + ldClient = LDClient.init(this.getApplication(), ldConfig, context, INIT_WAIT_SECONDS); + updateStatusString(ldClient.getConnectionInformation()); + ldClient.registerStatusListener(ldStatusListener); + ldClient.registerAllFlagsListener(allFlagsListener); } private void setupListeners() { diff --git a/launchdarkly-android-client-sdk/build.gradle b/launchdarkly-android-client-sdk/build.gradle index 05e9fc44c..b09c029da 100644 --- a/launchdarkly-android-client-sdk/build.gradle +++ b/launchdarkly-android-client-sdk/build.gradle @@ -27,10 +27,30 @@ android { // that the app's state is completely cleared between tests. testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" testInstrumentationRunnerArguments["clearPackageData"] = "true" + if (System.getenv("LD_EVENT_BENCH") == "1") { + // The device benchmark writes one results file for the class. Orchestrator's + // "pm clear" between methods would wipe it, and the runner does not inherit the + // shell environment, so the gate is passed as an instrumentation argument. + testInstrumentationRunnerArguments["clearPackageData"] = "false" + testInstrumentationRunnerArguments["LD_EVENT_BENCH"] = "1" + } } testOptions { - execution = "ANDROIDX_TEST_ORCHESTRATOR" + // EventPersistenceBenchmark on a device keeps one process for the class so its results + // file survives; orchestrator would "pm clear" between methods and wipe it. + execution = (System.getenv("LD_EVENT_BENCH") == "1") ? "HOST" : "ANDROIDX_TEST_ORCHESTRATOR" + + // EventPersistenceBenchmark reports its measurements on stdout, which Gradle otherwise + // swallows, and it is worth re-running even when nothing has changed. Both only apply when + // LD_EVENT_BENCH is set, which is also what un-skips the benchmark, so an ordinary test run + // is unaffected. + unitTests.all { test -> + if (System.getenv("LD_EVENT_BENCH") == "1") { + test.testLogging.showStandardStreams = true + test.outputs.upToDateWhen { false } + } + } } buildFeatures { @@ -79,7 +99,7 @@ ext.versions = [ "jacksonDatabind": "2.10.5.1", "junit": "4.13", "launchdarklyJavaSdkCommon": "2.4.0", - "launchdarklyJavaSdkInternal": "1.9.0", + "launchdarklyJavaSdkInternal": "1.12.1", "launchdarklyLogging": "1.1.1", "okhttp": "4.12.0", "testHelpers": "2.1.0", @@ -107,6 +127,7 @@ dependencies { testImplementation("org.easymock:easymock:4.3") testImplementation("com.launchdarkly:test-helpers:${versions.testHelpers}") testImplementation(project(":shared-test-code")) + testImplementation("com.squareup.okhttp3:mockwebserver:${versions.okhttp}") androidTestImplementation("junit:junit:${versions.junit}") androidTestImplementation("com.squareup.okhttp3:mockwebserver:${versions.okhttp}") diff --git a/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/BatchLocksAndroidTest.java b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/BatchLocksAndroidTest.java new file mode 100644 index 000000000..990d04860 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/BatchLocksAndroidTest.java @@ -0,0 +1,77 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; + +/** + * The way expiry opens a batch on Android before API 26, run on whatever device is attached: the JVM tests + * cannot reach it. + */ +@RunWith(AndroidJUnit4.class) +public class BatchLocksAndroidTest { + private final File directory = + new File(ApplicationProvider.getApplicationContext().getCacheDir(), "batch-locks-test"); + + @Before + public void createDirectory() { + assertTrue(directory.isDirectory() || directory.mkdirs()); + } + + @After + public void deleteDirectory() { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + file.delete(); + } + } + directory.delete(); + } + + @Test + public void neverCreatesABatchThatHasGone() throws IOException { + File gone = new File(directory, "ready-gone-1"); + try { + BatchLocks.openThroughParcelFileDescriptor(gone).close(); + fail("opened a batch that is not there"); + } catch (FileNotFoundException expected) { + // What expiry takes for a batch someone else has dealt with. + } + assertFalse(gone.exists()); + } + + @Test + public void locksABatchExclusivelyAndLeavesItAsItWas() throws IOException { + File batch = new File(directory, "ready-there-1"); + byte[] contents = { 1, 2, 3 }; + try (FileOutputStream out = new FileOutputStream(batch)) { + out.write(contents); + } + + try (FileChannel channel = BatchLocks.openThroughParcelFileDescriptor(batch); + FileLock exclusive = channel.tryLock()) { + assertNotNull(exclusive); + assertFalse(exclusive.isShared()); + } + + assertEquals(contents.length, batch.length()); + } +} diff --git a/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java new file mode 100644 index 000000000..bca87eaf1 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java @@ -0,0 +1,659 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assume.assumeTrue; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.AttributeRef; +import com.launchdarkly.sdk.ContextKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; + +import androidx.test.platform.app.InstrumentationRegistry; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.IntConsumer; + +/** + * The Android counterpart of the iOS {@code EventPersistenceBenchmark}: what it costs to record an + * analytics event, so the encoding research can be argued from measurements on this platform rather + * than from the ones taken on iOS. + *

+ * This exists because the two platforms are not obviously comparable. iOS encodes with {@code Codable}, + * Android encodes with {@code EventOutputFormatter} in java-sdk-internal, and a JIT with escape + * analysis behaves nothing like ahead-of-time compiled Swift. The iOS finding -- that the cost is in + * the serializer's per-value machinery rather than in walking and redacting the context -- is a claim + * about {@code Codable}, and nothing about it transfers here by assumption. + *

+ * Instrumented copy of the JVM {@code EventPersistenceBenchmark}: same corpus, same loops, running + * on ART on a device rather than on a desktop JVM. Skipped unless the instrumentation argument + * {@code LD_EVENT_BENCH=1} is set, because nothing here asserts anything a regression would trip: + * 

+ *   ./gradlew :launchdarkly-android-client-sdk:connectedDebugAndroidTest \\
+ *       -Pandroid.testInstrumentationRunnerArguments.LD_EVENT_BENCH=1 \\
+ *       -Pandroid.testInstrumentationRunnerArguments.class=com.launchdarkly.sdk.android.EventPersistenceBenchmark
+ *
+ * Results are written to {@code event-bench.txt} in the target app's files directory (pulled after + * the run) as well as to logcat, because Gradle's instrumented-test logger does not reliably keep + * {@code System.out}. + */ +public class EventPersistenceBenchmark { + private static final String FLAG_KEY = "benchmark-flag"; + private static final int FLAG_VERSION = 7; + private static final int VARIATION = 1; + + /** + * Fixed rather than {@code System.currentTimeMillis()}, so that repeated runs encode identical + * bytes and the clock call is not charged to the encoder. The event-construction row below covers + * what the real path pays around it. + */ + private static final long CREATION_DATE = 1_700_000_000_000L; + + /** + * One already serialized event, byte for byte what the iOS benchmark stages, so the store figures on the + * two platforms are measuring the same amount of work reaching the same kind of file. + */ + private static final String SERIALIZED_EVENT = + "{\"kind\":\"feature\",\"key\":\"benchmark-flag\",\"value\":true,\"default\":false," + + "\"variation\":1,\"version\":7,\"creationDate\":1740000000000}"; + + /** How many times each timed loop is repeated; the fastest round is reported. */ + private static final int ROUNDS = 5; + + /** + * The sizes the commit-point figures are swept over. The same three are used by the iOS benchmark, so + * the two platforms' tables line up row for row. + */ + private static final int[] BULK_SIZES = {100, 1_000, 10_000}; + + /** + * Consumes results so that the JIT cannot prove the work is unused and delete it. Never read for + * its value. + */ + private static long blackhole; + + /** + * Holds an allocation so that escape analysis cannot scalar-replace it. Assigning to a static field + * is what makes the object genuinely exist, which is the difference between measuring an allocation + * and measuring nothing at all. + */ + private static Object escaped; + + @Rule + public TemporaryFolder eventsDirectory = new TemporaryFolder( + InstrumentationRegistry.getInstrumentation().getTargetContext().getCacheDir()); + + private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); + + private static PrintWriter logFile; + + @BeforeClass + public static void openLog() throws IOException { + File file = new File( + InstrumentationRegistry.getInstrumentation().getTargetContext().getFilesDir(), + "event-bench.txt"); + logFile = new PrintWriter(new FileWriter(file, true), true); + logFile.println("device=" + android.os.Build.MODEL + + " manufacturer=" + android.os.Build.MANUFACTURER + + " device=" + android.os.Build.DEVICE + + " abi=" + android.os.Build.SUPPORTED_ABIS[0] + + " sdk=" + android.os.Build.VERSION.SDK_INT + + " release=" + android.os.Build.VERSION.RELEASE + + " hardware=" + android.os.Build.HARDWARE); + } + + @AfterClass + public static void closeLog() { + if (logFile != null) { + logFile.close(); + } + } + + @Before + public void requireBenchmarking() { + assumeTrue("Set instrumentation argument LD_EVENT_BENCH=1 to measure event recording", + "1".equals(InstrumentationRegistry.getArguments().getString("LD_EVENT_BENCH"))); + } + + /** + * M1 and M2 for Android: where the time goes when one feature event is serialized, across the + * context shapes and privacy settings that change how much work redaction has to do. + *

+ * The shape of the result is the interesting part. If the cost is dominated by walking and + * redacting the context, it should grow with the number of attributes and grow again when private + * attributes force the filtering path. If it is dominated by the serializer's fixed per-event + * machinery, it should be far flatter than the attribute counts suggest. + */ + @Test + public void encodingCostByContextShape() { + List building = new ArrayList<>(); + List serializing = new ArrayList<>(); + + for (ContextShape shape : ContextShape.values()) { + final LDContext context = makeContext(shape, "benchmark-key"); + + building.add(new Measurement(shape.label, measure(200_000, i -> { + escaped = featureEvent(context, true); + }))); + + for (PrivacyShape privacy : privacyShapes()) { + final OutboundEventBuffer buffer = makeBuffer(privacy); + + byte[] sample = buffer.serialize(featureEvent(context, true)); + assertNotNull("the corpus produced an event that will not serialize", sample); + + double nanos = measure(50_000, i -> { + byte[] out = buffer.serialize(featureEvent(context, true)); + blackhole += out == null ? 0 : out.length; + }); + serializing.add(new Measurement( + shape.label + ", " + privacy.name + ", " + sample.length + " bytes", nanos)); + } + } + + report("building one feature event, no serialization", building); + report("serializing one feature event", serializing); + } + + /** + * The figures a customer actually pays, mirroring what {@link DirectEventProcessor} does per call. + *

+ * The three rows are the three paths through {@code recordEvaluationEvent} and {@code recordCustomEvent}: + * an evaluation of an untracked flag is a counter increment; an evaluation of a tracked flag also + * serializes and stages a full event; and a {@code track} is a commit point, which additionally turns + * the counters into summary events and writes everything to the log. + *

+ * The sequence is reproduced here rather than driven through the processor itself, which would need + * an HTTP server, a scheduler and a {@code ClientContext} whose costs would swamp what is being + * measured. It is kept deliberately close to the original; if the processor changes, this should too. + */ + @Test + public void recordingCostPerCall() throws IOException { + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + final EventStore store = EventStore.create(eventsDirectory.getRoot(), "benchmark-key", + Integer.MAX_VALUE, true, logger); + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + + try { + List results = new ArrayList<>(); + + results.add(new Measurement("evaluation, summary only", measure(200_000, i -> { + buffer.summarize(featureEvent(context, false)); + }))); + + results.add(new Measurement("evaluation, full event", measure(50_000, i -> { + Event.FeatureRequest event = featureEvent(context, true); + buffer.summarize(event); + byte[] out = buffer.serialize(event); + if (out != null) { + store.stage(out); + blackhole += out.length; + } + }))); + + results.add(new Measurement("evaluation + track (commit point)", measure(20_000, i -> { + buffer.summarize(featureEvent(context, false)); + byte[] out = buffer.serialize( + new Event.Custom(CREATION_DATE, "benchmark-event", context, LDValue.ofNull(), null)); + if (out != null) { + store.stage(out); + blackhole += out.length; + } + for (byte[] summary : buffer.serializeSummariesAndReset()) { + store.stage(summary, true); + blackhole += summary.length; + } + store.commit(); + }))); + + report("recording, per call", results); + } finally { + store.close(); + } + } + + /** + * What a commit point costs when it goes badly, rather than on average. + *

+ * {@code track} and {@code identify} commit on the caller's thread by design -- that is the durable + * barrier -- so this is the one figure here an application can observe as a stall. The mean is the number + * that gets quoted; the tail is the number a user notices. + *

+ * The counterpart iOS test runs this against a SQLite store as well, where the tail is the point: a + * database in WAL mode folds its write-ahead log back into the file every thousand pages, on whichever + * thread happens to be committing, so one unlucky caller pays for all of them. An append has no such + * thing, and these rows are what that claim is measured against. + *

+ * Two rows per size. {@code track} commits where it stands, so its tail is the write. The other row + * encodes and stages without committing, and the store never writes on its own, so its tail is the + * encode and the copy; the gap between the two is what the write itself costs. + *

+ * p99.9 only separates from the worst at ten thousand samples. Below that the index falls on the + * last sample and the two columns are the same number by construction. They are left side by side so the + * table lines up across the sweep, and so the size at which they part is visible rather than asserted. + */ + @Test + public void commitPointOutliers() throws IOException { + final byte[] serialized = SERIALIZED_EVENT.getBytes(StandardCharsets.UTF_8); + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + + // A store of its own for each size and each row, so that a log inherited from an earlier measurement + // does not charge the next one for the file it had already grown. + for (int calls : BULK_SIZES) { + final EventStore committing = EventStore.create(eventsDirectory.newFolder(), "benchmark-key", + Integer.MAX_VALUE, true, logger); + final EventStore staging = EventStore.create(eventsDirectory.newFolder(), "benchmark-key", + Integer.MAX_VALUE, true, logger); + try { + List results = new ArrayList<>(); + + results.add(new DistributionMeasurement("track, commits on this thread", + measureDistribution(calls, i -> { + committing.stage(serialized); + committing.commit(); + }))); + + results.add(new DistributionMeasurement("evaluation, staged only", + measureDistribution(calls, i -> { + Event.FeatureRequest event = featureEvent(context, true); + buffer.summarize(event); + byte[] out = buffer.serialize(event); + if (out != null) { + staging.stage(out); + blackhole += out.length; + } + }))); + + reportDistribution("one call, " + count(calls) + " of them, on the caller's thread", results); + } finally { + committing.close(); + staging.close(); + } + } + } + + /** + * Whether O2 -- caching the encoded context and reusing it while the context has not changed -- + * could work here, which comes down to what the cache key would cost to check. + *

+ * Android differs from iOS in a way that matters. {@code LDContext} is an immutable object passed by + * reference, and nothing on the event path copies it, so an application that holds one context hands + * the same instance to every event and {@code equals} can settle on reference identity. iOS had to + * reach that fast path by not rebuilding a value type. The row that decides the question is the other + * one: an application that rebuilds its context between evaluations produces an equal object sharing + * nothing, and that comparison is the price of a cache hit in the worst case. + */ + @Test + public void contextComparisonCost() { + List comparisons = new ArrayList<>(); + List hashing = new ArrayList<>(); + + for (ContextShape shape : ContextShape.values()) { + final LDContext context = makeContext(shape, "benchmark-key"); + final LDContext sameInstance = context; + final LDContext rebuilt = makeContext(shape, "benchmark-key"); + + // Both must be equal, or the figures below measure an early exit rather than a comparison. + assertEquals(shape.label, context, sameInstance); + assertEquals(shape.label, context, rebuilt); + + comparisons.add(new Measurement("same instance, " + shape.label, measure(500_000, i -> { + blackhole += context.equals(sameInstance) ? 1 : 0; + }))); + comparisons.add(new Measurement("independently built, " + shape.label, measure(200_000, i -> { + blackhole += context.equals(rebuilt) ? 1 : 0; + }))); + hashing.add(new Measurement(shape.label, measure(500_000, i -> { + blackhole += context.hashCode(); + }))); + } + + report("comparing two equal contexts", comparisons); + report("hashing a context", hashing); + } + + /** + * What the deferred half of the bill costs: summaries are not serialized per evaluation but at + * commit points and deliveries, and the size of that payment depends on how many distinct flags the + * counters have accumulated. + *

+ * Counting and serializing are reported separately, because only the second is deferred -- the + * counter increment is already paid on every evaluation. + */ + @Test + public void summarySerializationCost() { + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + List results = new ArrayList<>(); + + for (final int flagCount : new int[] {1, 10, 50}) { + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + + double countingOnly = measure(20_000, i -> { + for (int flag = 0; flag < flagCount; flag++) { + buffer.summarize(summaryEvent(context, flag)); + } + }); + // Drained, so the counters left behind above do not inflate the first serialized round. + buffer.serializeSummariesAndReset(); + + double countingAndSerializing = measure(20_000, i -> { + for (int flag = 0; flag < flagCount; flag++) { + buffer.summarize(summaryEvent(context, flag)); + } + for (byte[] summary : buffer.serializeSummariesAndReset()) { + blackhole += summary.length; + } + }); + + results.add(new Measurement(flagCount + " flags: counting only", countingOnly)); + results.add(new Measurement(flagCount + " flags: counting + serializing", + countingAndSerializing)); + results.add(new Measurement(flagCount + " flags: serializing alone", + countingAndSerializing - countingOnly)); + } + + report("turning counters into a summary event", results); + } + + // MARK: Corpus + + /** + * The same five shapes the iOS benchmark uses, so the two sets of numbers describe the same + * contexts. + */ + private enum ContextShape { + KEY_ONLY("key only"), + STUB("stub, 9 attributes"), + WIDE("20 flat attributes"), + NESTED("nested attributes"), + MULTI("multi-context"); + + final String label; + + ContextShape(String label) { + this.label = label; + } + } + + private static LDContext makeContext(ContextShape shape, String key) { + switch (shape) { + case KEY_ONLY: + return LDContext.create(key); + case STUB: + return stubContext(key); + case WIDE: { + com.launchdarkly.sdk.ContextBuilder builder = LDContext.builder(key).name("Wide"); + for (int i = 0; i < 20; i++) { + builder.set("attribute" + i, "value" + i); + } + return builder.build(); + } + case NESTED: + return LDContext.builder(key) + .name("Nested") + .set("address", LDValue.buildObject() + .put("street", "1 Main St") + .put("city", "Springfield") + .put("geo", LDValue.buildObject().put("lat", 1.5).put("lon", -2.5).build()) + .build()) + .set("tags", LDValue.buildArray() + .add("a").add("b").add("c").add("d").add("e").build()) + .build(); + case MULTI: + return LDContext.createMulti( + stubContext(key), + LDContext.builder(ContextKind.of("device"), "device-" + key) + .set("os", LDValue.buildObject() + .put("name", "Android").put("version", 14).build()) + .build()); + default: + throw new IllegalArgumentException(shape.name()); + } + } + + /** Nine attributes, matching the shape iOS calls its stub context. */ + private static LDContext stubContext(String key) { + return LDContext.builder(key) + .name("Stub Context") + .set("firstName", "Ada") + .set("lastName", "Lovelace") + .set("email", "ada@example.com") + .set("country", "US") + .set("age", 36) + .set("score", 12.5) + .set("verified", true) + .set("plan", "enterprise") + .build(); + } + + /** The privacy settings that change what redaction has to do. */ + private static final class PrivacyShape { + final String name; + final boolean allAttributesPrivate; + final Collection privateAttributes; + + PrivacyShape(String name, boolean allAttributesPrivate, Collection privateAttributes) { + this.name = name; + this.allAttributesPrivate = allAttributesPrivate; + this.privateAttributes = privateAttributes; + } + } + + private static PrivacyShape noRedaction() { + return new PrivacyShape("no redaction", false, Collections.emptyList()); + } + + private static List privacyShapes() { + return Arrays.asList( + noRedaction(), + new PrivacyShape("1 global private", false, + Collections.singletonList(AttributeRef.fromLiteral("email"))), + new PrivacyShape("all private", true, Collections.emptyList())); + } + + private OutboundEventBuffer makeBuffer(PrivacyShape privacy) { + return new OutboundEventBuffer(privacy.allAttributesPrivate, privacy.privateAttributes, + true, Integer.MAX_VALUE, logger); + } + + private static Event.FeatureRequest featureEvent(LDContext context, boolean requireFullEvent) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY, context, FLAG_VERSION, VARIATION, + LDValue.of(true), LDValue.of(false), null, null, requireFullEvent, null, false); + } + + private static Event.FeatureRequest summaryEvent(LDContext context, int flagIndex) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY + "-" + flagIndex, context, + FLAG_VERSION, VARIATION, LDValue.of(true), LDValue.of(false), null, null, false, null, + false); + } + + // MARK: Harness + + /** + * Times {@code body}, returning nanoseconds per iteration. + *

+ * Warmed first, because a JVM measured cold reports the interpreter rather than the compiled code. + * Then run several times with the fastest round reported: a garbage collection or a descheduled + * thread can only ever make a round slower, so the minimum is the closest estimate of the work + * itself. That is also why these numbers should not be read as what a busy device would see. + */ + private double measure(int iterations, IntConsumer body) { + int warmup = Math.max(10_000, iterations / 4); + for (int i = 0; i < warmup; i++) { + body.accept(i); + } + + double best = Double.MAX_VALUE; + for (int round = 0; round < ROUNDS; round++) { + long start = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + body.accept(i); + } + long elapsed = System.nanoTime() - start; + best = Math.min(best, (double) elapsed / iterations); + } + return best; + } + + /** How long one call took, across many of them, when the tail is the point. */ + private static final class Distribution { + final double median; + final double mean; + final double p999; + final double worst; + + Distribution(double median, double mean, double p999, double worst) { + this.median = median; + this.mean = mean; + this.p999 = p999; + this.worst = worst; + } + } + + /** + * Times every call rather than the enclosing loop, so the tail survives to be reported. + *

+ * One round, and the median rather than the fastest. Every other figure here reports the cheapest of + * several rounds, on the reasoning that noise can only make a round slower -- but here an outlier is the + * measurement, and that aggregation would throw away the answer. + */ + private Distribution measureDistribution(int iterations, IntConsumer body) { + int warmup = Math.max(10_000, iterations / 4); + for (int i = 0; i < warmup; i++) { + body.accept(i); + } + + long[] samples = new long[iterations]; + for (int i = 0; i < iterations; i++) { + long start = System.nanoTime(); + body.accept(i); + samples[i] = System.nanoTime() - start; + } + + long total = 0; + for (long sample : samples) { + total += sample; + } + Arrays.sort(samples); + return new Distribution(samples[iterations / 2], + (double) total / iterations, + samples[Math.min(iterations - 1, (int) ((long) iterations * 999 / 1000))], + samples[iterations - 1]); + } + + private static final class DistributionMeasurement { + final String name; + final Distribution distribution; + + DistributionMeasurement(String name, Distribution distribution) { + this.name = name; + this.distribution = distribution; + } + } + + private static final class Measurement { + final String name; + final double nanosPerOp; + + Measurement(String name, double nanosPerOp) { + this.name = name; + this.nanosPerOp = nanosPerOp; + } + } + + private static void report(String title, List results) { + int width = 0; + for (Measurement result : results) { + width = Math.max(width, result.name.length()); + } + StringBuilder out = new StringBuilder("\n").append(title).append('\n'); + for (Measurement result : results) { + out.append(" ").append(pad(result.name, width)).append(" ") + .append(format(result.nanosPerOp)).append('\n'); + } + emit(out); + } + + private static void reportDistribution(String title, List results) { + int width = 0; + for (DistributionMeasurement result : results) { + width = Math.max(width, result.name.length()); + } + StringBuilder out = new StringBuilder("\n").append(title).append('\n') + .append(" ").append(pad("", width)) + .append(" ").append(column("median")) + .append(" ").append(column("mean")) + .append(" ").append(column("p99.9")) + .append(" ").append(column("worst")).append('\n'); + for (DistributionMeasurement result : results) { + out.append(" ").append(pad(result.name, width)) + .append(" ").append(format(result.distribution.median)) + .append(" ").append(format(result.distribution.mean)) + .append(" ").append(format(result.distribution.p999)) + .append(" ").append(format(result.distribution.worst)).append('\n'); + } + emit(out); + } + + private static void emit(StringBuilder out) { + String text = out.toString(); + System.out.print(text); + System.out.flush(); + if (logFile != null) { + logFile.print(text); + logFile.flush(); + } + } + + /** Renders a count with thousands separators, so a table title reads as a quantity, not a code literal. */ + private static String count(int value) { + return String.format("%,d", value); + } + + /** Right-aligns a column heading over the fixed width {@link #format} produces. */ + private static String column(String title) { + StringBuilder padded = new StringBuilder(); + for (int i = title.length(); i < 11; i++) { + padded.append(' '); + } + return padded.append(title).toString(); + } + + private static String pad(String value, int width) { + StringBuilder padded = new StringBuilder(value); + while (padded.length() < width) { + padded.append(' '); + } + return padded.toString(); + } + + private static String format(double nanoseconds) { + if (nanoseconds >= 1_000_000) { + return String.format("%8.2f ms", nanoseconds / 1_000_000); + } + if (nanoseconds >= 1_000) { + return String.format("%8.2f \u00b5s", nanoseconds / 1_000); + } + return String.format("%8.1f ns", nanoseconds); + } +} diff --git a/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/LDClientEventTest.java b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/LDClientEventTest.java index 544c416bd..86dd42309 100644 --- a/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/LDClientEventTest.java +++ b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/LDClientEventTest.java @@ -17,7 +17,9 @@ import com.launchdarkly.sdk.android.DataModel.Flag; import com.launchdarkly.sdk.android.LDConfig.Builder.AutoEnvAttributes; import com.launchdarkly.sdk.android.integrations.DedupingHook; +import com.launchdarkly.sdk.android.integrations.LDCrashHandler; import com.launchdarkly.sdk.android.integrations.Hook; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; import com.launchdarkly.sdk.android.subsystems.PersistentDataStore; import com.launchdarkly.sdk.internal.GsonHelpers; import com.launchdarkly.sdk.json.JsonSerialization; @@ -26,8 +28,14 @@ import org.junit.Test; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; import okhttp3.HttpUrl; +import okhttp3.mockwebserver.Dispatcher; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; @@ -95,6 +103,165 @@ public void testTrackData() throws IOException, InterruptedException { } } + @Test + public void flushAndWaitReportsDelivery() throws IOException, InterruptedException { + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + mockEventsServer.enqueue(new MockResponse()); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer).build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + client.track("test-event"); + + assertTrue(client.flushAndWait(5, TimeUnit.SECONDS)); + LDValue[] events = getEventsFromLastRequest(mockEventsServer, 2); + assertCustomEvent(events[1], ldContext, "test-event"); + } + } + } + + @Test + public void crashHandlerDeliversTheEventsBeforePassingTheCrashOn() throws IOException, InterruptedException { + Thread.UncaughtExceptionHandler originalDefault = Thread.getDefaultUncaughtExceptionHandler(); + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + mockEventsServer.enqueue(new MockResponse()); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer).build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + int[] requestsSeenByNextHandler = {-1}; + Thread.setDefaultUncaughtExceptionHandler((thread, throwable) -> + requestsSeenByNextHandler[0] = mockEventsServer.getRequestCount()); + LDCrashHandler.install(5, TimeUnit.SECONDS); + client.track("test-event"); + + Thread.getDefaultUncaughtExceptionHandler() + .uncaughtException(Thread.currentThread(), new RuntimeException("boom")); + + assertEquals(1, requestsSeenByNextHandler[0]); + LDValue[] events = getEventsFromLastRequest(mockEventsServer, 2); + assertCustomEvent(events[1], ldContext, "test-event"); + } + } finally { + Thread.setDefaultUncaughtExceptionHandler(originalDefault); + } + } + + @Test + public void flushAndWaitReportsFailureOnceClosed() throws IOException { + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer).build(); + LDClient client = LDClient.init(application, ldConfig, ldContext, 0); + client.close(); + + assertFalse(client.flushAndWait(5, TimeUnit.SECONDS)); + } + } + + @Test + public void flushAndWaitDeliversEveryEnvironmentAtOnce() throws Exception { + // Each post is held for longer than half the budget, so the call can only report true if the + // environments' deliveries ran side by side rather than one after the other. + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.setDispatcher(new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + return new MockResponse().setHeadersDelay(1_500, TimeUnit.MILLISECONDS); + } + }); + mockEventsServer.start(); + + Map secondaryKeys = new HashMap<>(); + secondaryKeys.put("second", "second-mobile-key"); + LDConfig ldConfig = baseConfigBuilder(mockEventsServer) + .secondaryMobileKeys(secondaryKeys) + .build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + client.track("primary-event"); + LDClient.getForMobileKey("second").track("second-event"); + + assertTrue(client.flushAndWait(2_500, TimeUnit.MILLISECONDS)); + assertEquals(2, mockEventsServer.getRequestCount()); + } + } + } + + @Test + public void flushAndWaitWithTheMostNegativeTimeoutDoesNotWait() throws IOException { + // toNanos saturates at Long.MIN_VALUE, and unclamped that wraps round to a wait for as long + // as the delivery takes -- which would then be reported as delivered. + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + mockEventsServer.enqueue(new MockResponse().setHeadersDelay(3, TimeUnit.SECONDS)); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer).build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + client.track("test-event"); + + long started = System.nanoTime(); + assertFalse(client.flushAndWait(Long.MIN_VALUE, TimeUnit.NANOSECONDS)); + long waitedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started); + assertTrue("waited " + waitedMillis + "ms", waitedMillis < 1_000); + } + } + } + + @Test + public void flushAndWaitReportsFailureWhenTheDeliveryIsCanceled() throws IOException { + // A custom event processor may hand back a future that is canceled; Future.get then throws + // CancellationException, which is unchecked and must not escape a boolean answer. + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer) + .events(clientContext -> new CancelingEventProcessor()) + .build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + assertFalse(client.flushAndWait(5, TimeUnit.SECONDS)); + } + } + } + + /** An event processor whose deliveries are always canceled before they can report. */ + private static final class CancelingEventProcessor implements EventProcessor { + @Override + public Future flushAsync() { + FutureTask delivery = new FutureTask<>(() -> true); + delivery.cancel(false); + return delivery; + } + + @Override + public void flush() {} + + @Override + public void blockingFlush() {} + + @Override + public void setInBackground(boolean inBackground) {} + + @Override + public void setOffline(boolean offline) {} + + @Override + public void close() {} + + @Override + public void recordEvaluationEvent(LDContext context, String flagKey, int flagVersion, + int variation, LDValue value, EvaluationReason reason, + LDValue defaultValue, boolean requireFullEvent, + Long debugEventsUntilDate) {} + + @Override + public void recordIdentifyEvent(LDContext context) {} + + @Override + public void recordCustomEvent(LDContext context, String eventKey, LDValue data, + Double metricValue) {} + } + @Test public void testTrackDataValueNull() throws IOException, InterruptedException { try (MockWebServer mockEventsServer = new MockWebServer()) { diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AnalyticsEventSender.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AnalyticsEventSender.java new file mode 100644 index 000000000..b5cb88e50 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AnalyticsEventSender.java @@ -0,0 +1,165 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; +import com.launchdarkly.sdk.internal.http.HttpHelpers; +import com.launchdarkly.sdk.internal.http.HttpProperties; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.util.Date; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; + +import okhttp3.Headers; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** + * Posts a batch of analytics events under a payload ID the caller chooses. + *

+ * This exists for the sake of one header. java-sdk-internal's sender invents a payload ID per call, + * which is the right thing when the events being sent only ever existed in memory: the sole reason to + * send the same events twice is its own immediate retry, and it holds the ID for that. Once events are + * kept on disk, a delivery can also be repeated by a later run of the application, after a process died + * without learning whether the service had received the batch. Those two attempts are the same delivery + * and have to say so, or the events are counted twice - and counting them twice is worse than losing + * them, because an inflated exposure count silently biases an experiment rather than visibly shrinking + * it. + *

+ * The ID belongs to the batch, which is why it is the batch's identity in {@link EventStore} rather than + * something generated at send time. + */ +final class AnalyticsEventSender implements Closeable { + private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); + private static final String EVENT_SCHEMA_HEADER = "X-LaunchDarkly-Event-Schema"; + private static final String EVENT_SCHEMA_VERSION = "4"; + private static final String PAYLOAD_ID_HEADER = "X-LaunchDarkly-Payload-ID"; + + private final OkHttpClient client; + private final boolean ownsClient; + private final Headers baseHeaders; + private final LDLogger logger; + + AnalyticsEventSender(HttpProperties httpProperties, LDLogger logger) { + this.logger = logger; + if (httpProperties.getSharedHttpClient() == null) { + this.client = httpProperties.toHttpClientBuilder().build(); + this.ownsClient = true; + } else { + this.client = httpProperties.getSharedHttpClient(); + this.ownsClient = false; + } + this.baseHeaders = httpProperties.toHeadersBuilder().build(); + } + + /** What one post came to. */ + static final class Result { + /** Whether the service took the events. */ + final boolean success; + /** + * For a failure, whether it is unlikely to pass soon: a 4xx other than 400, 408 and 429, or a TLS or + * certificate failure. The batch is kept either way; this only decides how long the next post waits. + */ + final boolean unexpected; + /** The HTTP status, or 0 where no response arrived. */ + final int status; + /** The wait a failed response asked for with {@code Retry-After}, or -1 where it asked for none. */ + final long retryAfterMillis; + /** The service's clock, from an accepted response's {@code Date}, or null. */ + final Date serverTime; + + Result(boolean success, boolean unexpected, int status, long retryAfterMillis, Date serverTime) { + this.success = success; + this.unexpected = unexpected; + this.status = status; + this.retryAfterMillis = retryAfterMillis; + this.serverTime = serverTime; + } + } + + /** + * Posts a batch once. The retry is the caller's, so that the wait before it does not hold a thread. + * + * @param payload the JSON request body, sent from the array it is in rather than copied out of it + * @param eventCount how many events the body holds, for logging + * @param payloadId identifies this delivery, the same on every attempt at it + * @param eventsBaseUri the events service + * @return whether the events arrived and, where they did not, how long the next attempt should wait + */ + Result sendBatch(EventLogFormat.Payload payload, int eventCount, String payloadId, URI eventsBaseUri) { + URI uri = HttpHelpers.concatenateUriPath(eventsBaseUri, + StandardEndpoints.ANALYTICS_EVENTS_REQUEST_PATH); + Request request = new Request.Builder() + .url(uri.toString()) + .headers(baseHeaders) + .addHeader("Content-Type", "application/json") + .addHeader(EVENT_SCHEMA_HEADER, EVENT_SCHEMA_VERSION) + .addHeader(PAYLOAD_ID_HEADER, payloadId) + .post(RequestBody.create(payload.bytes, JSON, 0, payload.length)) + .build(); + + try (Response response = client.newCall(request).execute()) { + if (response.isSuccessful()) { + logger.debug("Posted {} event(s)", eventCount); + return new Result(true, false, response.code(), -1, parseDate(response)); + } + logger.warn("Error posting events: HTTP status {}", response.code()); + return new Result(false, LDUtil.isHttpErrorUnexpected(response.code()), response.code(), + retryAfterMillis(response), null); + } catch (IOException e) { + logger.warn("Error posting events: {}", LogValues.exceptionSummary(e)); + // A certificate that will not validate is a misconfiguration, which waits like a refusal. + boolean unexpected = e instanceof SSLHandshakeException || e instanceof SSLPeerUnverifiedException; + return new Result(false, unexpected, 0, -1, null); + } + } + + /** {@code Retry-After} as seconds or as an HTTP date, or -1 where it is absent or unreadable. */ + private static long retryAfterMillis(Response response) { + String value = response.header("Retry-After"); + if (value == null) { + return -1; + } + try { + long seconds = Long.parseLong(value.trim()); + return seconds < 0 ? -1 : TimeUnit.SECONDS.toMillis(seconds); + } catch (NumberFormatException e) { + // An HTTP date, then. + } + Date date; + try { + date = response.headers().getDate("Retry-After"); + } catch (RuntimeException e) { + return -1; + } + return date == null ? -1 : Math.max(0, date.getTime() - System.currentTimeMillis()); + } + + /** + * The service's own clock, which the SDK trusts over the device's when deciding whether a debug + * window has closed. + */ + private Date parseDate(Response response) { + try { + return response.headers().getDate("Date"); + } catch (RuntimeException e) { + return null; + } + } + + @Override + public void close() { + if (!ownsClient) { + return; + } + client.dispatcher().executorService().shutdown(); + client.connectionPool().evictAll(); + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java index 357648917..ac6e90aee 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java @@ -169,6 +169,11 @@ public File getCacheDir() { return application.getCacheDir(); } + @Override + public File getNoBackupFilesDir() { + return application.getNoBackupFilesDir(); + } + @Override public void close() { connectivityChangeListeners.clear(); diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/BatchLocks.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/BatchLocks.java new file mode 100644 index 000000000..657b717b6 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/BatchLocks.java @@ -0,0 +1,164 @@ +package com.launchdarkly.sdk.android; + +import android.annotation.SuppressLint; +import android.os.ParcelFileDescriptor; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; + +/** + * The locks that tell every live process which batches have a request out, so that expiry never deletes a + * batch someone is still sending: were that request to fail, there would be no file left to retry from. + *

+ * A sender holds a shared lock on the batch for as long as its request is out. Expiry deletes a batch only + * while holding an exclusive lock on it, which it cannot have while anyone holds the shared one. The kernel + * drops a lock when its process dies, so a batch whose sender is gone is expired like any other. + *

+ * A POSIX lock belongs to the process rather than to the descriptor it was taken through, and closing any + * descriptor this process has on the file releases it. So the batches locked here are remembered, and expiry + * in this process asks that rather than opening a descriptor on one. The lock is taken after the batch has + * been read for sending, since the read closes a descriptor of its own; a batch expired by another process in + * the moment between is still sent from what was read, but has no file for a retry. + */ +final class BatchLocks { + /** A batch this process is sending, the descriptor its lock was taken through, and how many send it. */ + private static final class Held { + final RandomAccessFile file; + FileLock lock; + int senders; + + Held(RandomAccessFile file) { + this.file = file; + } + } + + /** + * The batches this process has a request out for, by absolute path. Guards the probes and deletions below + * as well, so that a store in this process cannot start sending a batch another is expiring. + */ + private static final Map heldInThisProcess = new HashMap<>(); + + private BatchLocks() { + } + + /** + * Locks a batch for as long as a request for it is out. Each call is ended by one + * {@link #stoppedSending}; a batch two deliveries send at once stays locked until both have stopped. + */ + static void startedSending(File batch) { + synchronized (heldInThisProcess) { + String path = batch.getAbsolutePath(); + Held held = heldInThisProcess.get(path); + try { + if (held == null) { + held = new Held(new RandomAccessFile(batch, "r")); + heldInThisProcess.put(path, held); + } + held.senders++; + if (held.lock != null) { + // Taken again rather than kept: this sender read the batch through a descriptor of its own, + // and closing that released the lock the first one took. + held.lock.release(); + held.lock = null; + } + held.lock = held.file.getChannel().tryLock(0L, Long.MAX_VALUE, true); + } catch (FileNotFoundException e) { + // Gone already, so there is nothing left for expiry to take. + } catch (IOException | OverlappingFileLockException e) { + // Still remembered, which keeps expiry in this process away from it; only other processes + // cannot tell. + if (held != null) { + held.lock = null; + } + } + } + } + + /** Ends one {@link #startedSending}, releasing the lock once the last sender has stopped. */ + static void stoppedSending(File batch) { + synchronized (heldInThisProcess) { + String path = batch.getAbsolutePath(); + Held held = heldInThisProcess.get(path); + if (held == null || --held.senders > 0) { + return; + } + heldInThisProcess.remove(path); + try { + // Releases the lock with it. + held.file.close(); + } catch (IOException ignored) { + // Released by the kernel all the same once the descriptor is gone. + } + } + } + + /** Whether a store in this process is sending a batch, answered without opening the file. */ + static boolean isSentInThisProcess(File batch) { + synchronized (heldInThisProcess) { + return heldInThisProcess.containsKey(batch.getAbsolutePath()); + } + } + + /** + * Deletes a batch for expiry, unless some live process is sending it. + * + * @return whether this call deleted it; false for one being sent, one that could not be looked at, and one + * already gone, whose events are someone else's to count + */ + static boolean deleteUnlessBeingSent(File batch) { + synchronized (heldInThisProcess) { + if (heldInThisProcess.containsKey(batch.getAbsolutePath())) { + return false; + } + try (FileChannel channel = openWithoutCreating(batch); + FileLock exclusive = channel.tryLock()) { + // Deleted while still locked, so no sender can take its lock between the look and the delete. + return exclusive != null && batch.delete(); + } catch (IOException | OverlappingFileLockException e) { + return false; + } + } + } + + private static volatile boolean hasNioFiles = true; + + /** + * Opens a batch for writing, which an exclusive lock needs, failing if it is missing. Never creates it: a + * batch another process delivered or expired since the listing would be back for a moment, empty, and a + * delivery reading it then would report its events lost. + */ + static FileChannel openWithoutCreating(File batch) throws IOException { + if (hasNioFiles) { + try { + return NioFiles.openWithoutCreating(batch); + } catch (LinkageError e) { + // Android before API 26, which has no java.nio.file. + hasNioFiles = false; + } + } + return openThroughParcelFileDescriptor(batch); + } + + /** {@link #openWithoutCreating} for any Android version. Not on the JVM, where this class is a stub. */ + static FileChannel openThroughParcelFileDescriptor(File batch) throws FileNotFoundException { + ParcelFileDescriptor descriptor = ParcelFileDescriptor.open(batch, ParcelFileDescriptor.MODE_READ_WRITE); + // Closing the channel closes the stream, which closes the descriptor. + return new ParcelFileDescriptor.AutoCloseOutputStream(descriptor).getChannel(); + } + + /** Apart, so that Android without java.nio.file fails when this is called rather than when BatchLocks loads. */ + @SuppressLint("NewApi") + private static final class NioFiles { + static FileChannel openWithoutCreating(File batch) throws IOException { + return FileChannel.open(batch.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE); + } + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/ComponentsImpl.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/ComponentsImpl.java index 34b47a2fe..f78d2578d 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/ComponentsImpl.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/ComponentsImpl.java @@ -4,6 +4,7 @@ import com.launchdarkly.sdk.EvaluationReason; import com.launchdarkly.sdk.LDContext; import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.integrations.EventPersistence; import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; import com.launchdarkly.sdk.android.integrations.HooksConfigurationBuilder; import com.launchdarkly.sdk.android.integrations.HttpConfigurationBuilder; @@ -20,14 +21,13 @@ import com.launchdarkly.sdk.android.subsystems.HookConfiguration; import com.launchdarkly.sdk.android.subsystems.HttpConfiguration; import com.launchdarkly.sdk.android.subsystems.PluginsConfiguration; -import com.launchdarkly.sdk.internal.events.DefaultEventProcessor; import com.launchdarkly.sdk.internal.events.DefaultEventSender; -import com.launchdarkly.sdk.internal.events.Event; -import com.launchdarkly.sdk.internal.events.EventsConfiguration; +import com.launchdarkly.sdk.internal.events.EventSender; -import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Random; +import java.util.concurrent.Future; /** * This class contains the package-private implementations of component factories and builders whose @@ -75,6 +75,12 @@ public void flush() {} @Override public void blockingFlush() {} + @Override + public Future flushAsync() { + // Nothing was recorded, so there is nothing undelivered to warn the caller about. + return new LDSuccessFuture<>(true); + } + @Override public void setInBackground(boolean inBackground) {} @@ -98,36 +104,83 @@ public void recordCustomEvent(LDContext context, String eventKey, LDValue data, static final class EventProcessorBuilderImpl extends EventProcessorBuilder implements DiagnosticDescription { + // The backoff's delays, settable by tests the specified ones would hold for minutes; a negative + // normal ceiling means the specified delays. + private long retryInitialDelayMillis; + private long retryNormalCeilingMillis = -1; + private long retryExtendedCeilingMillis; + + EventProcessorBuilderImpl retryDelaysForTesting(long initialDelayMillis, long normalCeilingMillis, + long extendedCeilingMillis) { + retryInitialDelayMillis = initialDelayMillis; + retryNormalCeilingMillis = normalCeilingMillis; + retryExtendedCeilingMillis = extendedCeilingMillis; + return this; + } + + private DeliveryRetryState makeRetryState() { + if (retryNormalCeilingMillis < 0) { + return new DeliveryRetryState(flushIntervalMillis); + } + return new DeliveryRetryState(retryInitialDelayMillis, retryNormalCeilingMillis, + retryExtendedCeilingMillis, new Random()); + } + @Override public EventProcessor build(ClientContext clientContext) { ClientContextImpl clientContextImpl = ClientContextImpl.get(clientContext); - EventsConfiguration eventsConfig = new EventsConfiguration( - allAttributesPrivate, - capacity, - null, // contextDeduplicator - not needed for client-side use - diagnosticRecordingIntervalMillis, - clientContextImpl.getDiagnosticStore(), - new DefaultEventSender( - LDUtil.makeHttpProperties(clientContext), - StandardEndpoints.ANALYTICS_EVENTS_REQUEST_PATH, - StandardEndpoints.DIAGNOSTIC_EVENTS_REQUEST_PATH, - 0L, // use default retry delay - false, // disable gzip compression for Android + EventSender diagnosticEventSender = new DefaultEventSender( + LDUtil.makeHttpProperties(clientContext), + StandardEndpoints.ANALYTICS_EVENTS_REQUEST_PATH, + StandardEndpoints.DIAGNOSTIC_EVENTS_REQUEST_PATH, + 0L, // use default retry delay + false, // disable gzip compression for Android + clientContext.getBaseLogger()); + boolean commitOnCallerThread = eventPersistence == EventPersistence.IMMEDIATE; + return new DirectEventProcessor( + new OutboundEventBuffer( + allAttributesPrivate, + privateAttributes, + true, // perContextSummarization - enable for client SDK + capacity, clientContext.getBaseLogger()), - 1, // eventSendingThreadPoolSize + makeEventStore(clientContext, clientContextImpl), + diagnosticEventSender, + new AnalyticsEventSender(LDUtil.makeHttpProperties(clientContext), + clientContext.getBaseLogger()), + makeRetryState(), clientContext.getServiceEndpoints().getEventsBaseUri(), + clientContextImpl.getDiagnosticStore(), + capacity, + commitOnCallerThread, flushIntervalMillis, + diagnosticRecordingIntervalMillis, + DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, clientContext.isInBackground(), true, // initiallyOffline - privateAttributes, - true // perContextSummarization - enable for client SDK - ); - return new DefaultEventProcessorWrapper(new DefaultEventProcessor( - eventsConfig, EventUtil.makeEventsTaskExecutor(), - Thread.NORM_PRIORITY, // note, we may want to make this configurable as it is in java-server-sdk + EventUtil.makeEventsCommitExecutor(), + EventUtil.makeDiagnosticsTaskExecutor(), clientContext.getBaseLogger() - )); + ); + } + + /** + * Builds the store this environment's events are kept in until LaunchDarkly has them. + *

+ * The mobile key and the process are both part of where it writes, so an application configured + * for several environments, or running the SDK in several processes, gets one store per + * combination and they never touch each other's events. + */ + private EventStore makeEventStore(ClientContext clientContext, ClientContextImpl impl) { + return EventStore.create( + impl.getPlatformState(), + clientContext.getMobileKey(), + capacity, + capacityBytes, + maxAgeMillis, + eventPersistence != EventPersistence.DISABLED, + clientContext.getBaseLogger()); } @Override @@ -136,76 +189,9 @@ public LDValue describeConfiguration(ClientContext clientContext) { .put("allAttributesPrivate", allAttributesPrivate) .put("diagnosticRecordingIntervalMillis", diagnosticRecordingIntervalMillis) .put("eventsCapacity", capacity) - .put("diagnosticRecordingIntervalMillis", diagnosticRecordingIntervalMillis) .put("eventsFlushIntervalMillis", flushIntervalMillis) .build(); } - - /** - * Adapter from the public component interface of EventProcessor to the internal - * implementation class from java-sdk-internal. - */ - private final class DefaultEventProcessorWrapper implements EventProcessor { - private final DefaultEventProcessor eventProcessor; - - DefaultEventProcessorWrapper(DefaultEventProcessor eventProcessor) { - this.eventProcessor = eventProcessor; - } - - @Override - public void recordEvaluationEvent( - LDContext context, - String flagKey, - int flagVersion, - int variation, - LDValue value, - EvaluationReason reason, - LDValue defaultValue, - boolean requireFullEvent, - Long debugEventsUntilDate - ) { - eventProcessor.sendEvent(new Event.FeatureRequest( - System.currentTimeMillis(), flagKey, context, flagVersion, variation, - value, defaultValue, reason, null, requireFullEvent, - debugEventsUntilDate, false)); - } - - @Override - public void recordIdentifyEvent(LDContext context) { - eventProcessor.sendEvent(new Event.Identify(System.currentTimeMillis(), context)); - } - - @Override - public void recordCustomEvent(LDContext context, String eventKey, LDValue data, Double metricValue) { - eventProcessor.sendEvent(new Event.Custom(System.currentTimeMillis(), eventKey, - context, data, metricValue)); - } - - @Override - public void setInBackground(boolean inBackground) { - eventProcessor.setInBackground(inBackground); - } - - @Override - public void setOffline(boolean offline) { - eventProcessor.setOffline(offline); - } - - @Override - public void flush() { - eventProcessor.flushAsync(); - } - - @Override - public void blockingFlush() { - eventProcessor.flushBlocking(); - } - - @Override - public void close() throws IOException { - eventProcessor.close(); - } - } } static final class HttpConfigurationBuilderImpl extends HttpConfigurationBuilder diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DeliveryRetryState.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DeliveryRetryState.java new file mode 100644 index 000000000..cf03f8dfd --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DeliveryRetryState.java @@ -0,0 +1,115 @@ +package com.launchdarkly.sdk.android; + +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * When event delivery may post again after a failure, following the SDK Retry Behavior specification as + * event durability §9 binds it. + *

+ * A failure never stops delivery, it only makes the next post wait. An ordinary one, such as a 503 or no + * network, waits from a second up to the flush interval, so it is never later than the schedule already + * would be. A refusal that is unlikely to pass soon, such as a 401, waits about eight and a half minutes, + * doubling up to an hour, rather than deleting the batch: a valid key is sometimes refused for minutes + * during an incident, and every event the fleet recorded meanwhile would be lost. A {@code Retry-After} + * sets the wait instead. One accepted delivery puts everything back. + *

+ * One per event processor, so each environment backs off on its own. Held only in memory, so a new run of + * the application posts at once. + */ +final class DeliveryRetryState { + /** The longest a {@code Retry-After} may make delivery wait, so a bad value cannot look like stopping. */ + static final long MAX_SERVER_DIRECTED_WAIT_MILLIS = TimeUnit.HOURS.toMillis(1); + + /** The ceiling once a refusal has been seen, until a delivery is accepted. */ + static final long EXTENDED_CEILING_MILLIS = TimeUnit.HOURS.toMillis(1); + + /** + * Where the attempt count goes at least to on a refusal, so its first wait is 2^9 initial delays, about + * eight and a half minutes, rather than starting from a second. + */ + static final int ATTEMPTS_AFTER_UNEXPECTED = 10; + + private final long initialDelayMillis; + private final long normalCeilingMillis; + private final long extendedCeilingMillis; + private final Random random; + + // Guarded by this. + private int attempts; + private long maxDelayMillis; + private boolean waiting; + private long nextAttemptNanos; + + /** + * @param flushIntervalMillis the normal ceiling; never below the initial delay, so a short interval does + * not make a failing service be posted to more often than once a second + */ + DeliveryRetryState(long flushIntervalMillis) { + this(DirectEventProcessor.RETRY_DELAY_MILLIS, Math.max(flushIntervalMillis, + DirectEventProcessor.RETRY_DELAY_MILLIS), EXTENDED_CEILING_MILLIS, new Random()); + } + + DeliveryRetryState(long initialDelayMillis, long normalCeilingMillis, long extendedCeilingMillis, + Random random) { + this.initialDelayMillis = initialDelayMillis; + this.normalCeilingMillis = normalCeilingMillis; + this.extendedCeilingMillis = extendedCeilingMillis; + this.random = random; + this.maxDelayMillis = normalCeilingMillis; + } + + /** The wait after a first failure, which a flush waiting on the delivery that failed sits out. */ + long initialDelayMillis() { + return initialDelayMillis; + } + + /** An accepted delivery, which is all it takes for the next failure to start from a second again. */ + synchronized void recordSuccess() { + attempts = 0; + maxDelayMillis = normalCeilingMillis; + waiting = false; + } + + /** + * @param unexpected whether the failure is unlikely to pass soon: a 4xx other than 400, 408 and 429, or + * a TLS or certificate failure + * @param retryAfterMillis the wait the response asked for, or a negative number where it gave none + * @return how long nothing may be posted + */ + synchronized long recordFailure(boolean unexpected, long retryAfterMillis) { + if (unexpected) { + attempts = Math.max(attempts + 1, ATTEMPTS_AFTER_UNEXPECTED); + maxDelayMillis = extendedCeilingMillis; + } else { + attempts++; + } + long waitMillis; + if (retryAfterMillis >= 0) { + // Every instance hears the same value at the same moment, so it is jittered to keep them from + // posting together when it runs out. + long directed = Math.min(retryAfterMillis, MAX_SERVER_DIRECTED_WAIT_MILLIS); + waitMillis = directed - (long) (random.nextDouble() * (directed / 2.0)); + } else { + // Not jittered: each instance's failures follow its own history, which already spreads them. + int doublings = Math.min(attempts - 1, 40); + waitMillis = Math.min(maxDelayMillis, initialDelayMillis << doublings); + } + waiting = true; + nextAttemptNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(waitMillis); + return waitMillis; + } + + /** How long until a post is allowed, in nanoseconds; zero once it is. */ + synchronized long remainingWaitNanos() { + if (!waiting) { + return 0; + } + long remaining = nextAttemptNanos - System.nanoTime(); + if (remaining <= 0) { + waiting = false; + return 0; + } + return remaining; + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java new file mode 100644 index 000000000..7c553bfa5 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java @@ -0,0 +1,1817 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.sdk.internal.events.DiagnosticEvent; +import com.launchdarkly.sdk.internal.events.DiagnosticStore; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.internal.events.EventSender; +import com.launchdarkly.sdk.internal.events.EventSummarizer; +import com.launchdarkly.sdk.internal.events.Sampler; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * The Android SDK's analytics event processor. + *

+ * Recording an event summarizes it immediately and, only if it has to be delivered in full, holds it. A full + * event is kept as an {@link Event} until a commit turns the run of them into bytes and writes them. So + * evaluations of untracked flags cost a counter increment, and evaluations of tracked ones cost that plus a + * list append. Nothing is encoded on the thread that evaluated. + *

+ * Where the line falls is the subject of O12 in the event-encoding research, and it falls between the events + * an application asked for by name and the ones it did not. {@code track} and {@code identify} are commit + * points, because an application reporting an error is saying this matters more than the microseconds it + * costs, and the crash it describes may be moments away. A commit encodes the whole held run, so the + * exposures leading up to the error go down with it. Evaluations get no such promise and are committed once + * {@link #PENDING_COMMIT_THRESHOLD} of them have accumulated where persistence is on, on {@code flush}, or + * when the application flushes from its own crash handler. + *

+ * Whether a commit point runs on the caller's thread is the application's choice, through + * {@link com.launchdarkly.sdk.android.integrations.EventProcessorBuilder#eventPersistence}. Only at + * {@code IMMEDIATE} can {@code track} promise the event is on disk by the time it returns. + */ +final class DirectEventProcessor implements EventProcessor { + /** + * How many full events may be held unencoded before one of them pays for a commit. + *

+ * It counts events rather than bytes because there are no bytes until the commit runs. Each commit writes + * everything it encoded at once, so this, with the commit points, is what decides how large each write to + * disk is. It sets two things at once. + *

+ * The first is how many evaluations a termination can take. That is narrower than everything a crash + * could take. An uncaught exception runs the application's handler while the process is still alive, and + * a handler that flushes commits the whole held run. So this is the window for the terminations that run + * nothing on the way out -- {@code SIGKILL}, a native crash, an ANR kill, and the system reclaiming a + * backgrounded process. A {@code track} closes it too, because it commits. + *

+ * The second is what a {@code track} usually costs, since it encodes whatever is held before returning. + * The commit executor keeps the run drained, so the common case is far below this; raising it trades that + * cost against the number of writes. + *

+ * Neither is a ceiling. Reaching this only schedules a commit, and recording does not wait for it, so + * where events arrive faster than the commit executor encodes them, or the executor is starved, the run + * grows up to the event capacity, and the next commit point encodes all of it. + */ + private static final int PENDING_COMMIT_THRESHOLD = 32; + + /** + * How long {@link #close()} spends waiting for the final delivery before it gives up and returns. + *

+ * {@code close()} runs on the caller's thread, which for an application shutting down is usually the + * main one, so this has to stay well inside the five seconds Android allows before an unanswered + * input event becomes an ANR. Two seconds is where the return on waiting longer falls off. The + * events client keeps pooled connections for only five seconds against a thirty-second flush + * interval, so this post nearly always pays a full DNS, TCP and TLS handshake. That is about three + * round trips, which two seconds covers up to a round-trip time of roughly 600 ms. A network slower + * than that is one the post is likely to fail on anyway. + *

+ * Overshooting the budget is cheaper than it looks, because the delivery is not canceled when the + * budget expires; see {@link #close()}. + */ + static final long DEFAULT_CLOSE_BUDGET_MILLIS = 2_000; + + /** + * The first wait of the backoff after a failure, as the other SDKs wait for their one retry. A flush + * waiting on the delivery that failed sits out this wait and hears the retry's outcome; any longer + * wait it does not, and is told the delivery failed. + */ + static final long RETRY_DELAY_MILLIS = 1_000; + + // Set in the constructor and never changed. + + private final EventStore store; + private final EventSender diagnosticEventSender; + private final AnalyticsEventSender analyticsEventSender; + /** When the next post may go, after a failure. Consulted before every post, and only there. */ + private final DeliveryRetryState retryState; + private final URI eventsUri; + private final DiagnosticStore diagnosticStore; + private final long flushIntervalMillis; + private final long diagnosticRecordingIntervalMillis; + private final long closeBudgetMillis; + private final ScheduledExecutorService scheduler; + /** + * Where a commit no caller waits for runs: a thread of its own, not {@link #scheduler}. + *

+ * The scheduler also delivers, and one delivery can hold it for as long as two network timeouts. A + * commit queued behind that would leave a {@code DEFERRED} {@code track} in memory for all of it, and + * let the pending run fill toward capacity meanwhile. + */ + private final ExecutorService commitExecutor; + private final ExecutorService diagnosticExecutor; + private final LDLogger logger; + + /** + * How many events the SDK will hold in total, across {@link #pending} and the store. + */ + private final int capacity; + + /** + * Whether a commit point encodes and writes before returning, rather than queuing that work. + */ + private final boolean commitOnCallerThread; + + // Guarded by commitLock. + + /** + * Held for the whole of a commit, so that only one runs at a time. + *

+ * This is what {@code track}'s guarantee rests on. Without it a commit already in flight could take the + * caller's event out of {@link #pending} before the caller got there, leaving the caller nothing to + * write and returning while those bytes were still being produced somewhere else. Waiting here instead + * means that when the call returns the event is on disk, whichever commit put it there. + *

+ * Distinct from {@link #recordLock}, which is only held long enough to hand the run and the counters + * over: encoding under this lock must not block a thread that is merely recording. + */ + private final Object commitLock = new Object(); + + /** + * The last recording a finished commit covered. Written under {@link #commitLock}, and read outside it + * so that a caller whose recording is already durable does not queue for the lock at all. + *

+ * This is what makes concurrent {@code IMMEDIATE} commit points a group commit. The first caller's + * commit takes every recording made before it took the run, including those of callers now waiting + * behind it. They find their recordings covered and return, and the next caller still uncovered + * commits everything that arrived in the meantime. Without it each waiter committed again in turn, + * and each of those commits wrote a summary of its own. + */ + private volatile long committedSequence; + + // Guarded by recordLock. + + /** + * Guards everything one recording writes: {@link #pending} and the summary counters behind + * {@link #eventBuffer}. + *

+ * One evaluation can produce a counter, a full event and a debug event, and the three have to + * land together. Taken separately, a flush landing between them splits one evaluation across two + * payloads. A {@link #close()} landing between them delivers the counter and then refuses the + * full event, leaving a summary that says an evaluation happened and no event to go with it. + *

+ * Held for the appends and the handover of the run, never across the encoding. Recording runs on + * whichever thread evaluated a flag, which on Android is usually the main one. The worst it may + * wait for is another thread's memory operation; if the encoder ran under this lock, an + * evaluation would instead wait on the dominant cost of the whole path. + */ + private final Object recordLock = new Object(); + + /** + * Full events recorded but not yet encoded, guarded by {@link #recordLock}. + *

+ * Held rather than serialized because serializing early would not make them durable: the store stages + * bytes into memory too, and only {@link EventStore#commit()} reaches the file. Both forms are equally + * lost to a crash, so the encoding may as well happen where it is cheapest. + */ + private final List pending = new ArrayList<>(); + + /** + * Numbers every recording, guarded by {@link #recordLock}, so a caller at a commit point can tell whether + * a commit has already covered it. + */ + private long recordedSequence; + + /** + * Its summary counters are guarded by {@link #recordLock}. Its serializer is reached only under + * {@link #commitLock}. + */ + private final OutboundEventBuffer eventBuffer; + + // Guarded by stateLock. + + /** + * Guards {@link #flushTask} and {@link #diagnosticTask}, and every change to {@link #inBackground} + * and {@link #offline}, so the tasks are rescheduled in the order the changes happen. Those two + * are atomic so they can be read without it. + */ + private final Object stateLock = new Object(); + private ScheduledFuture flushTask; + private ScheduledFuture diagnosticTask; + private final AtomicBoolean inBackground; + private final AtomicBoolean offline; + + // Guarded by submitLock. + + /** + * Guards the handover from running to shut down: held for the length of a submit, and by + * {@link #close()} while it queues the release of each thread's resources and stops the + * executors accepting work. Nothing blocking happens under it. + */ + private final Object submitLock = new Object(); + + /** Set under {@link #submitLock} once close() has queued the release of those resources. */ + private boolean shuttingDown = false; + + /** + * The retry queued after a delivery failed in a way that may pass, guarded by {@link #submitLock}. + * One at a time: it retries every batch still waiting, so a second would only repeat it. + */ + private ScheduledFuture pendingRetry; + + // Guarded by flushLock. + + /** + * Guards {@link #pendingFlush} and {@link #pendingFlushAnswersACaller}. Taken on a caller's + * thread and on the delivery thread, never while holding {@link #recordLock}, and nothing + * blocking happens under it. + */ + private final Object flushLock = new Object(); + + /** + * The delivery that is queued but has not started, which a flush request arriving now can wait + * on instead of queuing another. Null while nothing is queued, and cleared again as the queued + * delivery begins, which is the point past which it can no longer speak for what is recorded. + */ + private LDAwaitFuture pendingFlush; + + /** + * Whether a {@link #flushAsync()} caller, who is there to hear the outcome, is waiting on + * {@link #pendingFlush}, as opposed to only callers that discard it. + */ + private boolean pendingFlushAnswersACaller; + + // Atomic or volatile, so read and written without a lock. + + private final AtomicBoolean closed = new AtomicBoolean(false); + private volatile boolean diagnosticInitSent = false; + private final AtomicLong lastKnownPastTime = new AtomicLong(0); + /** + * Whether a previous run of the application left events behind. + *

+ * Those events are already as old as whatever happened to that run, so they are sent as soon as the + * SDK is allowed to send anything rather than at the next interval. The application in front of the + * user has moved on; an error report about a crash is not worth much thirty seconds late, and the + * process might not live that long either. + */ + private final AtomicBoolean hasEventsFromPreviousRun = new AtomicBoolean(false); + + /** + * Whether a commit is already queued, so a run of recordings past the threshold submits one task rather + * than one per event. + */ + private final AtomicBoolean commitScheduled = new AtomicBoolean(false); + + private final AtomicBoolean capacityExceeded = new AtomicBoolean(false); + + /** Whether the summarizer being full has already been reported, so it is logged once per run. */ + private final AtomicBoolean summaryContextsExceeded = new AtomicBoolean(false); + + private final AtomicLong droppedEvents = new AtomicLong(0); + + /** + * True while a diagnostic event is on its way to the service, so that a later one is dropped + * rather than queued behind it. + */ + private final AtomicBoolean diagnosticPostInFlight = new AtomicBoolean(false); + + /** + * How many events the SDK accepted and then dropped for good. + *

+ * A batch the service refuses is kept and retried, so it is not a loss, unless the refusal is a + * second 400 in a row. Otherwise what is lost is what the store never received: an event or a + * summary that could not be serialized when it was committed, or that serialized too large to + * store. A delivery that + * finds nothing to send cannot tell those apart from events that arrived, so without this it + * would report success for them. + *

+ * A count rather than a flag because a delivery can answer long after it ran: one that met a + * retryable failure answers only when its retry does, and another delivery can come and go in + * between. Each delivery takes the count as it closes its batch and answers only for the losses + * up to that mark, so it is never told about a loss that happened after it had taken everything + * it could speak for. Only the order of two values ever matters, never the difference between + * them, so a failure that lost several events need only move it once. + *

+ * Atomic because commits run on the commit executor and on callers' threads, and a caller + * waiting behind a retry is answered from a pooled thread. + */ + private final AtomicLong lostEvents = new AtomicLong(0); + + /** + * The loss count through which losses have already been reported to a {@link #flushAsync()} caller, so + * that no loss is reported twice and none goes unreported. + */ + private final AtomicLong reportedLostEvents = new AtomicLong(0); + + /** + * How many 400s in a row drop a batch. A 400 is a payload the service cannot read, which posting it + * again will not change, and a kept batch is the oldest and so the next posted: kept for good, it would + * hold every batch behind it. Two rather than one, so a 400 from a proxy or an incident costs nothing. + */ + static final int BAD_REQUESTS_BEFORE_DROPPING = 2; + + // The batch that last got a 400 and how many in a row: delivery goes oldest first and stops at a kept + // failure, so the next post is that batch's unless capacity dropped it. Held only in memory, so a new + // run of the application counts again. + private final Object badRequestLock = new Object(); + private String badRequestPayloadId; + private int badRequestsInARow; + + DirectEventProcessor( + OutboundEventBuffer eventBuffer, + EventStore store, + EventSender diagnosticEventSender, + AnalyticsEventSender analyticsEventSender, + DeliveryRetryState retryState, + URI eventsUri, + DiagnosticStore diagnosticStore, + int capacity, + boolean commitOnCallerThread, + long flushIntervalMillis, + long diagnosticRecordingIntervalMillis, + long closeBudgetMillis, + boolean initiallyInBackground, + boolean initiallyOffline, + ScheduledExecutorService scheduler, + ExecutorService commitExecutor, + ExecutorService diagnosticExecutor, + LDLogger logger + ) { + this.eventBuffer = eventBuffer; + this.store = store; + this.commitOnCallerThread = commitOnCallerThread; + this.diagnosticEventSender = diagnosticEventSender; + this.analyticsEventSender = analyticsEventSender; + this.retryState = retryState; + this.eventsUri = eventsUri; + this.diagnosticStore = diagnosticStore; + this.capacity = capacity; + this.flushIntervalMillis = flushIntervalMillis; + this.diagnosticRecordingIntervalMillis = diagnosticRecordingIntervalMillis; + this.closeBudgetMillis = closeBudgetMillis; + this.scheduler = scheduler; + this.commitExecutor = commitExecutor; + this.diagnosticExecutor = diagnosticExecutor; + this.logger = logger; + this.inBackground = new AtomicBoolean(initiallyInBackground); + this.offline = new AtomicBoolean(initiallyOffline); + + // Off the calling thread: this is on the path of LDClient.init, and it reads the filesystem. + submit(() -> { + store.preparePayloadsFromInterruptedLogs(); + reportDiscarded(store.discardExpired()); + if (store.pendingBatches().isEmpty()) { + return; + } + // Raised before offline is read, and claimed by whichever side sees both: going online writes + // offline and then claims the flag, so at least one of the two finds the other's write, and + // the claim makes it exactly one. Reading offline first left a window where going online + // found no flag yet and this then found the SDK still offline, and neither sent them. + hasEventsFromPreviousRun.set(true); + if (!offline.get() && !isStopped() && hasEventsFromPreviousRun.compareAndSet(true, false)) { + deliverPayload(); + } + }); + + synchronized (stateLock) { + updateScheduledTasks(initiallyInBackground, initiallyOffline); + } + } + + @Override + public void recordEvaluationEvent( + LDContext context, + String flagKey, + int flagVersion, + int variation, + LDValue value, + EvaluationReason reason, + LDValue defaultValue, + boolean requireFullEvent, + Long debugEventsUntilDate + ) { + try { + if (isStopped() || context == null) { + return; + } + long timestamp = System.currentTimeMillis(); + boolean debug = shouldDebugEvent(debugEventsUntilDate); + // Only built when something is kept beyond the counter. Most flags are not tracked, and for + // those an evaluation is a counter increment with nothing left behind for the collector. + // Built before the lock is taken, so that the critical section is only the writes. + Event.FeatureRequest event = requireFullEvent || debug + ? new Event.FeatureRequest(timestamp, flagKey, context, flagVersion, variation, value, + defaultValue, reason, null, requireFullEvent, debugEventsUntilDate, false) + : null; + Event debugEvent = debug ? event.toDebugEvent() : null; + boolean contextsExceeded; + boolean warnContextsExceeded; + boolean needsCommit; + synchronized (recordLock) { + if (closed.get()) { + return; + } + recordedSequence++; + contextsExceeded = !eventBuffer.summarize(timestamp, flagKey, flagVersion, variation, + value, defaultValue, context); + // Claimed under the lock that the commit resets it under, so the warning belongs to the + // run whose summarizer turned this evaluation away rather than to the next one. + warnContextsExceeded = contextsExceeded + && summaryContextsExceeded.compareAndSet(false, true); + if (requireFullEvent) { + addPending(event); + } + if (debugEvent != null) { + addPending(debugEvent); + } + needsCommit = pending.size() >= PENDING_COMMIT_THRESHOLD; + } + if (contextsExceeded) { + reportContextsExceeded(warnContextsExceeded); + } + // Deliberately no commit point. An evaluation is expected to cost what a map lookup costs, and it + // is usually the main thread doing it. The store writes these on its own thread once enough of them + // have piled up, and the next event recorded at a commit point makes them durable along with itself. + if (needsCommit) { + scheduleCommitWherePersisting(); + } + } catch (RuntimeException e) { + // This runs on the application's thread, usually inside a flag evaluation, and an + // analytics failure must not become the application's failure. Errors such as + // OutOfMemoryError are left to propagate, so the application's crash reporting sees them. + logUnexpectedError(e); + } + } + + @Override + public void recordIdentifyEvent(LDContext context) { + if (isStopped() || context == null) { + return; + } + commitAtCommitPoint(record(new Event.Identify(System.currentTimeMillis(), context))); + } + + @Override + public void recordCustomEvent(LDContext context, String eventKey, LDValue data, Double metricValue) { + if (isStopped() || context == null) { + return; + } + commitAtCommitPoint(record(new Event.Custom(System.currentTimeMillis(), eventKey, context, data, + metricValue))); + } + + /** + * Holds an event for the next commit to encode, counting it as dropped if the SDK is already full. + *

+ * Capacity is consulted before anything is encoded, so an event that will not be kept is never encoded. + * That ordering is what bounds an application re-evaluating a tracked flag in a render loop: once the + * limit is reached the cost of an evaluation falls back to its summary counter, however fast the loop runs. + * + * @return the recording's sequence number, for the commit point that follows it; 0 if it was refused + */ + long record(Event event) { + try { + long sequence; + boolean needsCommit; + synchronized (recordLock) { + // The close check that decides the outcome, as against the fast path the public record methods + // take before building the event. The commit lifts the run out under this same lock, so testing + // the flag here orders a record against close()'s final commit: either the event is in the list + // before that commit takes it, or it is refused. Tested outside the lock the two interleave, and + // an event can be left in a list that nothing will drain again. + if (closed.get()) { + return 0; + } + sequence = ++recordedSequence; + addPending(event); + needsCommit = pending.size() >= PENDING_COMMIT_THRESHOLD; + } + if (needsCommit) { + scheduleCommitWherePersisting(); + } + return sequence; + } catch (RuntimeException e) { + // As in recordEvaluationEvent: on the caller's thread, so a failure is logged, not thrown. + logUnexpectedError(e); + return 0; + } + } + + /** + * Commits at a commit point, on the caller's thread or off it as the application asked. + *

+ * Committing on the caller's thread is what lets {@code track} promise its event is on disk by the time + * it returns. Scheduling it instead keeps the encoding and the disk I/O off that thread, and the event + * is durable a moment later rather than immediately. When persistence is off that costs nothing at all, + * since there is no disk for an early commit to reach. + * + * @param through the last recording the caller needs durable: its own, or for a flush everything so far + */ + private void commitAtCommitPoint(long through) { + try { + if (commitOnCallerThread) { + commitDurablyThrough(through); + } else { + scheduleCommitWherePersisting(); + } + } catch (RuntimeException e) { + // As in recordEvaluationEvent: every commit point is on the caller's thread. + logUnexpectedError(e); + } + } + + /** + * Commits unless a finished commit already covers recording {@code through}, checked again once the lock + * is held, since the commit a caller waited behind is the one most likely to have covered it. + */ + private void commitDurablyThrough(long through) { + if (committedSequence >= through) { + return; + } + synchronized (commitLock) { + if (committedSequence >= through) { + return; + } + commitDurably(); + } + } + + /** @return the sequence number of the last recording so far */ + private long lastRecordedSequence() { + synchronized (recordLock) { + return recordedSequence; + } + } + + /** + * Queues the commit a full pending run or a commit point asks for, if there is a disk for it to reach. + *

+ * Without persistence a commit makes nothing durable. All it would do is move the encoding into the + * middle of the application's evaluations, competing with them for the CPU and splitting their + * counters across summaries. Left alone, the run waits for the flush, which encodes it anyway, and + * capacity still bounds how much is held. + */ + private void scheduleCommitWherePersisting() { + if (store.isPersisting()) { + scheduleCommit(); + } + } + + /** Queues a commit unless one is already queued, so a run of recordings asks for one rather than many. */ + private void scheduleCommit() { + // Read before the compare-and-set, which takes the line for writing even when it fails, and fails for + // every deferred track recorded while a commit is already queued. + if (!commitScheduled.get() && commitScheduled.compareAndSet(false, true)) { + try { + commitExecutor.execute(guarded(this::runScheduledCommit)); + } catch (RuntimeException e) { // shut down by close(), whose own commit takes these events + commitScheduled.set(false); + } + } + } + + /** + * Clears the scheduling flag before committing, not after, so events recorded while this runs can queue + * a commit of their own rather than waiting for the next one to be triggered. + *

+ * Does nothing once closed. {@link #close()} sets the flag before its final commit, which takes + * everything recorded up to then, and nothing is recorded after it. A commit reaching here later could + * only run against a store that has been released. + */ + private void runScheduledCommit() { + commitScheduled.set(false); + if (closed.get()) { + return; + } + commitDurably(); + } + + /** + * Encodes everything recorded since the last commit and makes it survive the process. + *

+ * This is the only place events are serialized. It runs on the commit executor when the pending run + * reaches {@link #PENDING_COMMIT_THRESHOLD}, and on the caller's thread for {@code flush}, where the + * caller has asked to wait for exactly this. + *

+ * The evaluations counted since the last commit are written too, because a summary is only worth having + * if it covers the evaluations that led up to whatever is about to happen. + */ + private void commitDurably() { + synchronized (commitLock) { + List run; + List summaries; + long through; + // Both taken at once, so that an evaluation's counter and its full event are staged by the + // same commit. Taken separately, a commit landing between the two appends one evaluation makes + // stages the counter and leaves the event for the next one -- or, at close, for none at all. + synchronized (recordLock) { + through = recordedSequence; + run = pending.isEmpty() ? Collections.emptyList() : new ArrayList<>(pending); + pending.clear(); + summaries = eventBuffer.takeSummaries(); + summaryContextsExceeded.set(false); + // Reserved in the same critical section the run leaves pending in, so capacity counts + // it in one place or the other throughout the encoding. Each staged event uses up its + // reservation as it arrives, so none is counted twice either. + store.reserve(run.size()); + } + try { + stageRun(run); + } finally { + store.releaseReservations(); + } + stageSummaries(summaries); + if (eventBuffer.takeSerializationFailure()) { + lostEvents.incrementAndGet(); + } + store.commit(); + committedSequence = through; + } + } + + /** + * Encodes a run of events as one run and stages the bytes. + *

+ * The run was taken under {@link #recordLock} and is encoded outside it, so recording does not wait on + * the encoder. Staging bypasses capacity because the decision to keep these events was already made in + * {@link #record}, and refusing them here would drop events the SDK has counted as accepted. The + * reservation taken with the run is what keeps them counted until they are staged. The byte capacity + * can still refuse one, since only its encoded size says whether it fits, and an event refused then is + * lost rather than dropped. + *

+ * Requires {@link #commitLock}: two threads staging separate runs would stage them in whichever order + * they finished encoding, which is not the order they were recorded in. + */ + private void stageRun(List run) { + for (byte[] serialized : eventBuffer.serializeAll(run)) { + if (!store.stageReserved(serialized)) { + reportRefusedByStore("event", serialized); + } + } + } + + /** + * Turns the evaluation counters already taken into summary events in the store. + */ + private void stageSummaries(List summaries) { + for (byte[] summary : eventBuffer.serializeSummaries(summaries)) { + // Bypassing capacity: a summary is not a new event, it is the record of evaluations already + // counted, and dropping it would lose all of them at once. Only the byte capacity, which keeps + // summaries staged while delivery keeps failing from growing without bound, can refuse it. + if (!store.stage(summary, true)) { + reportRefusedByStore("summary event", summary); + } + } + } + + /** + * Records the loss of something the store would not take. Capacity is bypassed on both paths, so + * what is left is a frame too large to store, or one the byte capacity has no room for, and the SDK + * had already accepted it either way. + */ + private void reportRefusedByStore(String what, byte[] serialized) { + if (serialized.length > EventLogFormat.MAX_FRAME_SIZE) { + logger.error("Dropping {} of {} bytes, too large to store", what, serialized.length); + } else { + logger.error("Dropping {} of {} bytes, which the event store has no room for", what, + serialized.length); + } + lostEvents.incrementAndGet(); + } + + /** + * Records a batch lost for good. The delivery that lost it fails, but a delivery nobody waits on has no + * caller to tell: the periodic one, a {@code flush()}, or the one going online starts. So the loss is + * also counted for the next {@link #flushAsync()} to hear about. A caller waiting on this delivery can + * then hear about it from the next one too. + */ + private void reportBatchLost(EventStore.Batch batch) { + lostEvents.addAndGet(batch.eventCount); + } + + /** + * Records stored events discarded for being older than the maximum age. + *

+ * Counted as dropped rather than lost: a flush is about the caller's events, and the ones discarded + * are long past anything a caller is waiting on. + */ + private void reportDiscarded(int events) { + if (events <= 0) { + return; + } + logger.debug("Discarded {} stored event(s) older than the maximum age", events); + droppedEvents.addAndGet(events); + } + + /** + * Claims the losses up to a mark on behalf of one {@link #flushAsync()} caller. + * + * @param mark the loss count as of the close of that caller's batch + * @return true if a loss at or before the mark had not yet been reported to anyone, in which + * case this caller is the one reporting it + */ + private boolean takeEventsLost(long mark) { + // A loop rather than getAndUpdate, which this SDK cannot use below API 24. + while (true) { + long reported = reportedLostEvents.get(); + if (mark <= reported) { + return false; + } + if (reportedLostEvents.compareAndSet(reported, mark)) { + return true; + } + } + } + + /** + * Holds one event for the next flush, unless it was sampled out or there is no room. Requires + * {@link #recordLock}. + */ + private void addPending(Event event) { + // Ahead of the capacity check, because an event the SDK was never going to send is not a + // loss and must not be counted as one. Sampling and capacity are different reasons not to + // keep an event, and only the second is one the SDK owes anyone a count of. + if (!Sampler.shouldSample(event.getSamplingRatio())) { + return; + } + // The bytes are checked here too, before the event is encoded, so that one there is no room for + // costs nothing; whether it fits is decided when it is staged. + if (!store.admitsEvent(pending.size(), capacity)) { + if (capacityExceeded.compareAndSet(false, true)) { + logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); + } + droppedEvents.incrementAndGet(); + return; + } + // Read before written: a write on every event would contend for the line with nothing to change. + if (capacityExceeded.get()) { + capacityExceeded.set(false); + } + pending.add(event); + } + + /** + * Counts an evaluation the summarizer turned away, which it does when counting it would have meant + * holding a context beyond the configured capacity. + *

+ * A refused evaluation is a loss in the same sense a refused event is -- nothing later reconstructs + * a counter -- so it is reported the same way, through the dropped count diagnostics carry. + */ + private void reportContextsExceeded(boolean warn) { + // Warned once per delivery run rather than once per process: the summarizer's contexts are + // cleared with each run, so a later overflow is a new one worth hearing about. + if (warn) { + logger.warn("Exceeded the number of contexts that can be summarized at once." + + " Increase capacity to avoid dropping evaluations."); + } + droppedEvents.incrementAndGet(); + } + + /** + * @return the number of events dropped for capacity since this was last called, counting + * evaluations the summarizer turned away + */ + long getAndClearDroppedCount() { + return droppedEvents.getAndSet(0); + } + + @Override + public void setInBackground(boolean inBackground) { + synchronized (stateLock) { + if (this.inBackground.getAndSet(inBackground) == inBackground) { + return; + } + updateScheduledTasks(inBackground, offline.get()); + } + } + + @Override + public void setOffline(boolean offline) { + synchronized (stateLock) { + if (this.offline.getAndSet(offline) == offline) { + return; + } + updateScheduledTasks(inBackground.get(), offline); + } + } + + @Override + public void flush() { + if (isStopped()) { + return; + } + commitBeforeDelivery(); + queueDelivery(false); + } + + @Override + public void blockingFlush() { + if (isStopped()) { + return; + } + commitBeforeDelivery(); + Future delivery = queueDelivery(false); + try { + delivery.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + logUnexpectedError(e.getCause() == null ? e : e.getCause()); + } + } + + @Override + public Future flushAsync() { + if (isStopped()) { + return new LDSuccessFuture<>(false); + } + queueCommitBeforeDelivery(); + return queueDelivery(true); + } + + /** + * The commit point {@link #flush()} and {@link #blockingFlush()} start with, before anything is + * handed to the events thread. + *

+ * Written on the caller's thread wherever events persist, not only where the application asked for + * that as it does for {@code track} and {@code identify}. A flush is what an application calls when it + * is about to end the process on purpose, and the process may end before a commit it only queued + * has run. Where nothing persists, a commit makes nothing durable, and is left to the delivery. + *

+ * It does not wait for the events thread, which can be held by a post or by a retry long after the + * caller has moved on. The delivery commits again when it runs, but that is too late to be the only + * commit. + */ + private void commitBeforeDelivery() { + try { + if (commitOnCallerThread || store.isPersisting()) { + commitDurablyThrough(lastRecordedSequence()); + } + } catch (RuntimeException e) { + // As in recordEvaluationEvent: every commit point is on the caller's thread. + logUnexpectedError(e); + } + } + + /** + * The commit point {@link #flushAsync()} starts with: the same commit as {@link #commitBeforeDelivery}, + * queued on the commit executor rather than written on the caller's thread. + *

+ * A caller of {@code flushAsync} has a deadline, which is how {@code flushAndWait} bounds the whole call, + * and it can only enforce one once it holds the future. Each step of a commit can block for as long as + * the device takes: on {@link #commitLock} behind a commit or a delivery that stalled on the disk, on the + * store's I/O lock, and on its own writing. Committed here, any of them ran before the deadline could + * apply, so a stalled flash write outlasted the timeout of the crash handler on its way out. + *

+ * Queued rather than left to the delivery, because the events thread can be held by a post or by a + * retry long after the caller has stopped waiting, and {@code flushAndWait} is for a caller that is about + * to be killed. The commit executor is not held by the network. The future still answers for the write: + * the delivery commits again before it sends, so a {@code true} means the events were written first. A + * commit that outlasts the caller's wait is left to finish, as {@link #close()} leaves its own. + */ + private void queueCommitBeforeDelivery() { + try { + if (commitOnCallerThread || store.isPersisting()) { + long through = lastRecordedSequence(); + commitExecutor.execute(guarded(() -> { + // As in runScheduledCommit: close() writes everything recorded before it with its own + // commit, and a commit reaching here later could only run against a released store. + if (!closed.get()) { + commitDurablyThrough(through); + } + })); + } + } catch (RuntimeException e) { + // Shut down by close(), whose own commit takes these events; or as in recordEvaluationEvent, + // a failure on the caller's thread is logged rather than thrown. + if (!closed.get()) { + logUnexpectedError(e); + } + } + } + + /** + * Queues a delivery, or hands back one that is already queued and has not started. + *

+ * A delivery that has not started yet will send everything recorded before it starts. That + * includes the caller's events, so waiting on it is as good as queuing a new one. Without this, + * flushes arriving faster than a post completes each queue their own, and the one that matters -- + * the {@code flushAndWait} at shutdown -- waits behind all of them. + * + * @param answersACaller true if the caller will hear the outcome, so that the delivery reports + * any events lost up to the batch it closes, and false if the caller discards it + */ + private Future queueDelivery(boolean answersACaller) { + synchronized (flushLock) { + if (pendingFlush != null) { + pendingFlushAnswersACaller |= answersACaller; + return pendingFlush; + } + LDAwaitFuture result = new LDAwaitFuture<>(); + if (submit(() -> runDelivery(result)) == null) { + // Shutting down, so there is no thread left to deliver on and nothing will be sent. + return new LDSuccessFuture<>(false); + } + pendingFlush = result; + pendingFlushAnswersACaller = answersACaller; + return result; + } + } + + /** + * Runs one delivery on behalf of every flush request that joined it, and tells them all how it + * went. + *

+ * A caller who will hear the answer is told no if the delivery fails. It is also told no if events + * were dropped for good up to the batch this delivery closed and nobody has been told yet, because a + * delivery that finds nothing to send is not evidence that everything recorded arrived. + */ + private void runDelivery(LDAwaitFuture result) { + boolean answersACaller = false; + synchronized (flushLock) { + // Requests arriving from here on need a delivery of their own: this one is about to take + // the buffer, and what it takes is all it can speak for. + if (pendingFlush == result) { + pendingFlush = null; + answersACaller = pendingFlushAnswersACaller; + pendingFlushAnswersACaller = false; + } + } + DeliveryOutcome outcome; + try { + outcome = commitAndDeliverReportingOutcome(); + } catch (Throwable t) { + // Caught here rather than left to guarded(), because a caller is waiting on the future + // and completing it matters more than the stack reaching the executor. + logUnexpectedError(t); + outcome = DeliveryOutcome.CLOSED_NOTHING; + } + if (outcome.retry == null) { + answer(result, outcome.delivered, answersACaller, outcome.lostEventsMark); + } else { + completeWhenRetryEnds(result, outcome, answersACaller); + } + } + + /** + * Answers the waiters when the queued retry does, rather than telling them a batch failed when + * it is about to be sent again. Waiting happens on a pooled thread because the retry runs a + * second from now on this one, which must be free to take it. + */ + private void completeWhenRetryEnds(LDAwaitFuture result, DeliveryOutcome outcome, + boolean answersACaller) { + LDAwaitFuture retried = LDFutures.fromFuture(outcome.retry); + retried.addListener(() -> { + boolean delivered = false; + try { + delivered = Boolean.TRUE.equals(retried.get()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + // Reported as not delivered. + } + // The mark the delivery took, not one taken now: a loss during the wait belongs to + // whichever delivery closes the batch after it, not to this one. + answer(result, delivered, answersACaller, outcome.lostEventsMark); + }); + } + + /** + * Completes a delivery's future. A caller who will hear the answer is told no if this delivery + * failed, or if events the SDK accepted were lost for good up to the mark it took and no caller + * has been told so yet. + */ + private void answer(LDAwaitFuture result, boolean delivered, boolean answersACaller, + long lostEventsMark) { + if (answersACaller) { + delivered &= !takeEventsLost(lostEventsMark); + } + result.set(delivered); + } + + @Override + public void close() throws IOException { + if (!closed.compareAndSet(false, true)) { + return; + } + synchronized (stateLock) { + updateScheduledTasks(inBackground.get(), offline.get()); + } + // One budget for the whole of close(), which runs on the caller's thread. For an application + // shutting down that is usually the main one, so it has to stay well inside the five seconds + // Android allows before an unanswered input event becomes an ANR. The budget bounds the caller, not + // the work: whatever outlasts it is left to finish on its own thread. + long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(closeBudgetMillis); + + // Staging and writing come first, and get the budget first. A caller who closed the client and + // then let the process end has to find those events on the next run, and that is worth more + // than the post below. A batch on disk is delivered next run, while an event never staged is + // gone whether the post would have succeeded or not. Where nothing persists there is nothing to + // write: the final delivery encodes what it sends, and offline nothing is sent. + Future finalCommit = store.isPersisting() ? commitWithinBudget(deadlineNanos) : null; + + // Then deliver what is buffered, with whatever is left. This waits rather than firing and + // forgetting because it is this run's last chance to send them. While offline that chance is + // not taken: offline is the application telling the SDK to stay off the network, and shutting + // down does not revoke that. The commit above has written those events down where persistence + // is on, so they go out on a later run; where it is off they are discarded. + Callable finalDelivery = this::deliverPayloadAndRetryLater; + Future delivery = submit(finalDelivery); + if (delivery != null) { + try { + awaitDelivery(delivery, Math.max(0, deadlineNanos - System.nanoTime())); + } catch (TimeoutException e) { + // Deliberately not canceled. The batch is closed and on its way, so interrupting now + // would make the loss certain, while leaving it to run costs nothing: the scheduler + // thread is a daemon, and returning from close() does not end an Android process. + logger.warn("Gave up waiting for the final event delivery after {}ms;" + + " it continues in the background", closeBudgetMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + logUnexpectedError(e.getCause() == null ? e : e.getCause()); + } + } + // The store and the senders, by contrast, are released on whichever thread uses them, after + // whatever is still in flight there. Closing them inline would pull them out from under a + // delivery, or a diagnostic post, that we just decided not to wait for. shutdown() then + // refuses new work while letting what is already queued finish -- unlike shutdownNow(), which + // would interrupt those and strand the futures of anything it discarded. + // + // Queueing and shutting down are one step under submitLock, so that a flush cannot land in + // between them and put a delivery behind the release, where it would find the store closed. + synchronized (submitLock) { + shuttingDown = true; + if (pendingRetry != null) { + // Not interrupted if it is already posting: it checks shuttingDown before it starts, and + // one that got past that runs ahead of the release queued below on the same thread. + pendingRetry.cancel(false); + pendingRetry = null; + } + // The store is released only once the final commit is through with it, which may be after + // the budget if the disk stalled. That waits here, on the scheduler thread, and never on the + // caller's. + try { + scheduler.submit(guarded(() -> { + awaitQuietly(finalCommit); + releaseDeliveryResources(); + })); + } catch (RuntimeException e) { // the executor was shut down under us + releaseDeliveryResources(); + } + queueRelease(diagnosticExecutor, this::releaseDiagnosticResources); + scheduler.shutdown(); + diagnosticExecutor.shutdown(); + // Commits already queued there still run, and find the processor closed. + commitExecutor.shutdown(); + } + } + + /** + * Commits everything recorded so far for close(), waiting for it no longer than the budget allows. + *

+ * On the commit executor rather than the caller's thread, because each step of a commit can block + * for as long as the device takes. {@link #commitLock} may be held by a commit or a delivery that + * stalled on the disk, and the store's I/O lock by the disk I/O it stalled on. This commit's own + * writing can stall the same way. Committing on the caller's thread was bounded by none of them, so + * a stalled flash write turned into an ANR on the main thread. + *

+ * A commit that outlasts the budget is left to finish, for the same reason as the delivery. The + * executor thread is a daemon, and returning from close() does not end the process. Writing cut off + * partway leaves a torn frame that the next run reads past, not events lost. + * + * @return the commit, for the store's release to wait on, or null if it could not be queued + */ + private Future commitWithinBudget(long deadlineNanos) { + Future commit; + try { + commit = commitExecutor.submit(guarded(this::commitDurably)); + } catch (RuntimeException e) { // shut down under us; the final delivery still commits first + return null; + } + try { + commit.get(Math.max(0, deadlineNanos - System.nanoTime()), TimeUnit.NANOSECONDS); + } catch (TimeoutException e) { + logger.warn("Gave up waiting for the final event commit after {}ms;" + + " it continues in the background", closeBudgetMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + logUnexpectedError(e.getCause() == null ? e : e.getCause()); + } + return commit; + } + + /** Waits for a task to finish however it ends. Only for threads that are not the caller's. */ + private static void awaitQuietly(Future task) { + if (task == null) { + return; + } + try { + task.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + // Logged where it ran. + } + } + + /** Queues a release on the thread that owns those resources, or runs it here if that thread has gone. */ + private void queueRelease(ExecutorService executor, Runnable release) { + try { + executor.submit(guarded(release)); + } catch (RuntimeException e) { // the executor was shut down under us + release.run(); // so nothing can still be using them + } + } + + /** What the delivery thread owns: the store it drains and the sender it posts batches through. */ + private void releaseDeliveryResources() { + store.close(); + closeQuietly(analyticsEventSender); + } + + /** What the diagnostics thread owns: the sender nothing else posts through. */ + private void releaseDiagnosticResources() { + closeQuietly(diagnosticEventSender); + } + + private void closeQuietly(Closeable closeable) { + try { + closeable.close(); + } catch (IOException e) { + logUnexpectedError(e); + } + } + + /** + * Serializes and sends everything buffered, for the periodic flush, which has nobody waiting to + * find out how it went. It is a fixed-delay series, so a run is only ever scheduled once the one + * before it has finished and these cannot pile up the way requested flushes could. + */ + private void deliverPayload() { + deliverPayloadAndRetryLater(); + } + + /** + * Writes everything accepted so far, then delivers. + *

+ * This commit looks redundant with the one in {@link #commitAndCloseBatch()}, and online it is: the + * second finds nothing pending and returns. It is kept for the case where it is the only one. A + * delivery refused for being offline returns before {@code commitAndCloseBatch}. Without + * this, a flush while offline would leave everything recorded in memory, and a process that then + * died would lose it -- the case persistence exists for. + *

+ * Only where events persist. Otherwise the commit makes nothing durable: online, the one in + * {@code commitAndCloseBatch} is the drain, and offline it would encode events that cannot be sent + * into bytes that wait in memory exactly as the events would have. + */ + private DeliveryOutcome commitAndDeliverReportingOutcome() { + if (store.isPersisting()) { + commitDurably(); + } + return deliverPayloadAndRetryLater(); + } + + /** + * How a delivery went, or the retry that will say, for a caller waiting to find out. + */ + private static final class DeliveryOutcome { + /** For a delivery that closed no batch, so there are no losses it can speak for. */ + static final DeliveryOutcome CLOSED_NOTHING = new DeliveryOutcome(false, null, 0); + + final boolean delivered; + /** + * Non-null when a batch failed and the retry of every batch is queued to run after the first wait of + * the backoff, which is as long as a caller waits to hear how it went. + */ + final Future retry; + /** + * The loss count as of the close of this delivery's batch, which is as far as it can answer + * for; see {@link #takeEventsLost(long)}. + */ + final long lostEventsMark; + + private DeliveryOutcome(boolean delivered, Future retry, long lostEventsMark) { + this.delivered = delivered; + this.retry = retry; + this.lostEventsMark = lostEventsMark; + } + } + + /** + * Waits for a delivery and, when it queued a retry, for that too, within one budget. + * + * @param timeoutNanos how long to wait in all + */ + private void awaitDelivery(Future delivery, long timeoutNanos) + throws InterruptedException, ExecutionException, TimeoutException { + long deadline = System.nanoTime() + timeoutNanos; + DeliveryOutcome outcome = delivery.get(timeoutNanos, TimeUnit.NANOSECONDS); + if (outcome != null && outcome.retry != null) { + outcome.retry.get(Math.max(0, deadline - System.nanoTime()), TimeUnit.NANOSECONDS); + } + } + + /** + * Delivers every batch waiting and then what has been committed since, queuing one retry for when the + * backoff allows if a batch is still waiting. + *

+ * The retry is handed back for the caller to wait on only where it comes after the first wait. Any + * longer one, a refusal's or one grown through an outage, is the backoff saying not to post for a + * while, and a flush that sat it out would hold its caller for minutes to say what it can say now. + *

+ * Runs on the scheduler thread for a requested or periodic delivery, and on the caller's thread + * at close. Deliveries can therefore overlap: a retry runs a second behind the delivery that + * queued it, and the one at close does not wait for the scheduler. Each closes a batch of its + * own under {@link #commitLock}, so no event is committed into two batches. A batch that two of + * them both send carries the same payload ID each time, and LaunchDarkly discards the repeat + * rather than counting the events twice. + */ + private DeliveryOutcome deliverPayloadAndRetryLater() { + if (offline.get()) { + return DeliveryOutcome.CLOSED_NOTHING; + } + + DeliveryPass pass = deliverWaitingThenNew(); + if (pass.outcome == BatchesOutcome.RETRYABLE) { + // A batch kept for a reason other than the network, such as a read that failed, has no wait of its + // own, and is retried after the first one. + long waitNanos = retryState.remainingWaitNanos(); + long delayNanos = waitNanos > 0 ? waitNanos + : TimeUnit.MILLISECONDS.toNanos(retryState.initialDelayMillis()); + Future retry = scheduleRetry(delayNanos); + boolean awaited = delayNanos <= TimeUnit.MILLISECONDS.toNanos(retryState.initialDelayMillis()); + return new DeliveryOutcome(false, awaited ? retry : null, pass.lostEventsMark); + } + return new DeliveryOutcome(pass.outcome == BatchesOutcome.DELIVERED, null, pass.lostEventsMark); + } + + /** How one pass over the batches went, and the loss count it can answer for. */ + private static final class DeliveryPass { + final BatchesOutcome outcome; + /** As {@link DeliveryOutcome#lostEventsMark}. */ + final long lostEventsMark; + + DeliveryPass(BatchesOutcome outcome, long lostEventsMark) { + this.outcome = outcome; + this.lostEventsMark = lostEventsMark; + } + } + + /** + * Delivers the batches already waiting and, only once none of them is left blocked, closes what has been + * committed into a batch of its own and delivers that. + *

+ * Closing first would add a batch for every pass a failure lasts: every flush and every periodic + * delivery of an outage would leave one more file, or one more batch held in memory, each with the + * summary its commit wrote. Held back, what is committed meanwhile goes on into the open log, which is + * closed once it is full or once the batches ahead of it are gone, so an outage costs a batch per + * {@link EventLogFormat#MAX_BATCH_SIZE} rather than one per pass. It is committed all the same, since + * a pass is a commit point. + */ + private DeliveryPass deliverWaitingThenNew() { + BatchesOutcome waiting = deliverPendingBatches(); + if (waiting == BatchesOutcome.RETRYABLE) { + return new DeliveryPass(waiting, commitLeavingTheLogOpen()); + } + Close close = commitAndCloseBatch(); + BatchesOutcome closed = deliverListedBatches(); + if (closed == BatchesOutcome.RETRYABLE || !close.closedEverything) { + // Events left in a log that would not close are still on the disk, for the next close to try again + // with; this pass cannot say they got out. + return new DeliveryPass(BatchesOutcome.RETRYABLE, close.lostEventsMark); + } + boolean delivered = waiting == BatchesOutcome.DELIVERED && closed == BatchesOutcome.DELIVERED; + return new DeliveryPass(delivered ? BatchesOutcome.DELIVERED : BatchesOutcome.FAILED, close.lostEventsMark); + } + + /** What {@link #commitAndCloseBatch()} did. */ + private static final class Close { + /** As {@link DeliveryOutcome#lostEventsMark}. */ + final long lostEventsMark; + /** Whether the close took every committed event into a batch. */ + final boolean closedEverything; + + Close(long lostEventsMark, boolean closedEverything) { + this.lostEventsMark = lostEventsMark; + this.closedEverything = closedEverything; + } + } + + /** + * Commits everything accepted so far and closes it into a batch to deliver, taking the loss + * count as of that close. + *

+ * Both under {@link #commitLock}, for two reasons. A commit landing between the two would stage + * events into the log this batch is about to close. That splits what one commit staged across + * two payloads: an evaluation's counter in this batch and its event in the next. And the mark + * has to be the count as of the close, because that is what this delivery can answer for: a loss + * after it is one whose events were never in this batch. + *

+ * A requested flush has usually just committed in {@link #commitAndDeliverReportingOutcome()}, so this + * commit often finds nothing to do. It cannot be the one left out: the periodic delivery, the one + * going online starts, the one recovery starts, and the final one at close all reach here without + * that earlier commit. + * + * @return the loss count as of the close, for {@link #takeEventsLost(long)}, and whether the close took + * every committed event into a batch; both under the lock, so they cover this close and nothing a later + * commit does + */ + private Close commitAndCloseBatch() { + synchronized (commitLock) { + commitDurably(); + store.closeBatch(); + return new Close(lostEvents.get(), !store.holdsUnclosedEvents()); + } + } + + /** + * {@link #commitAndCloseBatch()} for a pass that found an earlier batch blocked, and so closes nothing. + * The mark is as of this commit, the last point anything the caller recorded was staged. + * + * @return the loss count as of the commit, for {@link #takeEventsLost(long)} + */ + private long commitLeavingTheLogOpen() { + synchronized (commitLock) { + commitDurably(); + return lostEvents.get(); + } + } + + private enum BatchesOutcome { DELIVERED, FAILED, RETRYABLE } + + /** + * Makes one attempt at every batch waiting, oldest first: the others are deliveries an earlier attempt + * did not finish, or that a previous run of the application never got to start. + *

+ * Stops at the first failure that may pass. The batches behind it would most likely meet the same + * network, and the retry takes all of them again. + */ + private BatchesOutcome deliverPendingBatches() { + reportDiscarded(store.discardExpired()); + return deliverListedBatches(); + } + + /** + * {@link #deliverPendingBatches()} without the discard, for a second pass moments after the first. + *

+ * Where the directory could not be listed, delivers the batches the store last counted and answers + * {@link BatchesOutcome#RETRYABLE}: a batch the listing could not see is still on the disk, and the + * listing may work next time. + */ + private BatchesOutcome deliverListedBatches() { + EventStore.Listing listing = store.listPendingBatches(); + boolean allDelivered = true; + for (EventStore.Batch batch : listing.batches) { + if (offline.get()) { + return BatchesOutcome.FAILED; + } + BatchesOutcome outcome = deliver(batch); + if (outcome == BatchesOutcome.RETRYABLE) { + return outcome; + } + allDelivered &= outcome == BatchesOutcome.DELIVERED; + } + if (!listing.complete) { + return BatchesOutcome.RETRYABLE; + } + return allDelivered ? BatchesOutcome.DELIVERED : BatchesOutcome.FAILED; + } + + /** + * Queues the retry on the scheduler rather than sleeping for it there, which would hold every flush, + * and close()'s final delivery, behind the wait. + * + * @param delayNanos how long until the backoff allows the next post + * @return the retry, a retry already queued, or null if the processor is shutting down + */ + private Future scheduleRetry(long delayNanos) { + synchronized (submitLock) { + if (shuttingDown) { + return null; + } + if (pendingRetry != null && !pendingRetry.isDone()) { + return pendingRetry; + } + try { + pendingRetry = scheduler.schedule(this::retryPendingBatches, delayNanos, TimeUnit.NANOSECONDS); + } catch (RuntimeException e) { // the executor was shut down under us + return null; + } + logger.warn("Will retry posting events in {}ms after a failure", + TimeUnit.NANOSECONDS.toMillis(delayNanos)); + return pendingRetry; + } + } + + /** + * The one retry. A failure here queues no other; the batches wait on disk, or in memory, for the next + * flush, which posts once the backoff allows. Until it does, the periodic one each time queues this + * retry for when it will. + *

+ * A pass like the delivery's rather than only the batches it left: the delivery may have closed + * nothing, behind a batch that was blocked, and a {@code true} has to mean that the events the + * caller was waiting on went too. + */ + private boolean retryPendingBatches() { + synchronized (submitLock) { + if (shuttingDown) { + // close() has queued the release of the store and the sender behind this. + return false; + } + } + if (offline.get()) { + return false; + } + try { + return deliverWaitingThenNew().outcome == BatchesOutcome.DELIVERED; + } catch (RuntimeException e) { + logUnexpectedError(e); + return false; + } + } + + /** + * @return DELIVERED if the batch arrived, or held nothing left to send; FAILED if events were lost, + * whether the batch could not be read, arrived short, or was refused as malformed + * {@link #BAD_REQUESTS_BEFORE_DROPPING} times in a row; RETRYABLE if it is kept to be sent again, + * which is every other failure to post it, and a post the backoff did not allow + */ + private BatchesOutcome deliver(EventStore.Batch batch) { + if (retryState.remainingWaitNanos() > 0) { + // Not an attempt, so it moves nothing in the backoff. + return BatchesOutcome.RETRYABLE; + } + EventLogFormat.Payload payload; + try { + payload = store.payload(batch); + } catch (EventStore.UnreadableBatchException e) { + // Lost: nothing will ever send it, so it is removed, the rest are still attempted, and the + // caller hears that this pass lost events. + logger.error("Dropping {} stored event(s) this version cannot read", batch.eventCount); + store.remove(batch); + reportBatchLost(batch); + return BatchesOutcome.FAILED; + } catch (IOException e) { + // Kept: the file is still there, and a read that failed may not fail next time. + logger.warn("Could not read stored events: {}", LogValues.exceptionSummary(e)); + return BatchesOutcome.RETRYABLE; + } + if (payload == null) { + // Already delivered by another process of this application, or holding no events. Either way + // nothing is lost and there is nothing to keep. + store.remove(batch); + return BatchesOutcome.DELIVERED; + } + // Damage after the first frame: what came before it is sent, and the rest is gone with the file. Counted + // once the batch is removed rather than now, since one kept for a retry is read, and found short, again. + int leftOut = Math.max(0, batch.eventCount - payload.eventCount); + if (diagnosticStore != null) { + diagnosticStore.recordEventsInBatch(payload.eventCount); + } + store.startedSending(batch); + try { + AnalyticsEventSender.Result result = analyticsEventSender.sendBatch(payload, payload.eventCount, + batch.payloadId, eventsUri); + if (!result.success) { + retryState.recordFailure(result.unexpected, result.retryAfterMillis); + if (countBadRequest(batch, result.status) >= BAD_REQUESTS_BEFORE_DROPPING) { + // The rest are still attempted, once the backoff allows. + logger.error("Dropping {} event(s) the service refused as malformed {} times in a row", + batch.eventCount, BAD_REQUESTS_BEFORE_DROPPING); + store.remove(batch); + reportBatchLost(batch); + return BatchesOutcome.FAILED; + } + // Kept for any other failure, a 401 included: a valid key is sometimes refused for minutes, and + // deleting on a refusal would lose every event recorded meanwhile. The backoff decides when the + // retry, the next flush or the next run of the application sends it again. + return BatchesOutcome.RETRYABLE; + } + countBadRequest(batch, result.status); + retryState.recordSuccess(); + if (result.serverTime != null) { + recordPastTime(result.serverTime.getTime()); + } + store.remove(batch); + if (leftOut > 0) { + logger.error("Dropped {} damaged stored event(s) from a batch it delivered", leftOut); + lostEvents.addAndGet(leftOut); + return BatchesOutcome.FAILED; + } + return BatchesOutcome.DELIVERED; + } catch (Exception e) { + logUnexpectedError(e); + return BatchesOutcome.FAILED; + } finally { + // Counted again unless the batch was removed above. + store.stoppedSending(batch); + } + } + + /** + * @return how many 400s in a row this batch has had, counting this response; zero for any other + * status, which ends the run, and once the batch is dropped + */ + private int countBadRequest(EventStore.Batch batch, int status) { + synchronized (badRequestLock) { + if (status != 400) { + badRequestPayloadId = null; + badRequestsInARow = 0; + return 0; + } + if (!batch.payloadId.equals(badRequestPayloadId)) { + badRequestPayloadId = batch.payloadId; + badRequestsInARow = 0; + } + badRequestsInARow++; + int count = badRequestsInARow; + if (count >= BAD_REQUESTS_BEFORE_DROPPING) { + badRequestPayloadId = null; + badRequestsInARow = 0; + } + return count; + } + } + + /** + * Posts an event that has already been built. Requires the caller to have checked + * {@link #diagnosticsSuspended()}, because for the periodic event building it is what consumes + * the period. + */ + private void sendDiagnosticEvent(DiagnosticEvent diagnosticEvent, boolean isInit) { + try { + byte[] data = diagnosticEvent.getJsonValue().toJsonString() + .getBytes(StandardCharsets.UTF_8); + handleResponse(diagnosticEventSender.sendDiagnosticEvent(data, eventsUri)); + if (isInit) { + // Attempted, not delivered. A failed post gives back an unsuccessful Result rather + // than throwing, so this marks the init event done either way and the process never + // retries it. That matches DefaultEventProcessor, which is the behavior to keep: + // diagnostics are best-effort telemetry about the SDK, and a retry that outlived + // its own init would describe a configuration the application has moved on from. + diagnosticInitSent = true; + } + } catch (Exception e) { + logUnexpectedError(e); + } + } + + private void sendDiagnosticStats() { + if (diagnosticsSuspended() || diagnosticStore == null || !claimDiagnosticPost()) { + return; + } + DiagnosticStore store = diagnosticStore; + postDiagnostic(() -> { + // The check that decides the outcome, and it has to come before the event is built rather + // than after. createEventAndReset hands back the period and clears it: the stream init + // records, the events-in-batch count and the period start all move. getAndClearDroppedCount + // does the same for the dropped count. This runs on the diagnostics thread, which may have + // been busy with an earlier post for as long as that post took, so the state can easily + // have changed since the checks above. Bailing out after the event was built would discard + // a period outright and leave the next event describing a window that begins after the + // reset. Bailing out here leaves everything where it is, for the next period to carry. + if (diagnosticsSuspended()) { + return; + } + sendDiagnosticEvent(store.createEventAndReset(getAndClearDroppedCount(), 0), false); + }); + } + + /** + * Takes the diagnostic posting thread, or reports that the last event is still on it. + *

+ * A new event is dropped rather than queued behind the old one. Diagnostics are best-effort + * telemetry, and a post can take tens of seconds against a network that never answers. Queuing + * would let an outage accumulate events describing an SDK state the application has since moved + * past. The same reasoning makes a failed init event final rather than retried. + */ + private boolean claimDiagnosticPost() { + if (diagnosticPostInFlight.compareAndSet(false, true)) { + return true; + } + logger.debug("Skipped a diagnostic event because the previous one is still being posted"); + return false; + } + + /** + * Hands a claimed post to the diagnostics thread, releasing the claim once it ends. + * + * @param post what to run there, which must hold a claim from {@link #claimDiagnosticPost()} + */ + private void postDiagnostic(Runnable post) { + Runnable releasing = () -> { + try { + post.run(); + } finally { + diagnosticPostInFlight.set(false); + } + }; + try { + diagnosticExecutor.submit(guarded(releasing)); + } catch (RuntimeException e) { // the executor was shut down under us + diagnosticPostInFlight.set(false); + } + } + + /** + * Unlike analytics events, diagnostics are not sent while offline or in the background. + *

+ * {@link #updateScheduledTasks} cancels the periodic task when either becomes true, but that is + * not enough on its own. Canceling does not stop a run already underway, and the init event is + * submitted before it reaches the executor. Either can arrive here after the state changed. + */ + private boolean diagnosticsSuspended() { + return isStopped() || offline.get() || inBackground.get(); + } + + /** + * A diagnostic response is only read for the service's clock. One the service refused stops nothing: + * analytics delivery backs off on its own responses, and a diagnostic is not retried anyway. + */ + private void handleResponse(EventSender.Result result) { + if (result != null && result.getTimeFromServer() != null) { + recordPastTime(result.getTimeFromServer().getTime()); + } + } + + /** + * Moves the threshold forwards only. Analytics and diagnostic responses are handled on separate + * threads, so a plain assignment would let an older reading of the service clock overwrite a + * newer one and keep debug events alive past the date the service set. A loop rather than + * {@code accumulateAndGet}, which needs API 24. + */ + private void recordPastTime(long timeFromServer) { + long known; + do { + known = lastKnownPastTime.get(); + if (timeFromServer <= known) { + return; + } + } while (!lastKnownPastTime.compareAndSet(known, timeFromServer)); + } + + /** + * A debug event is emitted until the date the service gave us passes. We compare against the + * last date we know to be in the past according to the service as well as the device clock, so + * that a device whose clock is wrong errs on the side of stopping sooner. + */ + private boolean shouldDebugEvent(Long debugEventsUntilDate) { + if (debugEventsUntilDate == null || debugEventsUntilDate <= 0) { + return false; + } + return debugEventsUntilDate > lastKnownPastTime.get() + && debugEventsUntilDate > System.currentTimeMillis(); + } + + /** + * Must be called holding {@code stateLock}. Once closed, this only ever cancels. close() sets the + * flag and then calls this under the same lock, so a call that got here first has its tasks + * canceled by close(), and any call after it finds the flag set. + */ + private void updateScheduledTasks(boolean inBackground, boolean offline) { + boolean stopped = closed.get(); + // Flushing stays scheduled even while we are offline or in the background; a run while + // offline returns without doing anything. Canceling it for an outage would restart the + // interval on every reconnect, and a run of brief outages would then hold events back for + // far longer than one interval. Left running, what an outage buffered goes out at the first + // run after it ends. + flushTask = enableOrDisableTask(!stopped, flushTask, flushIntervalMillis, + this::deliverPayload); + // Events a previous run left on disk have already waited out a whole process, so they do not + // wait out an interval as well. An application that dies within one interval of every start + // would otherwise never deliver them. + if (!stopped && !offline && hasEventsFromPreviousRun.compareAndSet(true, false)) { + submit(this::deliverPayload); + } + boolean diagnosticsEnabled = !stopped && diagnosticStore != null && !offline && !inBackground; + diagnosticTask = enableOrDisableTask(diagnosticsEnabled, diagnosticTask, + diagnosticRecordingIntervalMillis, this::sendDiagnosticStats); + if (diagnosticsEnabled && !diagnosticInitSent && claimDiagnosticPost()) { + DiagnosticStore store = diagnosticStore; + postDiagnostic(() -> { + // Re-checked on the posting thread: going online and coming to the foreground are + // two separate calls, and both want to send the init event we never got to send. + // Suspension is re-checked for the same reason it is for the periodic event, and + // costs nothing here: getInitEvent consumes nothing, so a skipped init is simply + // built again the next time diagnostics are enabled. + if (!diagnosticInitSent && !diagnosticsSuspended()) { + sendDiagnosticEvent(store.getInitEvent(), true); + } + }); + } + } + + private ScheduledFuture enableOrDisableTask( + boolean shouldEnable, + ScheduledFuture currentTask, + long intervalMillis, + Runnable task + ) { + if (!shouldEnable) { + if (currentTask != null) { + currentTask.cancel(false); + } + return null; + } + if (currentTask != null && !currentTask.isDone()) { + return currentTask; + } + if (currentTask != null) { + // Backstop for a throwable that escaped guarded() anyway, such as one thrown while + // logging the first. The executor marks the repeating future done and never fires it + // again, and holding that future here would make every later enable a no-op. + currentTask.cancel(false); + } + try { + // Fixed delay rather than fixed rate: a cached process stops running its tasks without + // stopping the clock, so at a fixed rate it would come back owing every run it missed and + // fire them one after another. + return scheduler.scheduleWithFixedDelay(guarded(task), intervalMillis, intervalMillis, + TimeUnit.MILLISECONDS); + } catch (RuntimeException e) { // the executor was shut down under us + return null; + } + } + + /** + * Puts work on the delivery thread, and is the one place that decides whether there is still a + * thread willing to take it. The {@code isStopped} tests the callers make first are a fast path + * that saves the work of getting here, not a guarantee; this is what a delivery is actually + * ordered against close(). + * + * @return the submitted task, or null if the processor is shutting down or already has + */ + private Future submit(Runnable task) { + synchronized (submitLock) { + if (shuttingDown) { + // close() has already queued the release of the sender. Work accepted now would be + // behind it in the queue and would run against a sender that had been closed. + return null; + } + try { + return scheduler.submit(guarded(task)); + } catch (RuntimeException e) { // the executor was shut down under us + return null; + } + } + } + + /** + * As {@link #submit(Runnable)}, for the final delivery at close(), whose outcome close() waits + * for. Not wrapped in {@link #guarded}, because here the caller is there to receive what + * escapes. + * + * @return the submitted task, or null if the processor is shutting down or already has + */ + private Future submit(Callable task) { + synchronized (submitLock) { + if (shuttingDown) { + return null; + } + try { + return scheduler.submit(task); + } catch (RuntimeException e) { // the executor was shut down under us + return null; + } + } + } + + /** + * Keeps an unexpected failure from killing a repeating task or bubbling out of the executor. + * Anything that escapes a run suppresses the rest of a {@code scheduleWithFixedDelay} series, + * so this catches {@code Throwable} and not just {@code Exception}. Otherwise a + * {@code StackOverflowError} from nested {@code LDValue} data, or an {@code OutOfMemoryError} + * growing the payload stream, would stop flushing for the life of the process with nothing + * logged. + */ + private Runnable guarded(Runnable task) { + return () -> { + try { + task.run(); + } catch (Throwable t) { + logUnexpectedError(t); + } + }; + } + + private boolean isStopped() { + return closed.get(); + } + + private void logUnexpectedError(Throwable e) { + logger.error("Unexpected error in event processor: {}", LogValues.exceptionSummary(e)); + logger.debug("{}", LogValues.exceptionTrace(e)); + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiskEventLog.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiskEventLog.java new file mode 100644 index 000000000..71aafb915 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiskEventLog.java @@ -0,0 +1,286 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +/** + * The file of events on disk this process appends to, not to be confused with diagnostic logging. + *

+ * One per store rather than one per directory, so that only one writer ever appends to a given log. Each is + * named for a UUID picked when it is opened, which is what keeps two writers apart: nothing about it comes from + * the application, and no two processes, or two stores in one, ever pick the same one. + *

+ * The owner holds an exclusive lock on its log for as long as it is open, and the kernel releases it when the + * descriptor closes, including when the process dies. A log no one holds the lock on therefore belongs to no + * one alive, whatever process wrote it, and recovery takes it. A log is locked before it takes the name + * recovery looks for, so that a log being opened is never taken for one whose owner is gone. + *

+ * A POSIX lock belongs to the process rather than to the descriptor it was taken through, and closing any + * descriptor this process has on the file releases it. So nothing in this process opens another store's log: + * the logs open here are remembered, and recognized as live without a descriptor being opened on them. + *

+ * Belongs to an {@link EventStore}, which makes every call holding its {@code ioLock}. The lock is handed + * in so that each call can check it is held, rather than this taking a lock of its own that would have + * to be ordered against the store's. + */ +final class DiskEventLog { + /** + * Names a log whose events are not yet in a batch: one its owner is appending to, or one left by an owner + * that is gone, for recovery to close off. The rest of the name is the UUID it was opened under. + */ + static final String PREFIX = "pending-"; + /** + * Names a log while it is being opened, before it is locked and has its header. Recovery never looks at + * these; one left by a process that died while opening it holds no events, and expiry removes it. + */ + static final String OPENING_PREFIX = "new-"; + + /** How the log is opened, so a test can make a write fail partway through a session, as a full disk does. */ + interface Opener { + FileOutputStream openForAppending(File file) throws IOException; + } + + /** The absolute paths of the logs open in this process, which nothing here may open a descriptor on. */ + private static final Set openInThisProcess = new HashSet<>(); + + private final File directory; + private final Object ioLock; + private final Opener opener; + private final LDLogger logger; + // Guarded by ioLock. + private File file; + private FileOutputStream output; + private FileLock lock; + + DiskEventLog(File directory, Object ioLock, Opener opener, LDLogger logger) { + this.directory = directory; + this.file = newLogFile(); + this.ioLock = ioLock; + this.opener = opener; + this.logger = logger; + } + + private File newLogFile() { + return new File(directory, PREFIX + UUID.randomUUID()); + } + + /** + * Whether some live process holds a log, and recovery must leave it alone. + *

+ * A log open in this process is answered from memory. Any other is answered by trying for a shared lock + * on it, which fails while its owner holds the exclusive one. A log that cannot be looked at counts as + * held: leaving a dead process's log for a later run costs nothing, while taking a live one's would lose + * what it appends next. + */ + static boolean isHeldByLiveProcess(File log) { + synchronized (openInThisProcess) { + if (openInThisProcess.contains(log.getAbsolutePath())) { + return true; + } + } + try (RandomAccessFile probe = new RandomAccessFile(log, "r"); + FileChannel channel = probe.getChannel(); + FileLock held = channel.tryLock(0L, Long.MAX_VALUE, true)) { + return held == null; + } catch (FileNotFoundException e) { + // Gone, so no one is appending to it. + return false; + } catch (IOException | OverlappingFileLockException e) { + return true; + } + } + + /** Where the log is, whether it exists yet or not. */ + File file() { + return file; + } + + /** Requires {@code ioLock}. */ + boolean exists() { + assert Thread.holdsLock(ioLock); + return file.exists(); + } + + /** + * Appends to the log, opening it first where it is not open. + *

+ * Requires {@code ioLock}. + * + * @throws IOException if the log could not be opened or written to; none of {@code bytes} is then left + * in the log, and the log is closed, so the next append starts a new one + */ + void append(byte[] bytes) throws IOException { + assert Thread.holdsLock(ioLock); + FileOutputStream stream = outputForAppending(); + long start = -1; + try { + start = stream.getChannel().size(); + stream.write(bytes); + } catch (IOException | RuntimeException e) { + // Writing that fails partway can leave whole frames behind. The caller keeps every one of + // the bytes in memory to deliver from there, so frames left here as well would be sent + // twice: beside the memory copy when the log is closed this session, or by recovery on the + // next launch. + if (start >= 0) { + truncate(stream, start); + } + closeOutput(); + throw e; + } + } + + /** + * Truncates the log to {@code length}, where a failed write left it longer. Where it cannot, the + * batch this log is closed into still delivers only the events it was closed holding, which leaves out + * the frames left behind. + */ + private void truncate(FileOutputStream stream, long length) { + try { + stream.getChannel().truncate(length); + } catch (IOException | RuntimeException e) { + logger.warn("Could not truncate a failed write to the event log: {}", + LogValues.exceptionSummary(e)); + } + } + + /** + * Moves the log to {@code destination}, so that the next append starts a new one. + *

+ * Closed after the move rather than before it, so that the log is still locked while it moves: unlocked + * under this name, it could be taken by another process recovering logs in between. + *

+ * A move that fails leaves a log that is still there open and locked, to be appended to and moved + * again. Closed, it would be no longer this store's to write, and the next append would start another + * log while the caller still counted this one's events as the open log's: they would be out of reach + * this session, and the next batch would be named for more events than it holds. + *

+ * Requires {@code ioLock}. + * + * @return whether the log was moved + */ + boolean closeMovingTo(File destination) { + assert Thread.holdsLock(ioLock); + boolean moved = file.renameTo(destination); + if (moved || !file.exists()) { + closeOutput(); + } + return moved; + } + + /** + * Closes the stream, leaving the log where it is. Requires {@code ioLock}. + *

+ * Unlocked, it is from here on a log like one a process that died left behind, which recovery may take + * at any moment. So this log is never opened again: the next append starts a new one. + */ + void closeOutput() { + assert Thread.holdsLock(ioLock); + if (output == null) { + return; + } + try { + output.close(); + } catch (IOException e) { + logger.debug("Could not close the event log: {}", LogValues.exceptionSummary(e)); + } + output = null; + lock = null; + forget(file); + } + + /** + * Takes the lock on the open log again, after something in this process closed a descriptor of its own + * on the file, which released it. Requires {@code ioLock}. + */ + void relockAfterReading() { + assert Thread.holdsLock(ioLock); + if (output == null) { + return; + } + try { + if (lock != null) { + lock.release(); + } + lock = output.getChannel().tryLock(); + } catch (IOException | RuntimeException e) { + lock = null; + } + if (lock == null) { + logger.debug("Could not lock the event log again after reading it"); + } + } + + /** + * Opens a new log under a name of its own: created under {@link #OPENING_PREFIX}, locked, given its + * header, and only then moved to the name recovery looks for. + *

+ * Requires {@code ioLock}. + */ + private FileOutputStream outputForAppending() throws IOException { + if (output != null) { + return output; + } + if (!directory.exists() && !directory.mkdirs() && !directory.isDirectory()) { + throw new IOException("could not create " + directory); + } + if (file.exists()) { + // Left by an earlier open in this session, and unlocked since, so no longer this store's to write. + file = newLogFile(); + } + File opening = new File(directory, OPENING_PREFIX + file.getName().substring(PREFIX.length())); + remember(file); + FileOutputStream stream = null; + try { + stream = opener.openForAppending(opening); + FileLock taken = stream.getChannel().tryLock(); + if (taken == null) { + throw new IOException("could not lock " + opening); + } + stream.write(EventLogFormat.fileHeader()); + if (!opening.renameTo(file)) { + throw new IOException("could not move " + opening + " to " + file); + } + lock = taken; + output = stream; + return output; + } catch (IOException | RuntimeException e) { + // Not yet in output, so closeOutput would not reach it. + if (stream != null) { + try { + stream.close(); + } catch (IOException ignored) { + // The open already failed; that is the error worth reporting. + } + } + if (opening.exists() && !opening.delete()) { + logger.debug("Could not delete {}", opening); + } + forget(file); + throw e; + } + } + + private static void remember(File log) { + synchronized (openInThisProcess) { + openInThisProcess.add(log.getAbsolutePath()); + } + } + + private static void forget(File log) { + synchronized (openInThisProcess) { + openInThisProcess.remove(log.getAbsolutePath()); + } + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDirectory.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDirectory.java new file mode 100644 index 000000000..d7483dff4 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDirectory.java @@ -0,0 +1,998 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +/** + * The directory one store keeps its events in: the log this process appends to, and next to it the batches + * that have been closed off for delivery. + *

+ * Everything about the filesystem lives here: where the directory is, what the files in it are called, and + * how they are listed, read, moved and deleted. That leaves {@link EventStore} with the buffer, the + * capacity arithmetic, and the batches it holds in memory, none of which need a disk at all. A store the + * application told to keep events off the disk has no {@code EventDirectory}, rather than one it is careful + * never to use. + *

+ * The directory's location is worked out on first use rather than at construction. A store that never + * reaches for the disk then makes no filesystem call at all, not even to find out where it would have + * written. That matters because the store is built on the thread that called {@code LDClient.init}, usually the main + * one: the platform creates the no-backup directory the first time it is asked for it. + *

+ * Belongs to an {@link EventStore}, which makes every call holding its {@code ioLock}, exactly as + * {@link DiskEventLog} belongs to this. The lock is handed down so that each call can check it is held, + * rather than each level taking a lock of its own that would have to be ordered against the others. + */ +class EventDirectory { + /** + * Names a log closed off for delivery. The rest of the name is the batch's payload ID and how many + * events are in it, {@code ready-.}. Anything more a batch needs to say belongs in a + * frame type readers skip, rather than in the name. + *

+ * The count is in the name so that listing a batch this process did not close costs a directory entry + * rather than a read of the whole file. That is what every batch is at startup: the ones a previous run + * left behind, and the ones belonging to another process sharing the environment. Reading them all is + * done holding {@code ioLock}, which a commit at a commit point then waits behind. + *

+ * A batch is never appended to once it is closed, so the count is as good as the file is. + */ + private static final String BATCH_PREFIX = "ready-"; + private static final char NAME_FIELD_SEPARATOR = '.'; + private static final String DIRECTORY_NAME = "com.launchdarkly.events"; + + /** Where one store keeps its files. */ + interface Location { + File directory(); + } + + /** How a stored file is read, so a test can make a read fail while the file is intact. */ + interface Reader { + InputStream openForReading(File file) throws IOException; + } + + /** + * How many events, and how many bytes, something stored holds: the two things capacity counts. With how + * many of the events are summaries, which capacity leaves out, where this store wrote them. + */ + static final class Tally { + static final Tally NONE = new Tally(0, 0); + + final int events; + /** How many of {@link #events} are known to be summaries; none for a batch this store did not close. */ + final int summaries; + final long bytes; + + Tally(int events, long bytes) { + this(events, 0, bytes); + } + + Tally(int events, int summaries, long bytes) { + this.events = events; + this.summaries = summaries; + this.bytes = bytes; + } + + Tally plus(Tally other) { + return new Tally(events + other.events, summaries + other.summaries, bytes + other.bytes); + } + } + + /** What discarding expired files did. */ + static final class Discarded { + /** How many events went with them. */ + final int events; + /** How much of that this process was counting, which is what stops counting against capacity. */ + final Tally counted; + + Discarded(int events, Tally counted) { + this.events = events; + this.counted = counted; + } + } + + private final Location location; + private final Object ioLock; + private final LDLogger logger; + private final Reader reader; + private final DiskEventLog.Opener opener; + + // Guarded by ioLock, and null until locateHoldingIoLock has run. + private File directory; + private DiskEventLog diskLog; + /** Whether {@link #openInThisProcess} counts this store, from locating the directory until closing. */ + private boolean countedAsOpen; + + /** + * What this process knows about each batch in the directory, by payload ID, which is how everything + * outside this class names one. + *

+ * The file is remembered rather than rebuilt from the payload ID, because the count is part of the name + * and a batch named without one has no count to rebuild it with. The count is remembered so that such + * a batch is read at most once. + *

+ * Only to be used while holding {@code ioLock}. + */ + private final Map stored = new HashMap<>(); + + /** + * A batch in the directory: the file it is, how many events it holds, how many of them this store knows + * to be summaries, how big it is, and when it was last written, which is what orders batches oldest first. + */ + private static final class Stored { + final File file; + final int events; + final int summaries; + final long bytes; + final long modified; + + Stored(File file, int events, int summaries, long bytes, long modified) { + this.file = file; + this.events = events; + this.summaries = summaries; + this.bytes = bytes; + this.modified = modified; + } + + Tally tally() { + return new Tally(events, summaries, bytes); + } + } + + /** + * Batches LaunchDarkly has accepted whose file the filesystem would not let go of. + *

+ * Left out of every listing, because a batch that is delivered and merely still on the disk would otherwise + * be found again by the next delivery, and the one after that, and sent each time. Each listing tries to + * delete it once more, so a directory that stops being read-only clears itself. + *

+ * Only for this session: a later run, which may well be able to delete the file, treats it as it would any + * other batch a previous run left behind. LaunchDarkly discards a repeated payload ID, so the cost of that is + * a wasted request rather than events counted twice. + *

+ * Only to be used while holding {@code ioLock}. + */ + private final Set deliveredButUndeleted = new HashSet<>(); + + /** + * The batches this store has a request out for, by payload ID: the file each was locked under, so the lock + * is released from that file even once the batch has been removed or dropped from a listing, and how many + * of this store's deliveries are sending it. + *

+ * Only to be used while holding {@code ioLock}. + */ + private final Map sending = new HashMap<>(); + + private static final class Sending { + final File file; + int senders; + + Sending(File file) { + this.file = file; + } + } + + /** + * The directories of the stores open in this process, by absolute path, with how many stores have each + * open. Another environment's sweep leaves these alone, however long it is since they were written: a + * store that is open may be about to write, or be sending a batch older than the cutoff. + */ + private static final Map openInThisProcess = new HashMap<>(); + + EventDirectory(Location location, Object ioLock, LDLogger logger, Reader reader, + DiskEventLog.Opener opener) { + this.location = location; + this.ioLock = ioLock; + this.logger = logger; + this.reader = reader; + this.opener = opener; + } + + /** Requires {@code ioLock}. */ + private void locateHoldingIoLock() { + assert Thread.holdsLock(ioLock); + if (directory == null) { + File resolved = location.directory(); + diskLog = new DiskEventLog(resolved, ioLock, opener, logger); + directory = resolved; + countedAsOpen = true; + synchronized (openInThisProcess) { + Integer stores = openInThisProcess.get(resolved.getAbsolutePath()); + openInThisProcess.put(resolved.getAbsolutePath(), stores == null ? 1 : stores + 1); + } + } + } + + private static boolean isOpenInThisProcess(File environment) { + synchronized (openInThisProcess) { + return openInThisProcess.containsKey(environment.getAbsolutePath()); + } + } + + /** + * Where the batches are kept, worked out the first time the directory is reached for. + *

+ * Requires {@code ioLock}. + */ + File directory() { + locateHoldingIoLock(); + return directory; + } + + /** Requires {@code ioLock}. */ + private DiskEventLog diskLog() { + locateHoldingIoLock(); + return diskLog; + } + + /** + * A batch is identified by its payload ID rather than by a path, so that a batch listed from the + * directory and the same batch as it was closed are one thing. + *

+ * Requires {@code ioLock}. + * + * @param events what the caller holding the batch knows it to contain, which is what names it where + * this process has neither closed nor listed it - as when it delivers a batch another process closed + */ + private File batchFile(String payloadId, int events) { + Stored known = stored.get(payloadId); + return known == null ? newBatchFile(payloadId, events) : known.file; + } + + /** Requires {@code ioLock}. */ + private File newBatchFile(String payloadId, int events) { + return new File(directory(), BATCH_PREFIX + payloadId + NAME_FIELD_SEPARATOR + events); + } + + /** + * The count in a batch's name, {@code ready-.}: the second field, by position. + *

+ * Fields are split at {@link #NAME_FIELD_SEPARATOR}, which a payload ID never contains, so nothing + * about the count's own characters is needed to tell it apart. Fields after it, which a later + * version might add, are ignored. + * + * @return the count, or -1 where the name has none or it does not parse - a batch a recovery could not + * read, which is read once to count it + */ + private static int eventsInName(String fileName) { + String[] fields = fileName.substring(BATCH_PREFIX.length()).split("\\" + NAME_FIELD_SEPARATOR, -1); + if (fields.length < 2 || fields[1].isEmpty()) { + return -1; + } + for (int i = 0; i < fields[1].length(); i++) { + char c = fields[1].charAt(i); + if (c < '0' || c > '9') { + return -1; + } + } + try { + return Integer.parseInt(fields[1]); + } catch (NumberFormatException e) { + return -1; + } + } + + /** The payload ID in a batch's name: the first field, whatever follows it. */ + private static String payloadIdInName(String fileName) { + String rest = fileName.substring(BATCH_PREFIX.length()); + int end = rest.indexOf(NAME_FIELD_SEPARATOR); + return end < 0 ? rest : rest.substring(0, end); + } + + // MARK: writing + + /** + * Appends to the open log, opening a new one where none is open. + *

+ * Requires {@code ioLock}. + * + * @return whether every byte landed. False means none of them is in the log, and that the caller should + * give up on the disk. + */ + boolean append(byte[] bytes) { + assert Thread.holdsLock(ioLock); + try { + diskLog().append(bytes); + return true; + } catch (IOException | RuntimeException e) { + // Never let a full disk take the application down. The SDK gives up on persistence for the + // rest of the session instead, which is how other SDKs have crashed their hosts. + logger.warn("Giving up on persisting events: {}", LogValues.exceptionSummary(e)); + return false; + } + } + + /** What closing the open log produced. */ + static final class LogClose { + /** The batch the log became, or null where the close failed. */ + final EventStore.Batch batch; + /** + * Where the close failed, whether the log is still there to try again with; it is then still the + * open log, and its events still count as the open log's. Where it is not - a + * directory the platform reclaimed, say - its events are gone, and they have to stop counting + * against capacity, or the store would refuse events for the rest of the session. + */ + final boolean logSurvives; + + private LogClose(EventStore.Batch batch, boolean logSurvives) { + this.batch = batch; + this.logSurvives = logSurvives; + } + } + + /** + * Closes the open log into a batch, so the next append starts a new one. + *

+ * Requires {@code ioLock}. + * + * @param events how many events the log holds, which the caller has counted as it staged them + * @param summaries how many of {@code events} are summaries + * @param bytes how big the log is, which the caller has counted as it wrote it + */ + LogClose closeLog(int events, int summaries, long bytes) { + assert Thread.holdsLock(ioLock); + String payloadId = UUID.randomUUID().toString(); + File file = newBatchFile(payloadId, events); + if (!diskLog().closeMovingTo(file)) { + logger.warn("Could not close the event log for delivery"); + return new LogClose(null, diskLog().exists()); + } + + stored.put(payloadId, new Stored(file, events, summaries, bytes, System.currentTimeMillis())); + return new LogClose(new EventStore.Batch(payloadId, events), true); + } + + /** + * Closes the descriptor, leaving everything where it is, and stops counting the directory as open in this + * process. Requires {@code ioLock}. + */ + void closeOpenLog() { + assert Thread.holdsLock(ioLock); + // Not diskLog(): a store that never wrote has never asked the platform where it lives, and shutting + // down is no reason to start. + if (diskLog != null) { + diskLog.closeOutput(); + } + if (countedAsOpen) { + countedAsOpen = false; + synchronized (openInThisProcess) { + String path = directory.getAbsolutePath(); + Integer stores = openInThisProcess.get(path); + if (stores != null && stores > 1) { + openInThisProcess.put(path, stores - 1); + } else { + openInThisProcess.remove(path); + } + } + } + } + + /** + * Locks a batch against expiry, in this process and every other, for as long as a request for it is out. + * Each call is ended by one {@link #stoppedSending}. Requires {@code ioLock}. + * + * @param events what the caller holding the batch knows it to contain, as for {@link #payload} + */ + void startedSending(String payloadId, int events) { + assert Thread.holdsLock(ioLock); + Sending batch = sending.get(payloadId); + if (batch == null) { + batch = new Sending(batchFile(payloadId, events)); + sending.put(payloadId, batch); + } + batch.senders++; + BatchLocks.startedSending(batch.file); + } + + /** Ends one {@link #startedSending}. Requires {@code ioLock}. */ + void stoppedSending(String payloadId) { + assert Thread.holdsLock(ioLock); + Sending batch = sending.get(payloadId); + if (batch == null) { + return; + } + BatchLocks.stoppedSending(batch.file); + if (--batch.senders == 0) { + sending.remove(payloadId); + } + } + + // MARK: listing and reading + + /** + * Requires {@code ioLock}. + *

+ * Sizes come from the directory entries, so that counting the directory's bytes reads no file either. + *

+ * A directory that is not there holds nothing. One that is there and cannot be listed - out of file + * descriptors, or no longer readable - is not empty, so it answers with the batches the last listing or + * close counted, as they were counted, and says the listing is incomplete. + * + * @return the batches the directory holds, oldest first, which is the order they are delivered in + */ + EventStore.Listing listBatches() { + assert Thread.holdsLock(ioLock); + List ready = readyFilesOldestFirst(); + if (ready == null) { + logger.info("Could not list stored events, will try again at the next delivery"); + return new EventStore.Listing(knownBatchesOldestFirst(), false); + } + List batches = new ArrayList<>(); + for (ReadyFile entry : ready) { + File file = entry.file; + int events = eventsInName(file.getName()); + String payloadId = payloadIdInName(file.getName()); + if (isDeliveredButUndeleted(payloadId, file)) { + continue; + } + // Reading is only for a batch whose name does not count it, which a version that named them + // without a count may have left behind, and then only the once. + if (events < 0) { + Stored known = stored.get(payloadId); + if (known != null) { + events = known.events; + } else { + byte[] log; + try { + log = readFile(file); + } catch (IOException e) { + // Left for a later listing rather than deleted: the read may pass, and the events in + // it may be intact. Without a count it cannot be delivered now. + logger.warn("Could not read stored events, will try again at the next delivery: {}", + LogValues.exceptionSummary(e)); + continue; + } + if (log == null) { + // Delivered and removed by another process since the listing. + continue; + } + events = EventLogFormat.eventCount(log); + } + if (events < 0) { + // Written by a version whose format this one does not read, or damaged beyond what the + // torn-tail recovery tolerates. Either way it can never be delivered. + tryDelete(file); + continue; + } + } + // What the close knew of its summaries is kept; nothing in the file says. + Stored previous = stored.get(payloadId); + int summaries = previous != null && previous.events == events ? previous.summaries : 0; + stored.put(payloadId, new Stored(file, events, summaries, file.length(), entry.modified)); + batches.add(new EventStore.Batch(payloadId, events)); + } + + // A batch that left the directory without going through remove() would otherwise keep its count + // for the rest of the session. + Set listed = new HashSet<>(); + for (EventStore.Batch batch : batches) { + listed.add(batch.payloadId); + } + stored.keySet().retainAll(listed); + return new EventStore.Listing(batches, true); + } + + /** + * The batches the last listing or close counted, oldest first: every batch this store closed itself, but + * none another process closed or a previous run left since. Requires {@code ioLock}. + */ + private List knownBatchesOldestFirst() { + List> known = new ArrayList<>(stored.entrySet()); + Collections.sort(known, (a, b) -> Long.compare(a.getValue().modified, b.getValue().modified)); + List batches = new ArrayList<>(); + for (Map.Entry batch : known) { + batches.add(new EventStore.Batch(batch.getKey(), batch.getValue().events)); + } + return batches; + } + + /** The summaries known to be in the batches the last listing or close counted. Requires {@code ioLock}. */ + int storedSummaries() { + assert Thread.holdsLock(ioLock); + int summaries = 0; + for (Stored batch : stored.values()) { + summaries += batch.summaries; + } + return summaries; + } + + /** The bytes in every batch the last listing or close counted. Requires {@code ioLock}. */ + long storedBytes() { + assert Thread.holdsLock(ioLock); + long bytes = 0; + for (Stored batch : stored.values()) { + bytes += batch.bytes; + } + return bytes; + } + + /** + * What this process counts for a batch, or null if it counts nothing for it: one removed, or dropped by + * a listing it was no longer in. Requires {@code ioLock}. + */ + Tally counted(String payloadId) { + assert Thread.holdsLock(ioLock); + Stored known = stored.get(payloadId); + return known == null ? null : known.tally(); + } + + /** + * Whether a batch has already been delivered and only its file is left, trying once more on the way to + * delete the file. + *

+ * Requires {@code ioLock}. + */ + private boolean isDeliveredButUndeleted(String payloadId, File file) { + if (!deliveredButUndeleted.contains(payloadId)) { + return false; + } + tryDelete(file); + if (!file.exists()) { + deliveredButUndeleted.remove(payloadId); + } + return true; + } + + /** + * Requires {@code ioLock}. + * + * @return the JSON payload for a batch, assembled in the array the file was read into; or null if it is + * gone or holds no events + * @throws EventStore.UnreadableBatchException if the batch is there but holds nothing this version can + * send + * @throws IOException if the batch is there but could not be read this time + */ + EventLogFormat.Payload payload(String payloadId, int events) throws IOException { + assert Thread.holdsLock(ioLock); + byte[] log = readFile(batchFile(payloadId, events)); + if (log == null) { + return null; + } + // No more than the batch was closed holding. Writing that failed and could not be truncated + // leaves whole frames after them, whose events are delivered from memory instead, and would + // otherwise be sent twice. + EventLogFormat.Payload payload = + EventLogFormat.assemblePayloadInPlace(log, events > 0 ? events : Integer.MAX_VALUE); + // A log this version reads that holds no events has nothing to lose, unless its first frame is + // damaged. Where there is no payload the log is as it was read. + if (payload == null && (EventLogFormat.eventCount(log) != 0 || EventLogFormat.isDamaged(log))) { + throw new EventStore.UnreadableBatchException(payloadId); + } + return payload; + } + + /** + * Removes a batch. Requires {@code ioLock}. + * + * @return what it stopped counting, which is nothing for a batch that was already gone - one another + * process delivered, say, which a listing has stopped counting already + */ + Tally remove(String payloadId, int events) { + assert Thread.holdsLock(ioLock); + File file = batchFile(payloadId, events); + tryDelete(file); + if (file.exists()) { + // Its events stop counting all the same: LaunchDarkly has them, and holding capacity for a file this + // process may never be allowed to delete would cost the session events it could still have recorded. + logger.debug("Could not delete a delivered batch; it will not be sent again this session"); + deliveredButUndeleted.add(payloadId); + } + Stored removed = stored.remove(payloadId); + return removed == null ? Tally.NONE : removed.tally(); + } + + /** + * Every event in the directory, encoded exactly as it will be sent. Requires {@code ioLock}. + *

+ * For tests and for diagnosing a store rather than for the recording path: it reads every file. + */ + List storedPayloads() { + assert Thread.holdsLock(ioLock); + List logs = new ArrayList<>(); + List ready = readyFilesOldestFirst(); + for (ReadyFile entry : ready != null ? ready : Collections.emptyList()) { + logs.add(readFileOrNull(entry.file)); + } + if (diskLog().exists()) { + logs.add(readFileOrNull(diskLog().file())); + // Closing the descriptor that read it released this process's lock on it. + diskLog().relockAfterReading(); + } + + List payloads = new ArrayList<>(); + for (byte[] log : logs) { + payloads.addAll(EventLogFormat.eventPayloads(log)); + } + return payloads; + } + + /** A batch file, and when it was last written as of the listing that found it. */ + private static final class ReadyFile { + final File file; + final long modified; + + ReadyFile(File file) { + this.file = file; + this.modified = file.lastModified(); + } + } + + /** + * Requires {@code ioLock}. + *

+ * Each file's modification time is read once, before the sort, so that the order cannot change under it. + * + * @return every batch file in the directory, oldest first; empty if the directory is not there, and null if + * it is there and could not be listed + */ + private List readyFilesOldestFirst() { + List ready = new ArrayList<>(); + File[] files = directory().listFiles(); + if (files == null) { + // File says null for an I/O error as well as for a directory that is not there. + return directory().exists() ? null : ready; + } + for (File file : files) { + if (file.getName().startsWith(BATCH_PREFIX)) { + ready.add(new ReadyFile(file)); + } + } + Collections.sort(ready, (a, b) -> Long.compare(a.modified, b.modified)); + return ready; + } + + /** + * @return the file's bytes, or null if it no longer exists + * @throws IOException if it exists but could not be read. That may pass, and the file is intact: + * a process out of file descriptors fails to open it with the same exception a missing file + * gets, which is why existence is checked rather than the exception's type. + */ + private byte[] readFile(File file) throws IOException { + try (InputStream in = reader.openForReading(file)) { + // Read into an array of the file's size, so a batch is held once rather than in a growing + // buffer and a copy of it. Only a log still being appended to can turn out longer. + byte[] contents = new byte[(int) Math.min(file.length(), Integer.MAX_VALUE - 8)]; + int filled = 0; + while (true) { + if (filled == contents.length) { + int next = in.read(); + if (next < 0) { + break; + } + contents = Arrays.copyOf(contents, Math.max(2 * contents.length, 8192)); + contents[filled++] = (byte) next; + } + int read = in.read(contents, filled, contents.length - filled); + if (read < 0) { + break; + } + filled += read; + } + return filled == contents.length ? contents : Arrays.copyOf(contents, filled); + } catch (IOException e) { + if (!file.exists()) { + return null; + } + throw e; + } + } + + /** For inspecting the store, where a file that cannot be read is simply left out. */ + private byte[] readFileOrNull(File file) { + try { + return readFile(file); + } catch (IOException e) { + return null; + } + } + + private void tryDelete(File file) { + if (file.exists() && !file.delete()) { + logger.debug("Could not delete {}", file); + } + } + + // MARK: recovery + + /** + * Every log in the directory that no live process holds, for recovery to read with {@link #read} and + * close off with {@link #preparePayload}, so their events join the batches to deliver. + *

+ * Whoever recorded the events in such a log is gone, so there is no one left to add to it. That may be a + * previous run of this process, another process of the application, or a store in this one that has since + * closed. A log some live process holds, this store's own included, is left alone. None of those listed + * becomes held later: a log is opened under a name of its own and never reopened once closed. + *

+ * Requires {@code ioLock}. + */ + List interruptedLogs() { + assert Thread.holdsLock(ioLock); + List logs = new ArrayList<>(); + File[] files = directory().listFiles(); + if (files == null) { + return logs; + } + for (File file : files) { + if (file.getName().startsWith(DiskEventLog.PREFIX) && !DiskEventLog.isHeldByLiveProcess(file)) { + logs.add(file); + } + } + return logs; + } + + /** What {@link #read} found in a log a previous run left. */ + static final class InterruptedLog { + final File file; + /** Null where it could be read: it is gone, or {@link #events} and {@link #bytes} say what it holds. */ + final IOException failure; + final boolean gone; + /** As {@link EventLogFormat#eventCount}. */ + final int events; + final long bytes; + + private InterruptedLog(File file, IOException failure, boolean gone, int events, long bytes) { + this.file = file; + this.failure = failure; + this.gone = gone; + this.events = events; + this.bytes = bytes; + } + } + + /** + * Reads and counts one of the {@link #interruptedLogs}. Needs no lock: no one appends to the log, and nothing + * this store counts changes here, so a commit is not held up behind the read. + */ + InterruptedLog read(File log) { + try { + byte[] contents = readFile(log); + if (contents == null) { + return new InterruptedLog(log, null, true, 0, 0); + } + return new InterruptedLog(log, null, false, EventLogFormat.eventCount(contents), contents.length); + } catch (IOException e) { + return new InterruptedLog(log, e, false, 0, 0); + } + } + + /** + * Closes off a log {@link #read} has been through into a batch to deliver. Requires {@code ioLock}, so that + * the rename and what this store counts for the batch are one step to a listing, which would otherwise + * count it twice. + * + * @return what it closed off + */ + Tally preparePayload(InterruptedLog interrupted) { + assert Thread.holdsLock(ioLock); + File log = interrupted.file; + if (interrupted.gone || !log.exists()) { + // Recovered by another store or process since it was listed. + return Tally.NONE; + } + if (interrupted.failure != null) { + // Deleting it would lose events that may be intact. Named without a count, which is what a batch + // whose events have never been counted is, it is read and counted by whichever listing first + // manages to. + if (log.renameTo(new File(directory(), BATCH_PREFIX + UUID.randomUUID()))) { + logger.warn("Could not read events from a previous run, will try again at the next delivery: {}", + LogValues.exceptionSummary(interrupted.failure)); + } else if (log.exists()) { + // Left where it is, unlocked and appended to by no one, for the next launch to recover. + logger.warn("Could not read events from a previous run, will try again at the next launch: {}", + LogValues.exceptionSummary(interrupted.failure)); + } + return Tally.NONE; + } + int events = interrupted.events; + if (events <= 0) { + // Holds nothing, or nothing this version reads. + tryDelete(log); + return Tally.NONE; + } + String payloadId = UUID.randomUUID().toString(); + File file = newBatchFile(payloadId, events); + if (!log.renameTo(file)) { + if (log.exists()) { + // Left where it is for a later run: no one appends to it, so nothing is lost by waiting. + logger.warn("Could not prepare {} event(s) from a previous run, will try again at the next launch", + events); + } + return Tally.NONE; + } + Stored recovered = new Stored(file, events, 0, interrupted.bytes, file.lastModified()); + stored.put(payloadId, recovered); + logger.debug("Prepared {} event(s) from a previous run of this application for delivery", events); + return recovered.tally(); + } + + // MARK: expiry + + /** + * Deletes the batches in this directory last written before {@code cutoffMillis}, and the logs no live + * process holds that have not been written since then either. Requires {@code ioLock}. + *

+ * A batch's modification time is when its newest event was written, since a batch is renamed rather than + * rewritten when it is closed, so a batch is discarded only once everything in it is that old. A log is + * usually recovered rather than left to grow old, so one that gets here is one recovery could not move. + * + * A batch whose request is out is left alone however old it is, whichever store or process sent it; see + * {@link BatchLocks}. + * + * @param sending batches whose request is out from this store + */ + Discarded discard(long cutoffMillis, Set sending) { + assert Thread.holdsLock(ioLock); + File[] files = directory().listFiles(); + int events = 0; + Tally counted = Tally.NONE; + if (files == null) { + return new Discarded(0, counted); + } + for (File file : files) { + if (isWrittenSince(file, cutoffMillis)) { + continue; + } + String name = file.getName(); + if (name.startsWith(BATCH_PREFIX)) { + int named = eventsInName(name); + String payloadId = payloadIdInName(name); + if (sending.contains(payloadId) || deliveredButUndeleted.contains(payloadId) + || BatchLocks.isSentInThisProcess(file)) { + continue; + } + Stored known = stored.get(payloadId); + int held = named >= 0 ? named : known != null ? known.events : eventsReading(file); + if (!BatchLocks.deleteUnlessBeingSent(file)) { + if (!file.exists()) { + // Delivered by another process since the listing: no longer counted, and not dropped. + Stored removed = stored.remove(payloadId); + if (removed != null) { + counted = counted.plus(removed.tally()); + } + } + continue; + } + events += held; + Stored removed = stored.remove(payloadId); + if (removed != null) { + counted = counted.plus(removed.tally()); + } + } else { + events += discardIfAbandonedLog(file); + } + } + return new Discarded(events, counted); + } + + /** + * Empties and removes the directories of other environments that nothing has written to since + * {@code cutoffMillis}: those of a mobile key the application no longer uses, which no client will look + * in again. + *

+ * Age decides rather than the keys the application configures, because another process of the same + * application may be using a key this one does not configure. Nothing is touched here that this store + * reads or writes, so it needs no lock. A directory is removed only if it is empty, so a log another + * process starts in it meanwhile keeps the directory and itself. + *

+ * Only where {@code current} is where the SDK puts environments, so that a store handed some other + * directory never goes through that directory's siblings. The directory of a store open in this process + * is left alone, and so is a batch some live process is sending; see {@link BatchLocks}. + * + * @return how many events were discarded + */ + static int discardAbandonedEnvironments(File current, long cutoffMillis) { + File root = current.getParentFile(); + if (root == null || !DIRECTORY_NAME.equals(root.getName())) { + return 0; + } + File[] environments = root.listFiles(); + if (environments == null) { + return 0; + } + int events = 0; + for (File environment : environments) { + if (environment.equals(current) || !environment.isDirectory() || isOpenInThisProcess(environment)) { + continue; + } + File[] files = environment.listFiles(); + if (files != null) { + for (File file : files) { + if (isWrittenSince(file, cutoffMillis)) { + continue; + } + String name = file.getName(); + if (name.startsWith(BATCH_PREFIX)) { + if (BatchLocks.isSentInThisProcess(file)) { + continue; + } + int named = eventsInName(name); + int held = named >= 0 ? named : eventsReading(file); + if (BatchLocks.deleteUnlessBeingSent(file)) { + events += held; + } + } else { + events += discardIfAbandonedLog(file); + } + } + } + // Removes the directory only if it is empty. + delete(environment); + } + return events; + } + + /** + * A file whose modification time is unknown counts as written since, so it is never taken for an old one. + */ + private static boolean isWrittenSince(File file, long cutoffMillis) { + long modified = file.lastModified(); + return modified <= 0 || modified >= cutoffMillis; + } + + /** + * Deletes a log no live process holds, or one a process died opening, which holds no events. + * + * @return how many events went with it + */ + private static int discardIfAbandonedLog(File file) { + String name = file.getName(); + if (name.startsWith(DiskEventLog.OPENING_PREFIX)) { + delete(file); + return 0; + } + if (!name.startsWith(DiskEventLog.PREFIX) || DiskEventLog.isHeldByLiveProcess(file)) { + return 0; + } + int held = eventsReading(file); + return delete(file) ? held : 0; + } + + // Files.newInputStream needs API 26. + @SuppressWarnings("IOStreamConstructor") + private static int eventsReading(File file) { + ByteArrayOutputStream contents = new ByteArrayOutputStream(); + byte[] chunk = new byte[8192]; + try (InputStream in = new FileInputStream(file)) { + int read; + while ((read = in.read(chunk)) > 0) { + contents.write(chunk, 0, read); + } + } catch (IOException e) { + return 0; + } + return Math.max(0, EventLogFormat.eventCount(contents.toByteArray())); + } + + private static boolean delete(File file) { + return file.delete() || !file.exists(); + } + + // MARK: naming + + /** The directory the store for one environment belongs in. */ + static File defaultDirectory(File noBackupFilesDir, String mobileKey) { + return new File(new File(noBackupFilesDir, DIRECTORY_NAME), environmentDirectoryName(mobileKey)); + } + + /** + * Names the directory for one environment, hashed so the mobile key itself is never a filename. + * Not {@link LDUtil#urlSafeBase64Hash(String)}, for the reasons {@link LDUtil#hexDigest} gives. + */ + private static String environmentDirectoryName(String mobileKey) { + // Half a digest, which is far more than enough to keep two mobile keys apart. + return LDUtil.hexDigest(mobileKey, 16); + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventLogFormat.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventLogFormat.java new file mode 100644 index 000000000..855af4c8d --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventLogFormat.java @@ -0,0 +1,358 @@ +package com.launchdarkly.sdk.android; + +import androidx.annotation.VisibleForTesting; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.zip.CRC32; + +/** + * The on-disk shape of an event log. + *

+ * A log opens with a magic and a format version, and holds a sequence of frames: + * 

+ *     +- 4 bytes -+- 2 bytes -+     +- 2 bytes -+- 4 bytes -+- 4 bytes -+- n bytes -+
+ *     |   LDEV    |  version  | ... |   type    |  size (n) |   CRC-32  |  payload  |
+ *     +-----------+-----------+     +-----------+-----------+-----------+-----------+
+ *
+ * The size prefix is what makes a log recoverable: a reader can tell a frame the writer finished + * from one it did not, without having to parse the event to find where it ends. The checksum, over + * the type, the size and the payload, is what tells a frame the writer finished from one the storage + * has since damaged; a reader treats either the same way. The version is what makes an upgrade safe + * - a log this version does not understand is discarded rather than misread - and the frame type + * leaves room for a later version to write something new into a log this one still reads. + *

+ * The format is byte-for-byte the one the iOS SDK writes, so the two can be reasoned about, and + * debugged, as one thing. + */ +final class EventLogFormat { + /** Bump this whenever the framing or the meaning of a frame changes. */ + static final int VERSION = 1; + static final byte[] MAGIC = {'L', 'D', 'E', 'V'}; + static final int FILE_HEADER_SIZE = 6; + static final int FRAME_HEADER_SIZE = 10; + /** The only frame type written today; a reader skips a frame whose type it does not know. */ + static final int EVENT_FRAME = 1; + /** + * The most a batch may hold, file header included, so that no request it becomes is larger than the + * events service accepts. + */ + static final int MAX_BATCH_SIZE = 2 * 1024 * 1024; + /** + * A ceiling on a single frame's payload: the most that still fits in a batch on its own. It is also + * what keeps a corrupt size from making recovery allocate wildly. + */ + static final int MAX_FRAME_SIZE = MAX_BATCH_SIZE - FILE_HEADER_SIZE - FRAME_HEADER_SIZE; + + private EventLogFormat() { + } + + static byte[] fileHeader() { + byte[] header = new byte[FILE_HEADER_SIZE]; + System.arraycopy(MAGIC, 0, header, 0, MAGIC.length); + header[4] = (byte) (VERSION >> 8); + header[5] = (byte) VERSION; + return header; + } + + static void writeFrame(ByteArrayOutputStream target, byte[] serializedEvent) { + int length = serializedEvent.length; + byte[] header = new byte[FRAME_HEADER_SIZE]; + header[0] = (byte) (EVENT_FRAME >> 8); + header[1] = (byte) EVENT_FRAME; + writeInt32(header, 2, length); + CRC32 checksum = new CRC32(); + checksum.update(header, 0, 6); + checksum.update(serializedEvent, 0, length); + writeInt32(header, 6, (int) checksum.getValue()); + target.write(header, 0, FRAME_HEADER_SIZE); + target.write(serializedEvent, 0, length); + } + + /** + * @return whether the log holds frames that cannot be read back: torn, or failing their + * checksum; false for a log this version does not read at all + */ + static boolean isDamaged(byte[] log) { + Frames frames = new Frames(log); + if (!frames.isReadable()) { + return false; + } + while (frames.hasNext()) { + frames.next(); + } + return !frames.reachedEnd(); + } + + /** Frames on their own, as a log: only for the in-memory fallback, which has no file. */ + static byte[] withFileHeader(byte[] frames) { + return withFileHeader(frames, 0, frames.length); + } + + /** The frames from {@code start} to {@code end}, as a log of their own. */ + static byte[] withFileHeader(byte[] frames, int start, int end) { + byte[] log = new byte[FILE_HEADER_SIZE + end - start]; + System.arraycopy(fileHeader(), 0, log, 0, FILE_HEADER_SIZE); + System.arraycopy(frames, start, log, FILE_HEADER_SIZE, end - start); + return log; + } + + /** + * @return how many event frames the log holds, or -1 if it is not a log this version reads + */ + static int eventCount(byte[] log) { + Frames frames = new Frames(log); + if (!frames.isReadable()) { + return -1; + } + int events = 0; + while (frames.hasNext()) { + if (frames.next().type == EVENT_FRAME) { + events++; + } + } + return events; + } + + /** + * A JSON array of events at the start of an array that may run on past it, so that it can be sent + * from where it was assembled rather than copied out first. + */ + static final class Payload { + final byte[] bytes; + final int length; + /** + * How many events made it in: fewer than the batch holds where a frame after the first is torn or + * fails its checksum, since assembly stops there. + */ + final int eventCount; + + Payload(byte[] bytes, int length, int eventCount) { + this.bytes = bytes; + this.length = length; + this.eventCount = eventCount; + } + + byte[] toByteArray() { + return Arrays.copyOf(bytes, length); + } + } + + /** + * Turns the log into the JSON array LaunchDarkly expects, in the log's own array, without parsing + * the events: they were serialized on the way in and are shipped exactly as they were recorded. + *

+ * The array always has room: the file header becomes the '[', and each frame header the ',' or ']' + * after its payload. Each payload moves towards the start only once its frame has been checked, and + * never past where that frame began, so no frame still to be checked is written over. + * + * @return the payload of the first {@code maxEvents} events, leaving out any after them; or null if the + * log is unreadable or holds no events, in which case the log is left as it was + */ + static Payload assemblePayloadInPlace(byte[] log, int maxEvents) { + Frames frames = new Frames(log); + if (!frames.isReadable()) { + return null; + } + int events = 0; + int cursor = 1; + while (events < maxEvents && frames.hasNext()) { + Frame frame = frames.next(); + if (frame.type != EVENT_FRAME) { + continue; + } + System.arraycopy(log, frame.start, log, cursor, frame.length); + cursor += frame.length; + log[cursor++] = ','; + events++; + } + if (events == 0) { + return null; + } + log[0] = '['; + log[cursor - 1] = ']'; + return new Payload(log, cursor, events); + } + + /** + * {@link #assemblePayloadInPlace} into an array of its own, leaving the log as it was. + * + * @return the payload, or null if the log is unreadable or holds no events + */ + @VisibleForTesting + static byte[] assembleBody(byte[] log) { + return assembleBody(log, Integer.MAX_VALUE); + } + + @VisibleForTesting + static byte[] assembleBody(byte[] log, int maxEvents) { + Payload payload = assemblePayloadInPlace(log.clone(), maxEvents); + return payload == null ? null : payload.toByteArray(); + } + + /** + * @return each event in the log, serialized as it will be sent; empty if the log is unreadable + */ + static List eventPayloads(byte[] log) { + List payloads = new ArrayList<>(); + Frames frames = new Frames(log); + if (!frames.isReadable()) { + return payloads; + } + while (frames.hasNext()) { + Frame frame = frames.next(); + if (frame.type == EVENT_FRAME) { + payloads.add(Arrays.copyOfRange(log, frame.start, frame.start + frame.length)); + } + } + return payloads; + } + + /** One whole frame of a log: its type, and where its payload is in the log. */ + static final class Frame { + final int type; + final int start; + final int length; + + Frame(int type, int start, int length) { + this.type = type; + this.start = start; + this.length = length; + } + } + + /** + * Walks the frames of a log, stopping at the first one that is not wholly there or fails its + * checksum. + *

+ * A process that died partway through a write leaves a torn frame at the end. Everything before it + * is intact, so recovery keeps that and drops only the tail. A frame the storage damaged afterwards + * ends the walk the same way, because its size can no longer be trusted to say where the next frame + * starts. + *

+ * There is no next frame from the first one that is not wholly there or fails its checksum, and none + * at all in a log that is not {@link #isReadable() readable}. {@link #hasNext()} is what checks the + * next frame, so a caller may write anywhere before where the last frame it was handed ends. + */ + static final class Frames implements Iterator { + private final byte[] log; + private final boolean readable; + private final CRC32 checksum = new CRC32(); + private int cursor = FILE_HEADER_SIZE; + private boolean stopped; + private Frame checked; + + Frames(byte[] log) { + this.log = log; + this.readable = isReadable(log); + } + + /** Whether this is a log this version reads at all. */ + boolean isReadable() { + return readable; + } + + /** + * Whether the walk ended at the end of the log, rather than at a frame that is torn or fails its + * checksum. Only meaningful once {@link #hasNext()} has returned false. + */ + boolean reachedEnd() { + return !stopped && cursor == log.length; + } + + @Override + public boolean hasNext() { + if (checked != null) { + return true; + } + if (!readable || stopped || cursor + FRAME_HEADER_SIZE > log.length) { + return false; + } + int type = readUInt16(log, cursor); + int length = readInt32(log, cursor + 2); + int start = cursor + FRAME_HEADER_SIZE; + if (length <= 0 || length > MAX_FRAME_SIZE || start + length > log.length) { + stopped = true; + return false; + } + checksum.reset(); + checksum.update(log, cursor, 6); + checksum.update(log, start, length); + if ((int) checksum.getValue() != readInt32(log, cursor + 6)) { + stopped = true; + return false; + } + checked = new Frame(type, start, length); + cursor = start + length; + return true; + } + + @Override + public Frame next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + Frame frame = checked; + checked = null; + return frame; + } + + private static boolean isReadable(byte[] log) { + if (log == null || log.length < FILE_HEADER_SIZE) { + return false; + } + for (int i = 0; i < MAGIC.length; i++) { + if (log[i] != MAGIC[i]) { + return false; + } + } + return readUInt16(log, MAGIC.length) == VERSION; + } + } + + /** The size, header included, of the frame at {@code start} in frames this process wrote. */ + static int frameSize(byte[] frames, int start) { + return FRAME_HEADER_SIZE + readInt32(frames, start + 2); + } + + /** + * Where a run of whole frames from {@code start}, in frames this process wrote, ends if it is to take + * no more than {@code room} bytes. It takes the first frame whatever its size. + */ + static int endOfFramesFitting(byte[] frames, int start, long room) { + int end = start + frameSize(frames, start); + while (end < frames.length && end + frameSize(frames, end) - start <= room) { + end += frameSize(frames, end); + } + return end; + } + + /** How many frames there are between two frame boundaries in frames this process wrote. */ + static int frameCount(byte[] frames, int start, int end) { + int count = 0; + for (int cursor = start; cursor < end; cursor += frameSize(frames, cursor)) { + count++; + } + return count; + } + + private static int readUInt16(byte[] bytes, int offset) { + return ((bytes[offset] & 0xff) << 8) | (bytes[offset + 1] & 0xff); + } + + static int readInt32(byte[] bytes, int offset) { + return ((bytes[offset] & 0xff) << 24) | ((bytes[offset + 1] & 0xff) << 16) + | ((bytes[offset + 2] & 0xff) << 8) | (bytes[offset + 3] & 0xff); + } + + private static void writeInt32(byte[] bytes, int offset, int value) { + bytes[offset] = (byte) (value >> 24); + bytes[offset + 1] = (byte) (value >> 16); + bytes[offset + 2] = (byte) (value >> 8); + bytes[offset + 3] = (byte) value; + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventStore.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventStore.java new file mode 100644 index 000000000..333a620c5 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventStore.java @@ -0,0 +1,1237 @@ +package com.launchdarkly.sdk.android; + +import androidx.annotation.NonNull; +import androidx.annotation.VisibleForTesting; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +/** + * A crash-durable append log of serialized analytics events. + *

+ * Events are appended to an open log, which is closed into a batch when the event processor is ready to + * deliver one. A batch is deleted only once LaunchDarkly has accepted it, so a delivery interrupted by + * the process dying is retried on the next run rather than lost. + *

+ * Bytes reach the kernel through an ordinary write and are never {@code fsync}ed. What this defends + * against is the process dying, which is what an uncaught exception, an ANR kill, a native crash, and + * the system reclaiming a backgrounded application all are. A process cannot take back bytes the + * kernel already holds. Only losing the kernel itself, to a panic or a power cut, can discard them, and + * covering that would mean an {@code fsync} per event, which costs milliseconds where this costs + * microseconds. + *

+ * Several clients. An application configured for more than one environment gets one + * {@code LDClient} and one store per mobile key, and the mobile key's hash is part of the directory, so + * two environments never read or write each other's events. + *

+ * Several processes. An application may declare components in more than one process, and each + * gets its own copy of the SDK that shares nothing but the filesystem. Only the open log is per-process, + * named after the process appending to it. That keeps two processes from interleaving frames in one file, + * and keeps one process's startup recovery from stealing a log another is still writing to. + * Closed batches are deliberately left shared: a process that recorded events and then died may never + * run again, and a batch nobody but that process could see would never be delivered. If two processes + * do deliver the same batch, its payload ID is the same both times, and LaunchDarkly discards a repeated + * payload ID rather than counting the events twice. + *

+ * What a log looks like on disk is {@link EventLogFormat}; the log this process appends to is + * {@link DiskEventLog}; the directory the two of them live in is {@link EventDirectory}. This class is + * what decides when events move between memory, that directory, and the batches closed off for delivery, + * and what counts them against capacity on the way. It has no directory at all where the application asked + * for events to be kept off the disk, which is what keeps the filesystem out of that case entirely. + */ +class EventStore implements Closeable { + /** + * Where the events are kept, or null where the application asked for them to be kept off the disk. + *

+ * Everything about the filesystem is behind this, so that the store itself is the buffer, the capacity + * arithmetic, and the batches it holds in memory. A store with no disk runs the way the SDK ran before + * persistence existed, and never touches the filesystem, not even to look for a previous run's events. + */ + private final EventDirectory disk; + private final int capacity; + /** The most bytes summaries may take the store to. */ + private final long byteCapacity; + /** + * The most bytes full events may take the store to: the byte capacity less a fifth of it, held back so + * that summaries, which bypass {@link #capacity}, still have room once full events are being refused. + */ + private final long eventByteCapacity; + /** How long a batch may go unwritten before it is discarded rather than delivered. */ + private final long maxAgeMillis; + + /** + * Guards the buffer and the counters. Held for a copy and never while writing, so an + * evaluation recording an event waits on another thread's array copy at worst, never on the disk. + */ + private final Object bufferLock = new Object(); + /** + * Serializes writes so that two threads committing at once cannot interleave a frame. Taken before + * {@link #bufferLock}, never after it. + */ + private final Object ioLock = new Object(); + + // Guarded by bufferLock. + /** Full events staged, {@link #bufferedEventCount} of them. */ + private final FrameBuffer bufferData = new FrameBuffer(); + private int bufferedEventCount; + /** + * Summaries staged, {@link #bufferedSummaryCount} of them, apart from the full events because capacity + * does not count them. Everything staged is taken as the full events followed by the summaries, so that + * any part of it can say how many of its frames are summaries from where the summaries start. + */ + private final FrameBuffer summaryData = new FrameBuffer(); + private int bufferedSummaryCount; + /** + * How many of the frames counted in each state below are summaries, state for state, so capacity can + * leave them out. Changed in the same critical sections as the counts beside them. A batch this store + * did not close - one a previous run or another process left - counts none: its name says how many + * frames it holds, not what they are, so every one counts as an event, which errs towards refusing. + */ + private int inTransitSummaries; + private int committedSummaries; + private int closedSummaries; + private int sendingSummaries; + /** + * Events taken out of the buffer to be written to the log, or assembled into a batch held in + * memory, and not yet counted where they land. Both happen outside this lock, and for as long as + * they take the events are in neither place; without this they would be counted nowhere, and + * capacity would admit that many more. + */ + private int inTransitEvents; + private int committedEvents; + /** + * Events in closed batches: those the {@link #disk} is holding plus those in {@link #inMemoryBatches}. + * Changed only in the critical section that changes one of those, so that it cannot drift from them; a + * total worked out from them and written afterward would overwrite a close or a removal in between. + */ + private int closedEvents; + /** + * Closed batches whose request is out, by payload ID, and the events in them as {@link #closedEvents} + * counts them. Every one is also counted there: one is only added while it is counted, and leaves in the + * same critical section that stops counting it, so {@link #sendingEvents} can never exceed + * {@link #closedEvents}. + */ + private final Map sendingBatches = new HashMap<>(); + private int sendingEvents; + /** + * The bytes of the events counted above, state for state. Staged bytes are {@link #bufferData} itself. + * Changed in the same critical sections as the event counts beside them, so that bytes and events are + * never counted in different places. + */ + private long inTransitBytes; + private long committedBytes; + private long closedBytes; + private long sendingBytes; + /** + * Events a caller has taken to encode and promised to stage, counted against capacity until each is + * staged. Staging one moves it from here to {@link #bufferedEventCount} under one lock, so an event + * on its way in is counted exactly once throughout. + */ + private int reservedEvents; + /** + * Whether events are being written to disk, which is what the application asked for until writing + * fails and the store gives up on persistence for the rest of the session. + *

+ * Written under {@link #bufferLock} but volatile, so {@link #isPersisting} can answer without it: + * every {@code track} asks, and a commit takes that lock once for each event it stages. + */ + private volatile boolean persistEvents; + + /** + * Guards {@link #inMemoryBatches}. + *

+ * Its own lock rather than {@code ioLock}, so that a store the application told to keep off the disk + * never waits on the lock disk I/O is serialized under; there, held batches are the only ones there + * are. Taken inside {@code ioLock} where there is a disk, never the other way round, and taken + * before {@code bufferLock}, never after it. + */ + private final Object heldLock = new Object(); + /** + * Batches held in memory because the filesystem would not take them, or because it was never asked. + *

+ * Persistence failing should cost durability and nothing else, so the store falls back to what the + * SDK did before it existed: hold the events, deliver them, lose them only if the process dies. + *

+ * Only to be used while holding {@link #heldLock}. + */ + private final Map inMemoryBatches = new LinkedHashMap<>(); + + /** + * Whether the directories of other environments have been looked at for ones nobody uses anymore, + * which is done once per process. Only to be used while holding {@link #ioLock}. + */ + private boolean hasDiscardedAbandonedEnvironments; + + EventStore(final File directory, int capacity, boolean persistEvents, LDLogger logger) { + this(() -> directory, capacity, EventProcessorBuilder.DEFAULT_CAPACITY_BYTES, + EventProcessorBuilder.DEFAULT_MAX_AGE_MILLIS, persistEvents, logger); + } + + @VisibleForTesting + EventStore(final File directory, int capacity, long byteCapacity, long maxAgeMillis, + boolean persistEvents, LDLogger logger) { + this(() -> directory, capacity, byteCapacity, maxAgeMillis, persistEvents, logger); + } + + private EventStore(EventDirectory.Location location, int capacity, long byteCapacity, long maxAgeMillis, + boolean persistEvents, LDLogger logger) { + this.capacity = capacity >= 0 ? capacity : 1; + this.byteCapacity = byteCapacity > 0 ? byteCapacity : EventProcessorBuilder.DEFAULT_CAPACITY_BYTES; + this.eventByteCapacity = this.byteCapacity - this.byteCapacity / 5; + this.maxAgeMillis = maxAgeMillis > 0 ? maxAgeMillis : EventProcessorBuilder.DEFAULT_MAX_AGE_MILLIS; + // An application that has not asked for persistence gets the same store running the same way it + // runs once writing has failed: events are held, delivered, and lost only if the process dies. + // It also skips every read, so batches a run with persistence turned on left behind stay on disk + // until persistence is turned back on. + this.persistEvents = persistEvents; + this.disk = persistEvents + ? new EventDirectory(location, ioLock, logger, this::openForReading, this::openForAppending) + : null; + } + + /** + * Creates the store for one environment of one process, in a directory the caller already has, with the + * default byte capacity and maximum age. The SDK itself uses the {@link PlatformState} overload. + * + * @param noBackupFilesDir where the platform lets the SDK keep files it must not lose + * @param mobileKey identifies the environment, so several clients stay out of each other's way + * @param persistEvents whether the application asked for events to outlive the process + */ + @VisibleForTesting + static EventStore create( + File noBackupFilesDir, + String mobileKey, + int capacity, + boolean persistEvents, + LDLogger logger + ) { + return new EventStore(EventDirectory.defaultDirectory(noBackupFilesDir, mobileKey), capacity, + EventProcessorBuilder.DEFAULT_CAPACITY_BYTES, EventProcessorBuilder.DEFAULT_MAX_AGE_MILLIS, + persistEvents, logger); + } + + /** + * Creates the store for one environment of one process, leaving the platform to be asked where on + * first use, from one of the store's own threads. + * + * @param platformState where the no-backup directory comes from + * @param mobileKey identifies the environment, so several clients stay out of each other's way + * @param byteCapacity the most bytes the store holds, a fifth of which only summaries may use + * @param maxAgeMillis how long a batch may go unwritten before it is discarded + * @param persistEvents whether the application asked for events to outlive the process + */ + static EventStore create( + final PlatformState platformState, + String mobileKey, + int capacity, + long byteCapacity, + long maxAgeMillis, + boolean persistEvents, + LDLogger logger + ) { + EventDirectory.Location location = + () -> EventDirectory.defaultDirectory(platformState.getNoBackupFilesDir(), mobileKey); + return new EventStore(location, capacity, byteCapacity, maxAgeMillis, persistEvents, logger); + } + + // MARK: recording + + /** + * A batch being sent is left out so that a flush makes room as soon as it sends, as it did when events + * were only held in memory. Otherwise an event recorded while the request is out is refused for a batch + * LaunchDarkly may already have. A failed request puts the batch back, so a full store can briefly + * hold capacity plus that batch: one at a time, since deliveries run one after another, and no more than + * {@link EventLogFormat#MAX_BATCH_SIZE}. + * + * @return serialized events counted against capacity: those recorded and not yet accepted by + * LaunchDarkly, whether staged in memory, on their way out of it, written to the open log, or sitting + * in a closed batch - except a batch whose request is out, and except the summaries this store wrote + */ + int getPendingEventCount() { + synchronized (bufferLock) { + return countedEventsHoldingBufferLock(); + } + } + + /** + * Every event counted against capacity. The one place the counters are added up, so that neither + * reader can miss one of them. Requires {@link #bufferLock}. + */ + private int countedEventsHoldingBufferLock() { + // Summaries left out: they are not events but the record of evaluations already counted, and + // counting them would let the ones a long outage writes at every commit point refuse full events. + return bufferedEventCount + (inTransitEvents - inTransitSummaries) + (committedEvents - committedSummaries) + + ((closedEvents - closedSummaries) - (sendingEvents - sendingSummaries)) + reservedEvents; + } + + /** + * Every byte counted against the byte capacity. Requires {@link #bufferLock}. + *

+ * Events reserved while being encoded have no bytes yet; they are counted when they are staged. + */ + private long countedBytesHoldingBufferLock() { + return bufferData.size() + summaryData.size() + inTransitBytes + committedBytes + (closedBytes - sendingBytes); + } + + /** + * Whether full events can still be admitted by size, asked before an event is kept for encoding so + * that one there is no room for is refused without the cost of encoding it. Staging checks again + * against the event's actual size. + */ + boolean hasRoomForEventBytes() { + // A store kept off the disk holds events the way the SDK did before persistence, bounded by count + // alone. + if (disk == null) { + return true; + } + synchronized (bufferLock) { + return countedBytesHoldingBufferLock() < eventByteCapacity; + } + } + + /** + * {@link #getPendingEventCount} and {@link #hasRoomForEventBytes} in one acquisition of the lock, for the + * check every recorded event makes, where the lock is contended by the commit staging events. + * + * @param heldElsewhere events the caller holds that the store does not count yet + * @param capacity the most events to hold, counting {@code heldElsewhere} + * @return whether one more full event can be admitted, by count and by bytes + */ + boolean admitsEvent(int heldElsewhere, int capacity) { + synchronized (bufferLock) { + return heldElsewhere + countedEventsHoldingBufferLock() < capacity + && (disk == null || countedBytesHoldingBufferLock() < eventByteCapacity); + } + } + + /** + * Counts events against capacity before they are staged, for a caller that has taken them from + * somewhere capacity was already counting them and will encode them before staging. + *

+ * Each {@link #stageReserved} uses one up, and {@link #releaseReservations} gives back any that + * never made it, such as an event that could not be serialized. + */ + void reserve(int events) { + synchronized (bufferLock) { + reservedEvents += events; + } + } + + /** Gives back every reservation not used up by {@link #stageReserved}. */ + void releaseReservations() { + synchronized (bufferLock) { + reservedEvents = 0; + } + } + + /** + * @return whether a commit writes anything, which is false where persistence is off and once writing + * has failed and the store has given up on it + */ + boolean isPersisting() { + return persistEvents; + } + + /** + * Stages an already serialized event. + *

+ * Staging is a copy into a buffer and never writes, however much accumulates; {@link #commit()} is what + * makes the event outlive the process, so how many bytes one write carries is the caller's decision. + * + * @return false if the event was dropped because capacity or the byte capacity is reached + */ + boolean stage(byte[] serializedEvent) { + return stage(serializedEvent, false); + } + + /** + * @param bypassingCapacity true for a summary, which must be recorded even when the store is full of + * events, and may use the bytes held back from them; it is still refused once the byte capacity + * itself is reached, whether or not events are persisted + */ + boolean stage(byte[] serializedEvent, boolean bypassingCapacity) { + return stage(serializedEvent, bypassingCapacity, false); + } + + /** + * Stages an event {@link #reserve} already counted, turning its reservation into a staged event in + * one step. Bypasses capacity, which the reservation already answered to, but not the bytes left for + * full events, which only the event's encoded size can answer. + */ + boolean stageReserved(byte[] serializedEvent) { + return stage(serializedEvent, true, true); + } + + private boolean stage(byte[] serializedEvent, boolean bypassingCapacity, boolean reserved) { + if (serializedEvent == null || serializedEvent.length == 0 + || serializedEvent.length > EventLogFormat.MAX_FRAME_SIZE) { + return false; + } + long frameSize = (long) serializedEvent.length + EventLogFormat.FRAME_HEADER_SIZE; + // Only a summary bypasses capacity without a reservation, and only a summary may use the bytes held + // back. + boolean isSummary = bypassingCapacity && !reserved; + + synchronized (bufferLock) { + // In the same critical section as the append, so the event is never counted twice or not at all. + if (reserved && reservedEvents > 0) { + reservedEvents--; + } + if (!bypassingCapacity && countedEventsHoldingBufferLock() >= capacity) { + return false; + } + // Kept off the disk, full events are bounded by count alone, as they were before persistence. A + // summary bypasses the count wherever it is kept, so there its bytes are all that bounds it: a + // delivery that keeps failing would otherwise go on holding one more for the life of the process. + if ((disk != null || isSummary) + && countedBytesHoldingBufferLock() + frameSize > (isSummary ? byteCapacity : eventByteCapacity)) { + return false; + } + if (isSummary) { + EventLogFormat.writeFrame(summaryData, serializedEvent); + bufferedSummaryCount++; + } else { + EventLogFormat.writeFrame(bufferData, serializedEvent); + bufferedEventCount++; + } + } + return true; + } + + /** A buffer of frames that copies into a larger array without a copy of its own on the way. */ + private static final class FrameBuffer extends ByteArrayOutputStream { + void copyInto(byte[] target, int offset) { + System.arraycopy(buf, 0, target, offset, count); + } + } + + /** + * Everything staged, taken out of the buffers in one step: the full events, then the summaries. + */ + private static final class Staged { + final byte[] frames; + /** Where in {@link #frames} the summaries start. */ + final int summariesFrom; + /** Every frame in {@link #frames}, summaries included. */ + final int events; + final int summaries; + + Staged(byte[] frames, int summariesFrom, int events, int summaries) { + this.frames = frames; + this.summariesFrom = summariesFrom; + this.events = events; + this.summaries = summaries; + } + + /** How many of the frames from {@code from} up to {@code to} are summaries. */ + int summariesIn(int from, int to) { + return to <= summariesFrom ? 0 : EventLogFormat.frameCount(frames, Math.max(from, summariesFrom), to); + } + } + + /** + * Takes everything staged, counting it in transit until it is counted where it lands. Requires + * {@link #bufferLock}. + */ + private Staged takeStagedHoldingBufferLock() { + byte[] frames; + if (summaryData.size() == 0) { + frames = bufferData.toByteArray(); + } else { + frames = new byte[bufferData.size() + summaryData.size()]; + bufferData.copyInto(frames, 0); + summaryData.copyInto(frames, bufferData.size()); + } + Staged staged = new Staged(frames, bufferData.size(), bufferedEventCount + bufferedSummaryCount, + bufferedSummaryCount); + bufferData.reset(); + summaryData.reset(); + bufferedEventCount = 0; + bufferedSummaryCount = 0; + inTransitEvents += staged.events; + inTransitSummaries += staged.summaries; + inTransitBytes += frames.length; + return staged; + } + + /** + * Puts back what of {@code staged} from {@code from} on never landed, ahead of anything staged since, to + * be delivered from memory rather than lost. Requires {@link #bufferLock}. + */ + private void putBackHoldingBufferLock(Staged staged, int from) { + byte[] frames = staged.frames; + int summariesFrom = Math.max(from, staged.summariesFrom); + int events = EventLogFormat.frameCount(frames, from, summariesFrom); + int summaries = EventLogFormat.frameCount(frames, summariesFrom, frames.length); + inTransitEvents -= events + summaries; + inTransitSummaries -= summaries; + inTransitBytes -= frames.length - from; + prepend(bufferData, frames, from, summariesFrom); + prepend(summaryData, frames, summariesFrom, frames.length); + bufferedEventCount += events; + bufferedSummaryCount += summaries; + } + + private static void prepend(ByteArrayOutputStream target, byte[] frames, int from, int to) { + if (from == to) { + return; + } + byte[] later = target.toByteArray(); + target.reset(); + target.write(frames, from, to - from); + writeQuietly(target, later); + } + + /** + * Hands every staged byte to the kernel, so the events recorded so far survive the process dying. + */ + void commit() { + // Answered before ioLock, which a delivery can hold for as long as it takes to read a batch + // back: with persistence off there is nothing to write, so nothing worth waiting for. + synchronized (bufferLock) { + if (!persistEvents) { + return; + } + } + synchronized (ioLock) { + commitHoldingIoLock(); + } + } + + // MARK: delivery + + /** + * Commits, then closes the open log into a batch to deliver, and whatever is still staged in memory + * into another. + *

+ * Both can hold events at once. Writing that fails after earlier writes succeeded gives up on + * persistence with those earlier events in the log and the failed ones back in memory. Closing + * only the log would leave the rest for a delivery that reports success without them. + * + * @return the batches this closed, oldest first: the one from the log, then those from memory; usually + * one, and empty when there is nothing to send. A log a commit filled was closed into a batch of its + * own as it filled, and is not among them; {@link #pendingBatches()} lists every batch. + */ + List closeBatch() { + List closed = new ArrayList<>(1); + // Where the application asked for no disk there is no log to close, and so nothing to close it + // alongside. + if (disk == null) { + closed.addAll(closeInMemoryBatches()); + return closed; + } + synchronized (ioLock) { + commitHoldingIoLock(); + + int events; + int summaries; + long bytes; + int stillBuffered; + synchronized (bufferLock) { + events = committedEvents; + summaries = committedSummaries; + bytes = committedBytes; + stillBuffered = bufferedEventCount + bufferedSummaryCount; + } + + if (events > 0) { + addIfClosed(closed, closeOpenLogHoldingIoLock(events, summaries, bytes)); + } + // Only reachable once persistence has been given up on, or a full log could not be closed; a + // healthy commit leaves nothing staged behind. Closed after the log, so the older events are + // delivered first. + if (stillBuffered > 0) { + closed.addAll(closeInMemoryBatches()); + } + return closed; + } + } + + /** + * Whether committed events are outside every batch. Right after {@link #closeBatch()}, that means it could + * not close the log holding them; they stay in that log for the next close to try again. + */ + boolean holdsUnclosedEvents() { + synchronized (bufferLock) { + return committedEvents > 0; + } + } + + private static void addIfClosed(List closed, Batch batch) { + if (batch != null) { + closed.add(batch); + } + } + + /** + * Requires {@code ioLock}. + * + * @param summaries how many of {@code events} are summaries + * @param bytes the frames committed to the log, which with the file header are the batch's size + */ + private Batch closeOpenLogHoldingIoLock(int events, int summaries, long bytes) { + long batchBytes = bytes + EventLogFormat.FILE_HEADER_SIZE; + EventDirectory.LogClose closed = disk.closeLog(events, summaries, batchBytes); + if (closed.batch == null) { + if (!closed.logSurvives) { + // The log itself is gone, so its events are too. They have to stop counting against + // capacity or the store would refuse events for the rest of the session. + synchronized (bufferLock) { + committedEvents = 0; + committedSummaries = 0; + committedBytes = 0; + } + } + return null; + } + + synchronized (bufferLock) { + committedEvents = 0; + committedSummaries = 0; + committedBytes = 0; + closedEvents += events; + closedSummaries += summaries; + closedBytes += batchBytes; + } + return closed.batch; + } + + /** + * Closes whatever is staged into batches that never reach a file, as many as it takes to keep each + * within {@link EventLogFormat#MAX_BATCH_SIZE}. + *

+ * Called under {@code ioLock} where there is a disk, so that it and the log are closed as one step, + * and on its own where there is not. + * + * @return the batches, oldest first + */ + private List closeInMemoryBatches() { + Staged staged; + synchronized (bufferLock) { + staged = takeStagedHoldingBufferLock(); + } + byte[] frames = staged.frames; + List closed = new ArrayList<>(1); + int held = 0; + int heldEvents = 0; + int heldSummaries = 0; + try { + // Outside bufferLock, because it copies every staged frame; inTransitEvents is what keeps + // them counted meanwhile. + while (held < frames.length) { + int end = EventLogFormat.endOfFramesFitting(frames, held, + EventLogFormat.MAX_BATCH_SIZE - EventLogFormat.FILE_HEADER_SIZE); + int batchEvents = EventLogFormat.frameCount(frames, held, end); + int batchSummaries = staged.summariesIn(held, end); + EventLogFormat.Payload payload = EventLogFormat.assemblePayloadInPlace( + EventLogFormat.withFileHeader(frames, held, end), Integer.MAX_VALUE); + if (payload == null) { + break; + } + String payloadId = UUID.randomUUID().toString(); + synchronized (heldLock) { + inMemoryBatches.put(payloadId, new HeldBatch(payload, batchEvents, batchSummaries)); + synchronized (bufferLock) { + inTransitEvents -= batchEvents; + inTransitSummaries -= batchSummaries; + inTransitBytes -= end - held; + closedEvents += batchEvents; + closedSummaries += batchSummaries; + closedBytes += payload.length; + } + } + closed.add(new Batch(payloadId, batchEvents)); + held = end; + heldEvents += batchEvents; + heldSummaries += batchSummaries; + } + } finally { + if (held < frames.length) { + synchronized (bufferLock) { + inTransitEvents -= staged.events - heldEvents; + inTransitSummaries -= staged.summaries - heldSummaries; + inTransitBytes -= frames.length - held; + } + } + } + return closed; + } + + /** A batch the filesystem would not take, kept where the SDK used to keep all of them. */ + private static final class HeldBatch { + final EventLogFormat.Payload payload; + final int eventCount; + /** How many of {@link #eventCount} are summaries. */ + final int summaries; + + HeldBatch(EventLogFormat.Payload payload, int eventCount, int summaries) { + this.payload = payload; + this.eventCount = eventCount; + this.summaries = summaries; + } + + EventDirectory.Tally tally() { + return new EventDirectory.Tally(eventCount, summaries, payload.length); + } + } + + /** The batches awaiting delivery, and whether that is all of them. */ + static final class Listing { + final List batches; + /** + * False where the directory could not be listed. {@link #batches} are then the ones this process last + * knew to be there, which includes every batch it closed itself, but can miss one another process + * closed or a previous run left. + */ + final boolean complete; + + Listing(List batches, boolean complete) { + this.batches = batches; + this.complete = complete; + } + } + + /** + * @return batches awaiting delivery, oldest first, including any left by a previous run + */ + List pendingBatches() { + return listPendingBatches().batches; + } + + /** + * {@link #pendingBatches()}, saying whether the listing was complete. A directory that could not be listed + * leaves the batches this store last counted, and what it counts for them, as they were. It does not + * empty them. + */ + Listing listPendingBatches() { + // Listed in one critical section where there is a disk, so that the files and what is held cannot + // be counted either side of a close that moves events between them. + if (disk != null) { + synchronized (ioLock) { + Listing listing = disk.listBatches(); + addHeldBatchesAndRecount(listing.batches, disk.storedSummaries(), disk.storedBytes()); + return listing; + } + } + List batches = new ArrayList<>(); + addHeldBatchesAndRecount(batches, 0, 0); + return new Listing(batches, true); + } + + /** + * Adds the batches the filesystem would not take, after the files: anything held was closed by this run + * and so is newer than whatever a previous run left on the disk. Then counts everything listed as + * closed, which is how batches another process closed or delivered come to be counted or not. + *

+ * Requires {@code ioLock} where there is a disk, so that the count is of the listing as it stands. + * + * @param storedSummaries the summaries this store knows of in the batches the listing found on the disk + * @param storedBytes the bytes in the batches the listing found on the disk + */ + private void addHeldBatchesAndRecount(List batches, int storedSummaries, long storedBytes) { + synchronized (heldLock) { + int totalSummaries = storedSummaries; + long totalBytes = storedBytes; + for (Map.Entry held : inMemoryBatches.entrySet()) { + batches.add(new Batch(held.getKey(), held.getValue().eventCount)); + totalSummaries += held.getValue().summaries; + totalBytes += held.getValue().payload.length; + } + int total = 0; + Set listed = new HashSet<>(); + for (Batch batch : batches) { + total += batch.eventCount; + listed.add(batch.payloadId); + } + synchronized (bufferLock) { + closedEvents = total; + closedSummaries = totalSummaries; + closedBytes = totalBytes; + // One no longer listed is no longer counted, so it cannot be left out of the count either. + for (String payloadId : new ArrayList<>(sendingBatches.keySet())) { + if (!listed.contains(payloadId)) { + forgetSendingHoldingBufferLock(payloadId); + } + } + } + } + } + + /** + * A batch that is there to read and holds nothing this version can send: damaged, or written in a + * format it does not read. Unlike a batch that is gone, its events are lost, and unlike any other + * {@link IOException}, no later attempt changes that. + */ + static final class UnreadableBatchException extends IOException { + UnreadableBatchException(String payloadId) { + super("batch " + payloadId + " holds nothing this version can send"); + } + } + + /** + * @return the JSON payload for a batch, or null if it is gone or holds no events, neither of which + * loses anything. A batch on the disk is read into one array and assembled there, so it is held once. + * @throws UnreadableBatchException if the batch is there but holds nothing this version can send + * @throws IOException if the batch is there but could not be read this time, which a later attempt may + * manage + */ + EventLogFormat.Payload payload(Batch batch) throws IOException { + synchronized (heldLock) { + HeldBatch held = inMemoryBatches.get(batch.payloadId); + if (held != null) { + return held.payload; + } + } + + // A batch is either held or on the disk, never both and never moved between them, so a batch that + // is not held in a store with no disk does not exist. + if (disk == null) { + return null; + } + + synchronized (ioLock) { + return disk.payload(batch.payloadId, batch.eventCount); + } + } + + /** {@link #payload} in an array of its own. */ + @VisibleForTesting + byte[] body(Batch batch) throws IOException { + EventLogFormat.Payload payload = payload(batch); + return payload == null ? null : payload.toByteArray(); + } + + /** + * Removes a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. + */ + void remove(Batch batch) { + // Only a count this call takes away stops counting. Two deliveries can both get to the same batch, + // and a listing drops a batch another process delivered before this one gets to remove it. + synchronized (heldLock) { + HeldBatch held = inMemoryBatches.remove(batch.payloadId); + if (held != null) { + synchronized (bufferLock) { + closedEvents -= held.eventCount; + closedSummaries -= held.summaries; + closedBytes -= held.payload.length; + forgetSendingHoldingBufferLock(batch.payloadId); + } + return; + } + } + + if (disk == null) { + synchronized (bufferLock) { + forgetSendingHoldingBufferLock(batch.payloadId); + } + return; + } + synchronized (ioLock) { + EventDirectory.Tally counted = disk.remove(batch.payloadId, batch.eventCount); + synchronized (bufferLock) { + closedEvents -= counted.events; + closedSummaries -= counted.summaries; + closedBytes -= counted.bytes; + forgetSendingHoldingBufferLock(batch.payloadId); + } + } + } + + /** + * Stops counting a batch against capacity while its request is out. Ended by {@link #remove} or + * {@link #stoppedSending}. + *

+ * Does nothing for a batch this store no longer counts - one {@link #remove} forgot, or a listing + * dropped because another process delivered it - since leaving it out would take its events from the + * batches still counted. + *

+ * A batch on the disk is also locked against expiry, by this process and any other, until + * {@link #stoppedSending}: one deleted while its request is out would leave nothing to retry from. + */ + void startedSending(Batch batch) { + if (disk == null) { + startedSendingHoldingIoLock(batch.payloadId); + return; + } + synchronized (ioLock) { + startedSendingHoldingIoLock(batch.payloadId); + if (!isHeldInMemory(batch.payloadId)) { + disk.startedSending(batch.payloadId, batch.eventCount); + } + } + } + + private boolean isHeldInMemory(String payloadId) { + synchronized (heldLock) { + return inMemoryBatches.containsKey(payloadId); + } + } + + /** + * Requires {@code ioLock} where there is a disk, so that the batch cannot stop being counted between + * finding it counted and leaving it out. + */ + private void startedSendingHoldingIoLock(String payloadId) { + synchronized (heldLock) { + EventDirectory.Tally counted; + HeldBatch held = inMemoryBatches.get(payloadId); + if (held != null) { + counted = held.tally(); + } else if (disk != null) { + counted = disk.counted(payloadId); + } else { + counted = null; + } + if (counted == null) { + return; + } + synchronized (bufferLock) { + if (!sendingBatches.containsKey(payloadId)) { + sendingBatches.put(payloadId, counted); + sendingEvents += counted.events; + sendingSummaries += counted.summaries; + sendingBytes += counted.bytes; + } + } + } + } + + /** + * Counts a batch against capacity again, after a request that did not end with it removed. Does nothing + * for a batch {@link #remove} already forgot. Ends every {@link #startedSending}, removed or not, so that + * the lock against expiry is released either way. + */ + void stoppedSending(Batch batch) { + if (disk != null) { + synchronized (ioLock) { + disk.stoppedSending(batch.payloadId); + } + } + synchronized (bufferLock) { + forgetSendingHoldingBufferLock(batch.payloadId); + } + } + + /** + * Requires {@link #bufferLock}. + */ + private void forgetSendingHoldingBufferLock(String payloadId) { + EventDirectory.Tally sending = sendingBatches.remove(payloadId); + if (sending != null) { + sendingEvents -= sending.events; + sendingSummaries -= sending.summaries; + sendingBytes -= sending.bytes; + } + } + + /** + * Closes every log no live process holds, so their events join the batches to deliver. + *

+ * The events in them were recorded by processes that are gone, so there is no one left to add to them. + * A log a live process holds, this store's own included, is left alone. + */ + void preparePayloadsFromInterruptedLogs() { + if (disk == null) { + return; + } + // Each log is read without ioLock, one at a time, and the lock taken only to move it and count it. A + // commit at a commit point then waits for at most one rename rather than for every log a crash left to + // be read. + List logs; + synchronized (ioLock) { + logs = disk.interruptedLogs(); + } + for (File log : logs) { + EventDirectory.InterruptedLog interrupted = disk.read(log); + synchronized (ioLock) { + countRecoveredHoldingIoLock(disk.preparePayload(interrupted)); + } + } + } + + /** + * Counts events a recovery closed off into batches of their own, which belong to the closed batches + * from the moment the recovery moved them there. + *

+ * Requires {@code ioLock}. + */ + private void countRecoveredHoldingIoLock(EventDirectory.Tally recovered) { + if (recovered.events <= 0) { + return; + } + synchronized (bufferLock) { + closedEvents += recovered.events; + closedSummaries += recovered.summaries; + closedBytes += recovered.bytes; + } + } + + /** + * Deletes stored events older than the maximum age, so that what an application can no longer deliver + * does not keep the device's storage, or this store's capacity, forever. + *

+ * Its own directory first, under {@code ioLock}; then, once per process, the directories of environments + * nothing has written to for as long, outside it, since none of that is this store's. + * + * @return how many events were discarded, which the caller counts as dropped + */ + int discardExpired() { + if (disk == null) { + return 0; + } + long cutoff = System.currentTimeMillis() - maxAgeMillis; + int discarded; + File directory = null; + synchronized (ioLock) { + Set sending; + synchronized (bufferLock) { + sending = new HashSet<>(sendingBatches.keySet()); + } + EventDirectory.Discarded result = disk.discard(cutoff, sending); + // In the same critical section as the deletion, as for any other path that loses events without + // delivering them, so the store cannot go on counting what is gone. + synchronized (bufferLock) { + closedEvents -= result.counted.events; + closedSummaries -= result.counted.summaries; + closedBytes -= result.counted.bytes; + } + discarded = result.events; + if (!hasDiscardedAbandonedEnvironments) { + hasDiscardedAbandonedEnvironments = true; + directory = disk.directory(); + } + } + if (directory != null) { + discarded += EventDirectory.discardAbandonedEnvironments(directory, cutoff); + } + return discarded; + } + + // MARK: writing + + /** + * Requires {@code ioLock}. + *

+ * Closes the open log into a batch wherever the next frame would take it past + * {@link EventLogFormat#MAX_BATCH_SIZE}, so a commit can land in several batches. + */ + private void commitHoldingIoLock() { + Staged staged; + synchronized (bufferLock) { + if (!persistEvents || bufferedEventCount + bufferedSummaryCount == 0) { + // Staged bytes are left where they are: with nowhere durable to put them, memory is + // better than dropping them. + return; + } + staged = takeStagedHoldingBufferLock(); + } + byte[] bytes = staged.frames; + + int written = 0; + try { + while (written < bytes.length) { + long room; + synchronized (bufferLock) { + room = EventLogFormat.MAX_BATCH_SIZE - EventLogFormat.FILE_HEADER_SIZE - committedBytes; + } + if (EventLogFormat.frameSize(bytes, written) > room) { + if (!closeFullLogHoldingIoLock()) { + break; + } + continue; + } + int end = EventLogFormat.endOfFramesFitting(bytes, written, room); + int landing = EventLogFormat.frameCount(bytes, written, end); + int landingSummaries = staged.summariesIn(written, end); + boolean isWhole = written == 0 && end == bytes.length; + if (!append(isWhole ? bytes : Arrays.copyOfRange(bytes, written, end))) { + break; + } + // Counted where they landed in the same critical section that stops counting them in + // transit, so a capacity check never sees them in both places or in neither. + synchronized (bufferLock) { + inTransitEvents -= landing; + inTransitSummaries -= landingSummaries; + inTransitBytes -= end - written; + committedEvents += landing; + committedSummaries += landingSummaries; + committedBytes += end - written; + } + written = end; + } + } finally { + if (written < bytes.length) { + synchronized (bufferLock) { + putBackHoldingBufferLock(staged, written); + } + } + } + } + + /** + * Closes an open log that has no room for the next frame, so that the frame starts a batch of its own. + * Requires {@code ioLock}. + * + * @return whether the next append starts a new log; false if this one could not be closed, which + * leaves the frames to be held in memory rather than grow it past the size of a batch + */ + private boolean closeFullLogHoldingIoLock() { + int events; + int summaries; + long bytes; + synchronized (bufferLock) { + events = committedEvents; + summaries = committedSummaries; + bytes = committedBytes; + } + closeOpenLogHoldingIoLock(events, summaries, bytes); + synchronized (bufferLock) { + return committedBytes == 0; + } + } + + /** + * Requires {@code ioLock}. + * + * @return false if the bytes could not be written, in which case persistence is given up on + */ + private boolean append(byte[] bytes) { + synchronized (bufferLock) { + if (!persistEvents) { + return false; + } + } + if (!disk.append(bytes)) { + synchronized (bufferLock) { + persistEvents = false; + } + return false; + } + return true; + } + + /** Overridable so a test can make writing fail partway through a session, as a full disk does. */ + @VisibleForTesting + FileOutputStream openForAppending(File file) throws IOException { + return new FileOutputStream(file, true); + } + + @Override + public void close() { + synchronized (ioLock) { + commitHoldingIoLock(); + if (disk != null) { + disk.closeOpenLog(); + } + } + } + + /** @return where the events are kept, or null where the application asked for no disk */ + File getDirectory() { + if (disk == null) { + return null; + } + synchronized (ioLock) { + return disk.directory(); + } + } + + private static void writeQuietly(ByteArrayOutputStream target, byte[] bytes) { + try { + target.write(bytes); + } catch (IOException e) { + // ByteArrayOutputStream.write does not throw; this only satisfies the compiler. + } + } + + /** + * Overridable so a test can make a read fail while the file is intact. Not {@code Files.newInputStream}, + * which needs API 26. + */ + @SuppressWarnings("IOStreamConstructor") + @VisibleForTesting + InputStream openForReading(File file) throws IOException { + return new FileInputStream(file); + } + + /** + * A group of serialized events closed off for delivery. + *

+ * The identifier doubles as the payload ID sent to LaunchDarkly, so a batch retried after the process + * died mid-delivery is recognized upstream as the same delivery rather than counted twice. + */ + static final class Batch { + final String payloadId; + final int eventCount; + + Batch(String payloadId, int eventCount) { + this.payloadId = payloadId; + this.eventCount = eventCount; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Batch)) { + return false; + } + Batch o = (Batch) other; + return payloadId.equals(o.payloadId) && eventCount == o.eventCount; + } + + @Override + public int hashCode() { + return payloadId.hashCode() * 31 + eventCount; + } + + @NonNull + @Override + public String toString() { + return "Batch(" + payloadId + ", " + eventCount + " events)"; + } + } + + /** + * Every event the store is holding, serialized exactly as it will be sent. + *

+ * Reading the log means committing what is staged, so this is not free and is meant for tests and for + * diagnosing a store rather than for the recording path. + */ + List pendingEventPayloads() { + commit(); + + List payloads = new ArrayList<>(); + if (disk != null) { + synchronized (ioLock) { + payloads.addAll(disk.storedPayloads()); + } + } + synchronized (bufferLock) { + if (bufferedEventCount > 0) { + payloads.addAll(EventLogFormat.eventPayloads( + EventLogFormat.withFileHeader(bufferData.toByteArray()))); + } + if (bufferedSummaryCount > 0) { + payloads.addAll(EventLogFormat.eventPayloads( + EventLogFormat.withFileHeader(summaryData.toByteArray()))); + } + } + return payloads; + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventUtil.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventUtil.java index e605de085..e4c71f6a3 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventUtil.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventUtil.java @@ -15,6 +15,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -22,17 +23,46 @@ abstract class EventUtil { static ScheduledExecutorService makeEventsTaskExecutor() { - return Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + return Executors.newSingleThreadScheduledExecutor( + makeThreadFactory("LaunchDarkly-DefaultEventProcessor-%d")); + } + + /** + * A thread of its own for posting diagnostic events, kept apart from the one that delivers + * analytics events. + *

+ * A diagnostic post is an HTTP request like any other and can hold its thread for tens of seconds + * against a network that accepts connections and never answers. Sharing a thread with analytics + * delivery would let that stall the flushes, and an event buffer that is not being drained fills + * up and starts dropping what the application asked to send. This mirrors the separation + * {@code DefaultEventProcessor} has, where analytics go out on dedicated delivery workers and + * diagnostics on the shared executor. + */ + static ExecutorService makeDiagnosticsTaskExecutor() { + return Executors.newSingleThreadExecutor( + makeThreadFactory("LaunchDarkly-DiagnosticEventPoster-%d")); + } + + /** + * A thread of its own for the event commits no caller waits on, kept apart from the one that + * delivers, so that a write never queues behind a request held open by a slow network. + */ + static ExecutorService makeEventsCommitExecutor() { + return Executors.newSingleThreadExecutor( + makeThreadFactory("LaunchDarkly-event-store-%d")); + } + + private static ThreadFactory makeThreadFactory(String nameFormat) { + return new ThreadFactory() { final AtomicLong count = new AtomicLong(0); @Override public Thread newThread(@NonNull Runnable r) { Thread thread = Executors.defaultThreadFactory().newThread(r); - thread.setName(String.format(Locale.ROOT, "LaunchDarkly-DefaultEventProcessor-%d", - count.getAndIncrement())); + thread.setName(String.format(Locale.ROOT, nameFormat, count.getAndIncrement())); thread.setDaemon(true); return thread; } - }); + }; } static DiagnosticStore.SdkDiagnosticParams makeDiagnosticParams(ClientContext clientContext) { diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClient.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClient.java index f224668a1..18e1cda20 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClient.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClient.java @@ -6,6 +6,7 @@ import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; +import com.launchdarkly.logging.LDLogLevel; import com.launchdarkly.logging.LDLogger; import com.launchdarkly.logging.LogValues; import com.launchdarkly.sdk.EvaluationDetail; @@ -37,6 +38,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -84,6 +87,13 @@ public class LDClient implements LDClientInterface, Closeable { // If 15 seconds or more is passed as a timeout to init, we will log a warning. private static final int EXCESSIVE_INIT_WAIT_SECONDS = 15; + // Flags already reported unknown at info level. An application can poll a flag that does not exist + // on every redraw, and each of those would otherwise write to logcat. Bounded because the keys come + // from the application; past the bound an unknown flag is reported at debug only. + private static final int MAX_UNKNOWN_FLAGS_REPORTED = 100; + private final Set unknownFlagsReported = + Collections.newSetFromMap(new ConcurrentHashMap()); + // Shared by both registration paths, so that a plugin failing during init and the same plugin // failing under registerPlugin are reported identically. private static final String PLUGIN_GET_HOOKS_ERROR = @@ -641,7 +651,11 @@ private EvaluationDetail variationDetailInternal(@NonNull String key, @ EvaluationDetail result; if (flag == null) { - logger.info("Unknown feature flag \"{}\"; returning default value", key); + if (unknownFlagsReported.size() < MAX_UNKNOWN_FLAGS_REPORTED && unknownFlagsReported.add(key)) { + logger.info("Unknown feature flag \"{}\"; returning default value", key); + } else { + logger.debug("Unknown feature flag \"{}\"; returning default value", key); + } eventProcessor.recordEvaluationEvent(context, key, EventProcessor.NO_VERSION, EvaluationDetail.NO_VARIATION, defaultValue, null, defaultValue, false, null); @@ -700,7 +714,11 @@ private EvaluationDetail variationDetailInternal(@NonNull String key, @ ); } - logger.debug("returning variation: {} flagKey: {} context key: {}", result, key, context.getKey()); + // Checked first: three arguments go through the varargs overload, which builds its array whether + // or not debug logging is on. + if (logger.isEnabled(LDLogLevel.DEBUG)) { + logger.debug("returning variation: {} flagKey: {} context key: {}", result, key, context.getKey()); + } return result; } @@ -779,6 +797,55 @@ private void flushInternal() { eventProcessor.flush(); } + @Override + public boolean flushAndWait(long timeout, TimeUnit unit) { + // Clamped because toNanos saturates: a timeout at Long.MIN_VALUE nanos would make the + // remaining time below underflow, and wrap round to a wait with no bound at all. + long deadline = System.nanoTime() + Math.max(0, unit.toNanos(timeout)); + Map clients = getInstancesIfTheyIncludeThisClient(); + if (clients.isEmpty()) { + // This client has been closed, or replaced by a later init; either way it can deliver + // nothing, and saying otherwise would tell the caller its events were safe. + return false; + } + // Every environment is started before any of them is waited on. Each has its own event + // processor and its own thread, so waiting on one before starting the next would spend the + // caller's budget on deliveries that could have been running all along. + List> deliveries = new ArrayList<>(clients.size()); + for (LDClient client : clients.values()) { + deliveries.add(client.eventProcessor.flushAsync()); + } + boolean delivered = true; + for (Future delivery : deliveries) { + // Each wait gets what is left of the one budget rather than a fresh copy of it, so that + // the timeout the caller asked for is the time this call can take. + delivered &= awaitDelivery(delivery, Math.max(0, deadline - System.nanoTime())); + } + return delivered; + } + + private boolean awaitDelivery(Future delivery, long remainingNanos) { + try { + return Boolean.TRUE.equals(delivery.get(remainingNanos, TimeUnit.NANOSECONDS)); + } catch (TimeoutException e) { + // Left running rather than canceled: the events have been taken out of the buffer by + // now, so interrupting the delivery would only make losing them certain. + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (CancellationException e) { + // Not something the SDK's own processor does, but a custom one can hand back a future + // that is canceled, and that must not escape a call whose answer is a boolean. + return false; + } catch (ExecutionException e) { + Throwable cause = e.getCause() == null ? e : e.getCause(); + logger.error("Exception caught when flushing events: {}", LogValues.exceptionSummary(cause)); + logger.debug("{}", LogValues.exceptionTrace(cause)); + return false; + } + } + @VisibleForTesting void blockingFlush() { eventProcessor.blockingFlush(); diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClientInterface.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClientInterface.java index dd62ec5c9..43ecd4f95 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClientInterface.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClientInterface.java @@ -11,6 +11,7 @@ import java.io.Closeable; import java.util.Map; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; /** * The interface for the LaunchDarkly SDK client. @@ -142,10 +143,80 @@ public interface LDClientInterface extends Closeable { Future identify(LDContext context); /** - * Sends all pending events to LaunchDarkly. + * Writes down all pending events and sends them to LaunchDarkly. + *

+ * Recording an event does not on its own make it outlive the process. Where + * {@link com.launchdarkly.sdk.android.integrations.EventProcessorBuilder#eventPersistence( + * com.launchdarkly.sdk.android.integrations.EventPersistence)} is on, + * events are written in runs, so one recorded shortly before the process ends may never have been + * written at all; this call writes everything recorded so far, and the events then survive whether or + * not the delivery does. That write happens on the calling thread before this returns, at + * {@link com.launchdarkly.sdk.android.integrations.EventPersistence#DEFERRED} as well as at + * {@link com.launchdarkly.sdk.android.integrations.EventPersistence#IMMEDIATE}, so a call from the main + * thread touches the disk there. Where persistence is off, events live in memory only and nothing + * survives the process, whether or not this was called. + *

+ * That is what makes it worth calling where the process is about to end. An application that reports + * errors to LaunchDarkly and then crashes should flush from its uncaught exception handler, which runs + * while the process is still alive -- {@link #flushAndWait(long, TimeUnit)} is the better choice there, + * since it also waits for delivery. Terminations that run no application code, such as {@code SIGKILL}, + * a native crash, an ANR, or the system reclaiming a backgrounded process, cannot be covered this way, + * and they take whatever was recorded since the last write. */ void flush(); + /** + * Sends all pending events to LaunchDarkly and waits for them to be delivered. + *

+ * Unlike {@link #flush()}, which returns before the events reach the network, this reports + * whether they arrived, which is what makes it usable at a point where the application is about + * to lose the ability to send them: an uncaught exception handler, a move to the background, or + * any other last chance. Events buffered in memory do not survive the process, so a caller that + * knows the process is ending can use this to give them one. + * {@link com.launchdarkly.sdk.android.integrations.LDCrashHandler} does this for + * uncaught exceptions. + *

+ * It can only help while the process is still running code. An uncaught exception runs its + * handler first, and a move to the background is announced, so both leave time for this call. + * A {@code SIGKILL}, an ANR kill, a native crash, and the system reclaiming a backgrounded process + * run nothing at all, and events still in memory at that moment are lost whatever the + * application does. + *

+ * The timeout bounds the whole call, including when the SDK is configured for more than one + * environment. Choose it with the caller in mind: a dying process is not a good place to wait on + * a network request that may never answer. The call blocks the thread it is made on, so on the + * main thread the timeout also counts towards an ANR. On a network that does not answer, a + * delivery takes about 21 seconds to give up with the default HTTP configuration: two attempts a + * second apart, each allowed the timeout set by + * {@link com.launchdarkly.sdk.android.integrations.HttpConfigurationBuilder#connectTimeoutMillis(int)}. + * A shorter timeout returns {@code false} before then, so there it bounds the wait rather than + * reporting how the delivery went. + *

+ * Where event persistence is on, this writes down everything recorded so far, as {@link #flush()} does, + * but on a thread of the SDK's rather than the calling one, so that the timeout bounds the write as + * well: a disk that stalls cannot hold a crashing thread past it. A {@code true} means the events were + * written before they were sent. A {@code false} because the timeout expired can come before the write + * is done, and the events then survive the process only if it lives long enough for the write to finish. + *

+ * This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. + * It is experimental. + * + * @param timeout how long to wait for delivery + * @param unit the time unit of {@code timeout} + * @return true if the events were delivered, or there were none to deliver; false if the timeout + * expired first, the SDK is offline, closed, or otherwise unable to deliver them, or events + * recorded since the last time this was answered were lost on the way, by this delivery or an + * earlier one. Events the service refuses with an error that may pass, such as a 503, are not + * lost: they are retried a second later and again by every flush after that, so a later call + * can still deliver them, and where persistence is on so can a later run of the application. + * A 401 or 403 stops delivery for good. A {@code false} because the timeout expired does not + * mean the events were not sent: the delivery is left running when the caller stops waiting, + * and may still arrive if the process lives long enough. A caller that resends on + * {@code false} can therefore cause duplicates. + * @since 5.17.0 + */ + boolean flushAndWait(long timeout, TimeUnit unit); + /** * Returns a map of all feature flags for the current evaluation context. No events are sent to LaunchDarkly. * diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDFutures.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDFutures.java index c062d72eb..a1ffb6eb7 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDFutures.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDFutures.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -83,6 +84,31 @@ public static LDAwaitFuture fromFuture(Future future) { return result; } + /** + * Runs a blocking call on a pooled daemon thread and reports its result as a future. + *

+ * Use this where a caller has a deadline but the work it is waiting for has no way to take one. + * The call is left running if the caller stops waiting; nothing interrupts it. + *

+ * This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. + * It is experimental. + * + * @param task the blocking call + * @param result type + * @return a future that completes with the call's result, or with whatever it threw + */ + public static Future fromBlockingCall(Callable task) { + LDAwaitFuture result = new LDAwaitFuture<>(); + getBridgeExecutor().execute(() -> { + try { + result.set(task.call()); + } catch (Throwable t) { + result.setException(t); + } + }); + return result; + } + /** * Returns a future that completes when the first of the given futures completes. * Equivalent to CompletableFuture.anyOf. Works with any {@link Future} (API-level safe). diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDUtil.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDUtil.java index 4a77849b2..58fb5912a 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDUtil.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDUtil.java @@ -174,6 +174,29 @@ static String urlSafeBase64Hash(LDContext context) { return urlSafeBase64Hash(JsonSerialization.serialize(context)); } + /** + * A hash for a file or directory name. Unlike {@link #urlSafeBase64Hash(String)} it does not go + * through the Android framework's Base64, so it also runs in plain JVM tests, and it has no padding + * character. + * + * @param bytes how many bytes of the digest to render, so a caller can trade length for headroom + * @return the first {@code bytes} bytes of the input's SHA-256, in lowercase hex + */ + static String hexDigest(String input, int bytes) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256") + .digest(input.getBytes(Charset.forName("UTF-8"))); + StringBuilder hex = new StringBuilder(bytes * 2); + for (int i = 0; i < bytes; i++) { + hex.append(Character.forDigit((hash[i] >> 4) & 0xf, 16)); + hex.append(Character.forDigit(hash[i] & 0xf, 16)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); // shouldn't be possible; SHA-256 is built in + } + } + static String urlSafeBase64(LDContext context) { return Base64.encodeToString(JsonSerialization.serialize(context).getBytes(), Base64.URL_SAFE + Base64.NO_WRAP); diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java new file mode 100644 index 000000000..30a5e3205 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java @@ -0,0 +1,344 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; +import com.launchdarkly.sdk.AttributeRef; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.AggregatedEventSummarizer; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.internal.events.EventOutputFormatter; +import com.launchdarkly.sdk.internal.events.EventSummarizer; +import com.launchdarkly.sdk.internal.events.EventSummarizerInterface; +import com.launchdarkly.sdk.internal.events.EventsConfiguration; +import com.launchdarkly.sdk.internal.events.SameContextPerContextEventSummarizer; + +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The counters and the encoder behind {@link DirectEventProcessor}: evaluations are folded into + * summary counters as they are recorded, and a flush turns a run of full-fidelity events, together + * with the counters accumulated beside them, into the payload they will be sent as. + *

+ * The full events are held by the processor rather than here. Capacity is counted once, against + * everything the SDK is holding, and the processor is the only place that can see all of it. + *

+ * The summarization and the wire format come from java-sdk-internal rather than being + * reimplemented here, so there is one definition of what an event looks like on the wire. + * {@code DefaultEventProcessor} is not reused along with them because it only accepts individual + * events and summarizes them itself, on the far side of the bounded queue this is meant to get in + * front of. + */ +final class OutboundEventBuffer { + private static final int INITIAL_OUTPUT_BUFFER_SIZE = 2000; + + private final EventOutputFormatter formatter; + private final EventSummarizerInterface summarizer; + private final LDLogger logger; + + /** + * How many distinct contexts may be counted between two drains, and which ones already are. + *

+ * The per-context summarizer keeps a counter set per context, keyed on the whole context with + * every attribute retained, and offers no way to ask how many it holds. Without this the only + * bound on it is how long the SDK goes without a drain -- and it is not drained at all while the + * client is offline, which is exactly when an application is free to go on evaluating. + */ + private final int maxContexts; + private final Set countedContexts; + /** + * The context counted last, compared by reference so that the set above is only hashed into when the + * context changes. {@link LDContext#hashCode()} is not cached, and an application evaluates against + * one instance until it identifies again. + */ + private LDContext lastCountedContext; + + /** + * Set when an event or a summary was dropped for failing to serialize, until the processor asks. + * Serialization runs on whichever thread commits, so this is the one piece of state here that is + * written outside the caller's lock. + */ + private final AtomicBoolean serializationFailed = new AtomicBoolean(false); + + /** + * @param allAttributesPrivate true to redact every context attribute except the key + * @param privateAttributes the individual context attributes to redact + * @param perContextSummarization true to emit one summary per context rather than one overall + * @param maxContexts how many distinct contexts may be counted between two drains + * @param logger where to report an event that cannot be serialized + */ + OutboundEventBuffer( + boolean allAttributesPrivate, + Collection privateAttributes, + boolean perContextSummarization, + int maxContexts, + LDLogger logger + ) { + // Only the private-attribute settings affect the output; the rest of EventsConfiguration + // describes the delivery behavior that the processor now handles itself, capacity included. + EventsConfiguration outputConfig = new EventsConfiguration(allAttributesPrivate, 0, + null, 0, null, null, 1, null, 0, false, false, privateAttributes, + perContextSummarization); + this.formatter = new EventOutputFormatter(outputConfig); + this.summarizer = perContextSummarization + ? new SameContextPerContextEventSummarizer() + : new AggregatedEventSummarizer(); + // One bucket overall, so there is no cardinality to bound and nothing to track it with. + this.maxContexts = perContextSummarization ? maxContexts : Integer.MAX_VALUE; + this.countedContexts = perContextSummarization ? new HashSet<>() : null; + this.logger = logger; + } + + /** + * Folds an evaluation into the summary counters, unless the evaluation asked to be left out of + * them. + *

+ * A counter is an aggregate rather than a buffered event, so no number of evaluations of a context + * already being counted can make this drop anything. What capacity does bound is how many distinct + * contexts are counted at once, because each one costs a retained context and its own counters. + *

+ * Not synchronized: the caller holds its own lock across this and the rest of the recording, as it + * does across {@link #takeSummaries}, and a second monitor inside it would only be taken for nothing. + * + * @param event the evaluation + * @return false if this event was not counted, because counting it would have meant holding + * a context beyond the configured capacity + */ + boolean summarize(Event.FeatureRequest event) { + // Checked here rather than in DirectEventProcessor, for the same reason the sampling ratio + // is: this is where an event arrives from outside. The processor summarizes from the values + // themselves, which have no such flag. java-sdk-internal's DefaultEventProcessor, which this + // path replaced, honored it, and a counter is the one thing no later stage can reconstruct. + if (event.isExcludeFromSummaries()) { + return true; + } + return summarize(event.getCreationDate(), event.getKey(), event.getVersion(), event.getVariation(), + event.getValue(), event.getDefaultVal(), event.getContext()); + } + + /** + * As {@link #summarize(Event.FeatureRequest)}, from the evaluation's values, so that an evaluation + * nobody needs a full event for costs no event to be built. + */ + boolean summarize(long timestamp, String flagKey, int flagVersion, int variation, LDValue value, + LDValue defaultValue, LDContext context) { + // Only a context that is not being counted yet can be turned away, so reaching the limit costs + // an application evaluating against one context nothing, however many evaluations it does. + if (countedContexts != null && context != lastCountedContext) { + if (!countedContexts.contains(context)) { + if (countedContexts.size() >= maxContexts) { + return false; + } + countedContexts.add(context); + } + lastCountedContext = context; + } + summarizer.summarizeEvent(timestamp, flagKey, flagVersion, variation, value, defaultValue, context); + return true; + } + + /** + * Serializes one event into the bytes it will be sent as, for a caller that keeps events + * somewhere other than this buffer's list. + *

+ * Serializing at record time rather than at flush is what lets an event be written somewhere that + * outlives the process: bytes can be appended to a log, an {@link Event} cannot. + *

+ * Deliberately not synchronized. It reads the formatter and writes only locals, so it shares + * nothing with a thread recording an event, and taking the lock would make an encode wait on a + * counter increment and the other way round. + * + * Sampling is not applied here. {@link DirectEventProcessor} draws for it once, as the event is + * recorded, and drawing again for the same event would square the odds against it. + * + * @param event the event + * @return the event's JSON object, or null if it could not be serialized + */ + byte[] serialize(Event event) { + return writeSingleObject(new Event[]{ event }, Collections.emptyList()); + } + + /** + * Takes the counters and forgets which contexts they were counted for. + *

+ * The two have to move together. Every path that resets the summarizer has to come through here, + * because one that reset the counters alone would spend the cardinality limit on contexts whose + * counters had already gone out, and the limit would never lift. + *

+ * Separate from {@link #serializeSummaries} so that the caller can take the counters in the same + * critical section it lifts the full events in. An evaluation writes a counter and a full event, + * and a commit that took the two at different moments could split one evaluation across two + * payloads. + * Not synchronized, for the reason {@link #summarize(Event.FeatureRequest)} is not. + */ + List takeSummaries() { + List summaries = summarizer.getSummariesAndReset(); + if (countedContexts != null) { + countedContexts.clear(); + lastCountedContext = null; + } + return summaries; + } + + /** + * Takes the counters and serializes them in one call, for a caller with nothing to coordinate the + * take with. + *

+ * Not for the commit path. That has to take the counters in the same critical section it takes the + * pending events in, or a commit can stage an evaluation's counter and leave its full event behind. + */ + List serializeSummariesAndReset() { + return serializeSummaries(takeSummaries()); + } + + /** + * Turns summary counters already taken from {@link #takeSummaries} into their summary events. + *

+ * Counters live only in memory, so whatever has not been summarized is what a crash takes with + * it. Summarizing at points where the events are about to be made durable is what bounds that + * loss, and it costs nothing in accuracy: LaunchDarkly sums the counters of every summary it + * receives, so several summaries covering the same evaluations count the same as one. + *

+ * A summary that cannot be serialized is dropped whole and logged rather than counted back in, and + * {@link #takeSerializationFailure} reports it, so the flush that committed it answers false. Losing + * an aggregate of evaluations the SDK promised to report is bad, but everything the encoder can + * fail on is a property of the data it was handed -- the output stream is a byte array and cannot + * fail transiently -- so putting the counters back would make every later flush fail on the same + * summary while the counters behind it grew without bound. + *

+ * Deliberately not synchronized, for the reason {@link #serialize} is not: the counters are taken + * under the caller's lock and encoded outside it, so a thread recording an evaluation waits only + * for a reference swap and never for the encoder. + * + * @param summaries the counters taken for this commit + * @return one JSON object per summary, empty if there was nothing counted or nothing serialized + */ + List serializeSummaries(List summaries) { + if (summaries.isEmpty()) { + return Collections.emptyList(); + } + List serialized = new ArrayList<>(summaries.size()); + for (EventSummarizer.EventSummary summary : summaries) { + byte[] bytes = writeSingleObject(new Event[0], Collections.singletonList(summary)); + if (bytes != null) { + serialized.add(bytes); + } + } + return serialized; + } + + /** + * Serializes a run of events with one encoder instead of one per event. + *

+ * Each event still becomes its own JSON object, because the store frames them individually and a + * frame holding several objects could not be spliced into a payload. What the run shares is the + * output stream, the writer and the UTF-8 lookup, which {@link #writeSingleObject} otherwise + * allocates on every call. + *

+ * Deliberately not synchronized, and not sampling, for the reasons {@link #serialize} is neither. + * + * @param pending the events, in the order they were recorded + * @return one JSON object per event that serialized + */ + List serializeAll(List pending) { + if (pending.isEmpty()) { + return Collections.emptyList(); + } + List serialized = new ArrayList<>(pending.size()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(INITIAL_OUTPUT_BUFFER_SIZE); + Writer writer = new BufferedWriter( + new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), INITIAL_OUTPUT_BUFFER_SIZE); + Event[] one = new Event[1]; + for (Event event : pending) { + one[0] = event; + byte[] bytes = writeSingleObject(one, Collections.emptyList(), + outputStream, writer); + if (bytes != null) { + serialized.add(bytes); + } + } + return serialized; + } + + /** + * Serializes exactly one output event and returns the JSON object on its own. + *

+ * {@link EventOutputFormatter} only offers to write a whole request body, which is a JSON array, + * and the single-event method beside it is private. So the array is written and its brackets are + * dropped, which is sound precisely because the count it returns says how many objects are in + * there: one object means the bytes between the brackets are that object. + * + * @return the object's bytes, or null if the formatter did not write exactly one event + */ + private byte[] writeSingleObject(Event[] events, List summaries) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(INITIAL_OUTPUT_BUFFER_SIZE); + Writer writer = new BufferedWriter( + new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), INITIAL_OUTPUT_BUFFER_SIZE); + return writeSingleObject(events, summaries, outputStream, writer); + } + + /** + * @param outputStream reset before the write, so it may be shared across a run + * @param writer must wrap {@code outputStream}, and holds nothing buffered on entry + */ + private byte[] writeSingleObject(Event[] events, List summaries, + ByteArrayOutputStream outputStream, Writer writer) { + outputStream.reset(); + int written; + try { + written = formatter.writeOutputEvents(events, summaries, writer); + writer.flush(); + } catch (Exception e) { + serializationFailed.set(true); + logger.error("Dropping unserializable {}: {}", describe(events), + LogValues.exceptionSummary(e)); + logger.debug("{}", LogValues.exceptionTrace(e)); + // Whatever the writer buffered is pushed out and thrown away with the stream, so a failed + // event cannot bleed into the next one sharing this writer. + try { + writer.flush(); + } catch (Exception ignored) { + // The stream is reset by the next call either way. + } + outputStream.reset(); + return null; + } + if (written != 1) { + return null; + } + byte[] all = outputStream.toByteArray(); + if (all.length < 3 || all[0] != '[' || all[all.length - 1] != ']') { + // Not the shape this depends on. Refusing the event is the safe reading: a frame that is + // not one JSON object would corrupt every payload it was later spliced into. + serializationFailed.set(true); + logger.error("Dropping unserializable {}: the encoder did not produce a single JSON" + + " object", describe(events)); + return null; + } + return Arrays.copyOfRange(all, 1, all.length - 1); + } + + /** + * @return whether anything was dropped for failing to serialize since this was last called + */ + boolean takeSerializationFailure() { + return serializationFailed.getAndSet(false); + } + + /** Names what is being dropped, since this encoder is handed one event or one summary. */ + private static String describe(Event[] events) { + return events.length == 0 ? "summary event" + : "event of type " + events[0].getClass().getSimpleName(); + }} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/PlatformState.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/PlatformState.java index 6c66b28f2..d990f5ba2 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/PlatformState.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/PlatformState.java @@ -55,4 +55,16 @@ interface ForegroundChangeListener { * @return a directory path */ File getCacheDir(); + + /** + * Returns the preferred filesystem location for files the SDK must not lose. + *

+ * This is not the cache directory: the system is free to delete cache files whenever it wants + * space, which is the opposite of what data written to survive a crash needs. It is also excluded + * from backups, so events recorded on one device are not restored onto another and reported as if + * they had happened there. + * + * @return a directory path + */ + File getNoBackupFilesDir(); } diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventPersistence.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventPersistence.java new file mode 100644 index 000000000..ad20ece13 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventPersistence.java @@ -0,0 +1,62 @@ +package com.launchdarkly.sdk.android.integrations; + +/** + * How far the SDK goes to make a recorded event outlive the process that recorded it. + *

+ * An event lives in memory until it is delivered, so a process that dies before the next flush takes + * everything recorded since the last one — including the crash an application was reporting when it died. + * Writing events to disk is what closes that gap: they are delivered on a later run instead of being lost. + *

+ * {@link #DEFERRED} and {@link #IMMEDIATE} are not stable in this major version, and not subject to any + * backwards compatibility guarantees or semantic versioning. They are experimental: their behavior, their + * cost on the calling thread, and what they leave on disk may change in a minor release. + * + * @see EventProcessorBuilder#eventPersistence(EventPersistence) + */ +public enum EventPersistence { + /** + * Events are kept in memory only, and nothing survives the process ending. + *

+ * The SDK does not touch the filesystem for events at all. Events that an earlier run with persistence + * turned on left on disk are not delivered, and stay there until persistence is turned back on. + */ + DISABLED, + + /** + * Events are written to disk, off the thread that recorded them. + *

+ * Recording stays as cheap as it is without persistence. An event from {@code track} or {@code identify} + * becomes durable a moment after it was recorded rather than immediately, and a process that dies inside + * that window still loses it. Evaluating a flag does not schedule a write of its own: its events wait in + * memory until enough of them have built up, or until the next {@code track}, {@code identify}, flush or + * delivery writes them. + *

+ * {@code LDClient.flush} is the exception. It writes on the calling thread, as {@link #IMMEDIATE} does, + * so that an application ending the process on purpose can rely on it. A flush on the main thread with + * {@link android.os.StrictMode} detecting disk writes therefore sees a violation. {@code flushAndWait} + * writes on a thread of the SDK's instead, so that its timeout bounds the write. + *

+ * This level is experimental; see {@link EventPersistence}. + */ + DEFERRED, + + /** + * Events are written to disk before {@code track} and {@code identify} return. + *

+ * The write happens on the calling thread, costing tens to hundreds of microseconds depending on the + * device and on how many evaluations are waiting to be encoded alongside it. Evaluations are usually + * written in the background before many build up. A burst of them faster than that can leave up to the + * event capacity waiting, and the {@code track} or {@code identify} that follows encodes them all, which + * can take milliseconds. In exchange there is no window: a process that dies the instant after + * {@code track} returns still reports that event on the next run. Evaluating a flag stays in memory + * regardless. + *

+ * Because this is disk I/O on the calling thread, an application that calls {@code track} or + * {@code identify} on the main thread with {@link android.os.StrictMode} detecting disk writes will see + * a violation for each call. Choose {@link #DEFERRED} for those callers, which keeps these writes off + * the calling thread. + *

+ * This level is experimental; see {@link EventPersistence}. + */ + IMMEDIATE +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java index 989222ae2..f2792b479 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java @@ -29,7 +29,7 @@ public abstract class EventProcessorBuilder implements ComponentConfigurer - * The default value is {@link #DEFAULT_CAPACITY}. + * The same number bounds how many distinct contexts flag evaluations are summarized for between two + * flushes. Evaluations for contexts beyond it are not counted until the next flush. + *

+ * The default value is {@link #DEFAULT_CAPACITY}. A capacity below one is treated as one; to + * stop sending events altogether, use {@link Components#noEvents()} instead. * * @param capacity the capacity of the event buffer * @return the builder */ public EventProcessorBuilder capacity(int capacity) { - this.capacity = capacity; + this.capacity = capacity < 1 ? 1 : capacity; return this; } @@ -134,6 +168,69 @@ public EventProcessorBuilder flushIntervalMillis(int flushIntervalMillis) { return this; } + /** + * Sets how far the SDK goes to make a recorded event outlive the process that recorded it. + *

+ * Anything other than {@link EventPersistence#DISABLED} appends events to a log under the + * application's no-backup files directory and delivers them on a later run, which is what lets the + * crash an application was reporting when it died reach LaunchDarkly at all. + * {@link EventPersistence#IMMEDIATE} additionally puts the write on the thread that called + * {@code track} or {@code identify}, so there is no window in which the event exists only in memory. + *

+ * Both levels other than {@link EventPersistence#DISABLED} are experimental, and not subject to any + * backwards compatibility guarantees or semantic versioning. + *

+ * The default value is {@link #DEFAULT_EVENT_PERSISTENCE}. + * + * @param eventPersistence how far to go to make events outlive the process + * @return the builder + */ + public EventProcessorBuilder eventPersistence(EventPersistence eventPersistence) { + this.eventPersistence = eventPersistence == null ? DEFAULT_EVENT_PERSISTENCE : eventPersistence; + return this; + } + + /** + * Sets the maximum number of bytes of events the SDK keeps for each environment where + * {@link #eventPersistence(EventPersistence)} keeps events across launches, on disk and in memory + * together. + *

+ * When it is reached, the SDK discards new events until it successfully delivers the stored ones, as + * it does when {@link #capacity(int)} is reached; it never deletes stored events to make room. A fifth + * of it is kept for the summaries of flag evaluations, so that evaluations are still counted after + * full events start being discarded. + *

+ * Where event persistence is {@link EventPersistence#DISABLED}, full events are bounded by + * {@link #capacity(int)} alone, and this bounds only the summaries, which it never refuses: a + * delivery that keeps failing would otherwise leave them growing for the life of the process. The + * default value is {@link #DEFAULT_CAPACITY_BYTES}; a value of zero or less means the default. + * + * @param capacityBytes the most bytes of events to keep for each environment + * @return the builder + */ + public EventProcessorBuilder capacityBytes(long capacityBytes) { + this.capacityBytes = capacityBytes <= 0 ? DEFAULT_CAPACITY_BYTES : capacityBytes; + return this; + } + + /** + * Sets how long the SDK keeps events it has persisted but not yet delivered, where + * {@link #eventPersistence(EventPersistence)} keeps events across launches. + *

+ * Events older than this are discarded rather than delivered, and so is everything the SDK stored for + * an environment whose mobile key the application stopped using. + *

+ * It has no effect where event persistence is {@link EventPersistence#DISABLED}. The default value is + * {@link #DEFAULT_MAX_AGE_MILLIS}; a value of zero or less means the default. + * + * @param maxAgeMillis how long to keep undelivered events, in milliseconds + * @return the builder + */ + public EventProcessorBuilder maxAgeMillis(long maxAgeMillis) { + this.maxAgeMillis = maxAgeMillis <= 0 ? DEFAULT_MAX_AGE_MILLIS : maxAgeMillis; + return this; + } + /** * Marks a set of attribute names or subproperties as private. *

diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java new file mode 100644 index 000000000..98e2bbb26 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java @@ -0,0 +1,126 @@ +package com.launchdarkly.sdk.android.integrations; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; + +import com.launchdarkly.sdk.android.LDClient; +import com.launchdarkly.sdk.android.LDClientInterface; +import com.launchdarkly.sdk.android.LaunchDarklyException; + +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +/** + * An uncaught exception handler that gives the SDK its last chance to act before the process ends: + * it delivers the analytics events still in memory. + *

+ * Events are kept in memory until they are sent, so a crash loses the ones recorded since the last + * flush, and those are often the ones that explain it. An uncaught exception runs its handlers while + * the process is still alive, and this one uses that time to call + * {@link LDClientInterface#flushAndWait(long, TimeUnit)} for every configured environment before + * passing the exception on to the handler that was installed before it. + *

+ * Install it before your crash reporter: + * + * 


+ *     LDClient.init(application, config, context, 0);
+ *     LDCrashHandler.install(2, TimeUnit.SECONDS);
+ *     SentryAndroid.init(application, options -> { ... });
+ *
+ *

+ * Crash reporters install their own handler in front of the one they find and call it once they + * have saved their report, so installing this one first lets the report be saved before the events + * are sent. Installed after the crash reporter, this handler runs first and holds the report back + * for as long as the delivery takes, and a process that dies in that time loses the report as well. + *

+ * The timeout is how long the crash is held open for the events. The thread that threw waits for + * it, so when that is the main thread the application stays frozen until the events are delivered + * or the timeout expires; a couple of seconds is a reasonable budget. That is less than a delivery + * takes to give up on a network that does not answer, so on such a network the crash is passed on + * when the timeout expires. A delivery still running then is not canceled, and may finish while the + * rest of the handlers run. + *

+ * This handler only helps with uncaught Java and Kotlin exceptions. A {@code SIGKILL}, an ANR kill, + * a native crash, and the system reclaiming a backgrounded process run no handlers, so the events + * still in memory at that moment are lost. + *

+ * This class is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. + * It is experimental. + * + * @since 5.17.0 + */ +public final class LDCrashHandler implements Thread.UncaughtExceptionHandler { + private static final Object installLock = new Object(); + + private final Callable client; + private final long timeoutMillis; + @Nullable + private final Thread.UncaughtExceptionHandler next; + + @VisibleForTesting + LDCrashHandler( + @NonNull Callable client, + long timeoutMillis, + @Nullable Thread.UncaughtExceptionHandler next + ) { + this.client = client; + this.timeoutMillis = timeoutMillis; + this.next = next; + } + + /** + * Makes this handler the default uncaught exception handler, in front of whichever handler was + * the default before. + *

+ * It can be called before {@link LDClient#init}: a crash that happens before the client exists + * has no events to deliver, and is passed on straight away. Calling it again while this handler + * is still the default does nothing, so the timeout given first is the one that applies. + * + * @param timeout how long a crash waits for the events to be delivered + * @param unit the time unit of {@code timeout} + */ + public static void install(long timeout, @NonNull TimeUnit unit) { + Objects.requireNonNull(unit, "unit"); + long timeoutMillis = Math.max(0, unit.toMillis(timeout)); + synchronized (installLock) { + Thread.UncaughtExceptionHandler previous = Thread.getDefaultUncaughtExceptionHandler(); + if (previous instanceof LDCrashHandler) { + return; + } + Thread.setDefaultUncaughtExceptionHandler( + new LDCrashHandler(LDClient::get, timeoutMillis, previous)); + } + } + + @Override + public void uncaughtException(@NonNull Thread thread, @NonNull Throwable throwable) { + try { + flush(); + } finally { + if (next != null) { + next.uncaughtException(thread, throwable); + } else { + throwable.printStackTrace(); + } + } + } + + private void flush() { + try { + LDClientInterface ldClient = client.call(); + if (ldClient == null) { + return; + } + // Waiting on the crashing thread is safe because the SDK enforces the timeout instead of + // trusting the delivery to finish, which also covers a crash that is the reason the + // delivery cannot complete. + ldClient.flushAndWait(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (LaunchDarklyException notInitialized) { + // A crash before LDClient.init has no events to deliver. + } catch (Throwable ignored) { + // Nothing that goes wrong here is worth losing the crash over. + } + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java index d1c98af02..eda21d296 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java @@ -4,8 +4,10 @@ import com.launchdarkly.sdk.EvaluationReason; import com.launchdarkly.sdk.LDContext; import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.LDFutures; import java.io.Closeable; +import java.util.concurrent.Future; /** * Interface for an object that can send or store analytics events. @@ -90,7 +92,9 @@ void recordCustomEvent( * Specifies that any buffered events should be sent as soon as possible, rather than waiting * for the next flush interval. This method is asynchronous, so events still may not be sent * until a later time. However, calling {@link Closeable#close()} will synchronously deliver - * any events that were not yet delivered prior to shutting down. + * any events that were not yet delivered prior to shutting down, unless the SDK is offline. In + * that case nothing is sent; the events are kept for a later run if event persistence is on, and + * discarded otherwise. */ void flush(); @@ -98,4 +102,33 @@ void recordCustomEvent( * Specifies that any buffered events should be sent immediately, blocking until done. */ void blockingFlush(); + + /** + * Specifies that any buffered events should be sent immediately, and reports through the + * returned future whether they were delivered. + *

+ * This is the form the SDK itself uses, so that a caller with a deadline can wait for as long as + * it has and no longer, and so that several of these can be waited on together. Only the public + * API puts a timeout in a signature; see {@code LDClient.flushAndWait}. + *

+ * Nothing cancels the delivery when a caller stops waiting for it: by then the events have been + * taken out of the buffer, so interrupting the post would only make losing them certain. + *

+ * This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. + * It is experimental. + * + * @return a future that completes with true if the events reached the service, or there were + * none to send; false if they could not be sent, including when a delivery that started + * earlier took them and then lost them + * @since 5.17.0 + */ + default Future flushAsync() { + // An implementation written before this method existed has only its unbounded blocking + // flush, so that runs on a thread of its own: the caller's deadline then bounds the wait + // rather than the flush, and the outcome it reports is still the flush's own. + return LDFutures.fromBlockingCall(() -> { + blockingFlush(); + return true; + }); + } } diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizer.java new file mode 100644 index 000000000..c5498e349 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizer.java @@ -0,0 +1,109 @@ +package com.launchdarkly.sdk.internal.events; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * {@link PerContextEventSummarizer} with a fast path for the context it saw last, and without the Java 8 + * library calls that Android lacks below API 24. + *

+ * The fast path is the reason this outlives the API-level fix. {@link LDContext#hashCode()} is not cached: + * every call flattens and hashes every attribute, so a map keyed on the context costs one full hash per + * evaluation, usually on the main thread. An application evaluates against the same {@code LDContext} + * instance until it identifies again, so comparing references first skips that hash on nearly every call. + *

+ * The API level: java-sdk-internal's summarizer calls {@code Map.computeIfAbsent} with a + * {@code java.util.function.Function}, neither of which exists before API 24. An application that has not + * turned on core library desugaring gets a {@code NoClassDefFoundError} at the first evaluation it + * summarizes, and lint cannot see it because the call is inside a library jar. + *

+ * A per-context summary can only be built through {@link EventSummarizer}'s package-private members, + * which is why this sits in java-sdk-internal's package rather than the SDK's. Not thread-safe; the + * caller serializes every call. + */ +public final class SameContextPerContextEventSummarizer implements EventSummarizerInterface { + private final Map summarizersByContext = new HashMap<>(); + + /** The context the last evaluation was counted for, compared by reference, and its summarizer. */ + private LDContext lastContext; + private EventSummarizer lastSummarizer; + + @Override + public void summarizeEvent( + long timestamp, + String flagKey, + int flagVersion, + int variation, + LDValue value, + LDValue defaultValue, + LDContext context + ) { + EventSummarizer summarizer; + if (context == lastContext && lastSummarizer != null) { + summarizer = lastSummarizer; + } else { + summarizer = summarizersByContext.get(context); + if (summarizer == null) { + summarizer = new EventSummarizer(context); + summarizersByContext.put(context, summarizer); + } + lastContext = context; + lastSummarizer = summarizer; + } + summarizer.summarizeEvent(timestamp, flagKey, flagVersion, variation, value, defaultValue, context); + } + + @Override + public List getSummariesAndReset() { + List summaries = new ArrayList<>(); + for (EventSummarizer summarizer : summarizersByContext.values()) { + EventSummarizer.EventSummary summary = summarizer.getSummaryAndReset(); + if (!summary.isEmpty()) { + summaries.add(summary); + } + } + summarizersByContext.clear(); + forgetLastContext(); + return summaries; + } + + @Override + public void restoreTo(List previousSummaries) { + summarizersByContext.clear(); + forgetLastContext(); + for (EventSummarizer.EventSummary summary : previousSummaries) { + if (summary.context != null && !summary.isEmpty()) { + EventSummarizer summarizer = new EventSummarizer(summary.context); + summarizer.restoreTo(summary); + summarizersByContext.put(summary.context, summarizer); + } + } + } + + @Override + public boolean isEmpty() { + for (EventSummarizer summarizer : summarizersByContext.values()) { + if (!summarizer.isEmpty()) { + return false; + } + } + return true; + } + + @Override + public void clear() { + summarizersByContext.clear(); + forgetLastContext(); + } + + /** Required wherever the map is cleared, or the fast path would count into a summarizer no longer in it. */ + private void forgetLastContext() { + lastContext = null; + lastSummarizer = null; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/BatchLockHolder.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/BatchLockHolder.java new file mode 100644 index 000000000..4e5f5320b --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/BatchLockHolder.java @@ -0,0 +1,22 @@ +package com.launchdarkly.sdk.android; + +import java.io.RandomAccessFile; +import java.nio.channels.FileLock; + +/** + * Run in a process of its own by tests: holds a shared lock on a batch the way a sender in another process does, + * until its standard input closes. POSIX locks belong to a process, so only another one can show what this + * process's expiry makes of them. + */ +public final class BatchLockHolder { + public static void main(String[] args) throws Exception { + try (RandomAccessFile file = new RandomAccessFile(args[0], "r"); + FileLock ignored = file.getChannel().lock(0L, Long.MAX_VALUE, true)) { + System.out.println("locked"); + System.out.flush(); + while (System.in.read() >= 0) { + // Held until the test closes the stream. + } + } + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/BatchLocksTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/BatchLocksTest.java new file mode 100644 index 000000000..79c2d9604 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/BatchLocksTest.java @@ -0,0 +1,36 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +public class BatchLocksTest { + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void expiryLeavesNoFileWhereABatchHasGone() { + File gone = new File(tempFolder.getRoot(), "ready-gone-1"); + + assertFalse(BatchLocks.deleteUnlessBeingSent(gone)); + assertFalse("an empty batch was left where the one delivered had been", gone.exists()); + } + + @Test + public void expiryDeletesABatchNobodyIsSending() throws IOException { + File batch = tempFolder.newFile("ready-old-1"); + try (FileOutputStream out = new FileOutputStream(batch)) { + out.write(new byte[] { 1, 2, 3 }); + } + + assertTrue(BatchLocks.deleteUnlessBeingSent(batch)); + assertFalse(batch.exists()); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DeliveryRetryStateTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DeliveryRetryStateTest.java new file mode 100644 index 000000000..1308bb5b2 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DeliveryRetryStateTest.java @@ -0,0 +1,103 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import java.util.Random; +import java.util.concurrent.TimeUnit; + +public class DeliveryRetryStateTest { + private static final long SECOND = 1_000; + private static final long FLUSH_INTERVAL = 30 * SECOND; + + private final DeliveryRetryState state = new DeliveryRetryState(FLUSH_INTERVAL); + + @Test + public void nothingWaitsBeforeTheFirstFailure() { + assertEquals(0, state.remainingWaitNanos()); + } + + @Test + public void ordinaryFailuresDoubleTheWaitUpToTheFlushInterval() { + long[] expected = { 1, 2, 4, 8, 16, 30, 30 }; + for (long seconds : expected) { + assertEquals(seconds * SECOND, state.recordFailure(false, -1)); + } + } + + @Test + public void theNormalCeilingIsNeverBelowASecond() { + DeliveryRetryState shortInterval = new DeliveryRetryState(200); + shortInterval.recordFailure(false, -1); + assertEquals(SECOND, shortInterval.recordFailure(false, -1)); + } + + @Test + public void aRefusalWaitsAboutEightAndAHalfMinutesAndDoublesToAnHour() { + assertEquals(512 * SECOND, state.recordFailure(true, -1)); + assertEquals(1_024 * SECOND, state.recordFailure(true, -1)); + assertEquals(2_048 * SECOND, state.recordFailure(true, -1)); + assertEquals(TimeUnit.HOURS.toMillis(1), state.recordFailure(true, -1)); + } + + @Test + public void anOrdinaryFailureAfterARefusalKeepsTheLongerCeiling() { + state.recordFailure(true, -1); + assertEquals(1_024 * SECOND, state.recordFailure(false, -1)); + } + + @Test + public void anAcceptedDeliveryPutsEverythingBack() { + state.recordFailure(true, -1); + state.recordFailure(false, -1); + state.recordSuccess(); + assertEquals(0, state.remainingWaitNanos()); + assertEquals(SECOND, state.recordFailure(false, -1)); + } + + @Test + public void aRetryAfterSetsTheWaitJitteredToNoLessThanHalfOfIt() { + for (int i = 0; i < 100; i++) { + long wait = state.recordFailure(false, 10 * SECOND); + assertTrue(wait + "ms", wait >= 5 * SECOND && wait <= 10 * SECOND); + } + } + + @Test + public void aRetryAfterIsCappedAtAnHour() { + long wait = new DeliveryRetryState(SECOND, FLUSH_INTERVAL, TimeUnit.HOURS.toMillis(1), new FixedRandom(0)) + .recordFailure(false, TimeUnit.HOURS.toMillis(5)); + assertEquals(TimeUnit.HOURS.toMillis(1), wait); + } + + @Test + public void aFailureWithARetryAfterStillCountsTowardTheBackoff() { + state.recordFailure(false, -1); + state.recordFailure(false, 0); + assertEquals(4 * SECOND, state.recordFailure(false, -1)); + } + + @Test + public void theWaitRunsOut() throws InterruptedException { + DeliveryRetryState quick = new DeliveryRetryState(20, 20, 20, new Random()); + quick.recordFailure(false, -1); + assertTrue(quick.remainingWaitNanos() > 0); + Thread.sleep(40); + assertEquals(0, quick.remainingWaitNanos()); + } + + private static final class FixedRandom extends Random { + private final double value; + + FixedRandom(double value) { + this.value = value; + } + + @Override + public double nextDouble() { + return value; + } + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DiagnosticConfigTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DiagnosticConfigTest.java index 8243052b0..2806c65b2 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DiagnosticConfigTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DiagnosticConfigTest.java @@ -73,7 +73,7 @@ public void customDiagnosticConfigurationEvents() throws Exception { .events( Components.sendEvents() .allAttributesPrivate(true) - .capacity(1000) + .capacity(3000) .diagnosticRecordingIntervalMillis(1_800_000) .flushIntervalMillis(60_000) ) @@ -83,11 +83,26 @@ public void customDiagnosticConfigurationEvents() throws Exception { ObjectBuilder expected = makeExpectedDefaults(); expected.put("allAttributesPrivate", true); expected.put("diagnosticRecordingIntervalMillis", 1_800_000); - expected.put("eventsCapacity", 1000); + expected.put("eventsCapacity", 3000); expected.put("eventsFlushIntervalMillis",60_000); Assert.assertEquals(expected.build(), diagnosticJson); } + @Test + public void diagnosticConfigurationReportsTheCapacityTheSdkWillUse() throws Exception { + // A capacity of nought or less runs at one, so that is what the service should be told. + for (int capacity : new int[] { 0, -5 }) { + LDConfig ldConfig = new LDConfig.Builder(AutoEnvAttributes.Disabled) + .events(Components.sendEvents().capacity(capacity)) + .build(); + + LDValue diagnosticJson = makeDiagnosticJson(ldConfig); + + Assert.assertEquals("capacity " + capacity, + 1, diagnosticJson.get("eventsCapacity").intValue()); + } + } + @Test public void customDiagnosticConfigurationStreaming() throws Exception { LDConfig ldConfig = new LDConfig.Builder(AutoEnvAttributes.Disabled) diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorTest.java new file mode 100644 index 000000000..22b273412 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorTest.java @@ -0,0 +1,3522 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.integrations.EventPersistence; +import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.sdk.android.subsystems.HttpConfiguration; +import com.launchdarkly.sdk.internal.events.DiagnosticStore; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.internal.events.EventSender; +import com.launchdarkly.testhelpers.httptest.HandlerSwitcher; +import com.launchdarkly.testhelpers.httptest.Handlers; +import com.launchdarkly.testhelpers.httptest.HttpServer; +import com.launchdarkly.testhelpers.httptest.RequestInfo; + +import org.junit.After; +import org.junit.Test; + +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Behavior of the SDK's own event processor, covering the parts that are not about buffering under + * load (see {@link EventProcessorBufferingTest} for those). + */ +public class DirectEventProcessorTest extends EventProcessorTestBase { + private static final String FLAG_KEY = "flag-key"; + private static final int FLAG_VERSION = 10; + private static final int VARIATION = 1; + private static final LDValue FLAG_VALUE = LDValue.of(true); + private static final LDValue DEFAULT_VALUE = LDValue.of(false); + + private static final int DEFAULT_CAPACITY = 100; + private static final String PAYLOAD_ID_HEADER = "X-LaunchDarkly-Payload-ID"; + + // Enough concurrent evaluations that close() reliably lands between the two writes one evaluation + // makes. Against the unfixed code this failed in the first trial of every run, by two to four + // events -- roughly the number of recorders that can sit in the gap at once. + private static final int RACE_TRIALS = 20; + private static final int RACE_RECORDERS = 4; + private static final int EVALUATIONS_PER_RACE_RECORDER = 2_000; + private static final int EVALUATIONS_BEFORE_CLOSE = 200; + private static final int RACE_CAPACITY = 30; + // Past anything the recorders can produce, so that a commit short of a feature event is short + // because the evaluation was split rather than because the buffer was full. + private static final int NO_DROP_CAPACITY = RACE_RECORDERS * EVALUATIONS_PER_RACE_RECORDER * 2; + private static final int SPLIT_TRIALS = 8; + + // Long enough that the only delivery in a test is the one it asks for. + private static final int NO_PERIODIC_FLUSH_MILLIS = 600_000; + // Short enough to keep the close tests quick; the production value is chosen for a real network. + private static final long CLOSE_BUDGET_MILLIS = 200; + + /** Created by makeEventProcessor, which the tests call instead of building a processor. */ + private final List diagnosticExecutors = new ArrayList<>(); + /** The commit executor the last processor made here commits on, for a test to wait behind. */ + private ExecutorService lastCommitExecutor; + + @After + public void shutDownDiagnosticExecutors() { + for (ExecutorService executor : diagnosticExecutors) { + executor.shutdownNow(); + } + } + + @Test + public void untrackedEvaluationProducesOnlyASummary() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + recordEvaluation(eventProcessor, false, null); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, events.size()); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void trackedEvaluationProducesAFeatureEventAndASummary() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + + List events = flushAndCollect(eventProcessor, server); + + LDValue featureEvent = requireEventOfKind(events, "feature"); + assertEquals(LDValue.of(FLAG_KEY), featureEvent.get("key")); + assertEquals(LDValue.of(VARIATION), featureEvent.get("variation")); + assertEquals(FLAG_VALUE, featureEvent.get("value")); + assertEquals(LDValue.of(FLAG_VERSION), featureEvent.get("version")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushPersistsFeatureAndSummaryEventsSynchronouslyWhileOffline() throws Exception { + assumePersisting(); + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + eventProcessor.setOffline(true); + + eventProcessor.flush(); + + List persisted = payloadsOnDisk(); + int featureEvents = 0; + int summaryEvents = 0; + for (byte[] payload : persisted) { + String kind = LDValue.parse(new String(payload, "UTF-8")).get("kind").stringValue(); + featureEvents += "feature".equals(kind) ? 1 : 0; + summaryEvents += "summary".equals(kind) ? 1 : 0; + } + assertEquals(1, featureEvents); + assertEquals(1, summaryEvents); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void immediatePersistencePutsATrackOnDiskBeforeItReturns() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).eventPersistence(EventPersistence.IMMEDIATE), true); + try { + // Offline so a delivery cannot drain the store out from under the assertion. The commit + // this is about runs either way; only the sending is held back. + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + // No waiting and no flush: the guarantee is that the call did the work before returning, which + // is the whole of what this setting buys and the only way a SIGKILL here still reports it. + assertEquals(1, kindsOnDisk("custom")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void deferredPersistenceStillPutsATrackOnDisk() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).eventPersistence(EventPersistence.DEFERRED), true); + try { + // Offline for the same reason as the immediate case: delivery would race the read. + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + // Durable a moment later rather than immediately: the commit was queued, not skipped. + long deadline = System.currentTimeMillis() + 5_000; + while (kindsOnDisk("custom") == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + assertEquals(1, kindsOnDisk("custom")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aDeferredTrackReachesTheDiskWhileADeliveryHoldsTheEventsThread() throws Exception { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, scheduler); + CountDownLatch deliveryAnswers = new CountDownLatch(1); + try { + // Stands in for a post to a network that does not answer, which holds the events thread for + // as long as its timeouts allow. + scheduler.submit(() -> awaitQuietly(deliveryAnswers, 10, TimeUnit.SECONDS)); + + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + long deadline = System.currentTimeMillis() + 5_000; + while (kindsOnDisk("custom") == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + assertEquals(1, kindsOnDisk("custom")); + } finally { + deliveryAnswers.countDown(); + eventProcessor.close(); + } + } + + @Test + public void aFlushThatWaitsWritesWhileADeliveryHoldsTheEventsThread() throws Exception { + // An evaluation is not a commit point, so a flush is what makes one durable. flushAsync hands + // the delivery to the events thread, which a post or a retry can hold for longer than the + // caller's timeout -- and flushAndWait is what an application calls when it is about to be + // killed, so the write cannot be left to that thread. It goes to the commit executor instead, + // which the network never holds. + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + UNUSED_EVENTS_URI, null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, true); + CountDownLatch deliveryAnswers = new CountDownLatch(1); + try { + eventProcessor.setOffline(false); + // Stands in for a post to a network that does not answer, which holds the events thread for + // as long as its timeouts allow. + scheduler.submit(() -> awaitQuietly(deliveryAnswers, 10, TimeUnit.SECONDS)); + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + + assertFalse("the delivery answered even though the events thread was held", + awaitFlush(eventProcessor, 0, TimeUnit.MILLISECONDS)); + + // While the events thread is still held, so the delivery's own commit cannot be the one. + assertEquals(1, kindsOnDiskSoon("feature")); + assertEquals(1, kindsOnDiskSoon("summary")); + } finally { + deliveryAnswers.countDown(); + eventProcessor.close(); + } + } + + @Test + public void aDeferredFlushThatWaitsReachesTheDiskWhileADeliveryHoldsTheEventsThread() throws Exception { + // As above, where the application asked for writes to be queued rather than made on its own + // thread. + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, scheduler); + CountDownLatch deliveryAnswers = new CountDownLatch(1); + try { + eventProcessor.setOffline(false); + scheduler.submit(() -> awaitQuietly(deliveryAnswers, 10, TimeUnit.SECONDS)); + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + + assertFalse("the delivery answered even though the events thread was held", + awaitFlush(eventProcessor, 0, TimeUnit.MILLISECONDS)); + + assertEquals(1, kindsOnDiskSoon("feature")); + assertEquals(1, kindsOnDiskSoon("summary")); + } finally { + deliveryAnswers.countDown(); + eventProcessor.close(); + } + } + + @Test + public void aFlushThatWaitsGetsItsFutureWhileAWriteIsStalledOnTheDisk() throws Exception { + // flushAndWait can only enforce its timeout once it holds the future. A write stalled on the disk + // holds the commit lock, and a flush that committed before handing back its future waited behind + // it with no bound at all, on the crashing thread when the crash handler was the caller. + CountDownLatch writeStalled = new CountDownLatch(1); + CountDownLatch diskAnswers = new CountDownLatch(1); + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, true, + logging.logger) { + @Override + void commit() { + writeStalled.countDown(); + awaitQuietly(diskAnswers, 10, TimeUnit.SECONDS); + super.commit(); + } + }; + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, + EventUtil.makeEventsTaskExecutor()); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + assertTrue("the track's commit never reached the disk", writeStalled.await(5, TimeUnit.SECONDS)); + + eventProcessor.recordCustomEvent(CONTEXT, "before-exit", LDValue.ofNull(), null); + long start = System.nanoTime(); + eventProcessor.flushAsync(); + long tookMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + + assertTrue("flushAsync waited " + tookMillis + "ms for the stalled write", tookMillis < 1_000); + } finally { + diskAnswers.countDown(); + eventProcessor.close(); + } + } + + @Test + public void aDeferredFlushWritesWithoutWaitingForTheQueueATrackedEventsWriteGoesTo() throws Exception { + // A flush is what an application calls before ending the process on purpose, so at DEFERRED it + // writes on its own thread rather than queueing the write the way track does. Holding the commit + // executor keeps a queued write off the disk, so only a write on this thread can pass. + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, + EventUtil.makeEventsTaskExecutor()); + CountDownLatch commitsResume = new CountDownLatch(1); + try { + lastCommitExecutor.submit(() -> awaitQuietly(commitsResume, 10, TimeUnit.SECONDS)); + eventProcessor.recordCustomEvent(CONTEXT, "before-exit", null, null); + assertEquals(0, kindsOnDisk("custom")); + + eventProcessor.flush(); + + assertEquals(1, kindsOnDisk("custom")); + } finally { + commitsResume.countDown(); + eventProcessor.close(); + } + } + + @Test + public void aFailedDeliveryIsRetriedOnceUnderTheSamePayloadId() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.sequential(Handlers.status(503), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + // A flush that waits, waits for the retry too, so it reports the delivery that worked. + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + RequestInfo failed = server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + RequestInfo retried = server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + assertEquals(failed.getHeader("X-LaunchDarkly-Payload-ID"), + retried.getHeader("X-LaunchDarkly-Payload-ID")); + assertEquals(failed.getBody(), retried.getBody()); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void waitingToRetryDoesNotHoldTheEventsThread() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.flush(); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + + // Well inside the retry delay: a thread sleeping through it would not get to this. + scheduler.submit(() -> { }).get(DirectEventProcessor.RETRY_DELAY_MILLIS / 2, + TimeUnit.MILLISECONDS); + + // One retry and no more; the batch then waits for the next flush. + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(DirectEventProcessor.RETRY_DELAY_MILLIS * 2, + TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aStoredBatchThisVersionCannotReadIsDroppedAndReportedWhileTheRestAreDelivered() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.status(202))) { + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, + DEFAULT_CAPACITY, true, logging.logger); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY); + try { + // Online, and past the startup listing, before the batch exists, so the flush below is the + // delivery that finds it rather than one that recovery or coming online starts, which + // nobody waits on. + eventProcessor.setOffline(false); + scheduler.submit(() -> { }).get(2, TimeUnit.SECONDS); + store.stage("{\"kind\":\"custom\",\"key\":\"damaged\"}".getBytes(StandardCharsets.UTF_8)); + EventStore.Batch damaged = closeOne(store); + assertNotNull(damaged); + // Its name still counts it, so it is listed without being read; the damage is only found on + // sending. + File damagedFile = findFile(eventsDirectory.getRoot(), "ready-" + damaged.payloadId + ".1"); + assertNotNull(damagedFile); + java.nio.file.Files.write(damagedFile.toPath(), + "not an event log".getBytes(StandardCharsets.UTF_8)); + eventProcessor.recordCustomEvent(CONTEXT, "intact", LDValue.ofNull(), null); + + assertFalse("a flush that lost a batch reported success", + awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + RequestInfo sent = server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + assertTrue(sent.getBody().contains("intact")); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + assertFalse(damagedFile.exists()); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aStoredBatchDamagedAfterItsFirstFrameIsSentShortAndReportedAsLosingTheRest() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.status(202))) { + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, + DEFAULT_CAPACITY, true, logging.logger); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY); + try { + // As above, past the startup listing, so the flush is the delivery that finds the batch. + eventProcessor.setOffline(false); + scheduler.submit(() -> { }).get(2, TimeUnit.SECONDS); + for (String key : new String[] { "kept-0", "kept-1", "torn" }) { + store.stage(("{\"kind\":\"custom\",\"key\":\"" + key + "\"}").getBytes(StandardCharsets.UTF_8)); + } + EventStore.Batch damaged = closeOne(store); + File damagedFile = findFile(eventsDirectory.getRoot(), "ready-" + damaged.payloadId + ".3"); + assertNotNull(damagedFile); + byte[] log = java.nio.file.Files.readAllBytes(damagedFile.toPath()); + log[log.length - 1] ^= 0x01; + java.nio.file.Files.write(damagedFile.toPath(), log); + + assertFalse("a flush that sent a batch short reported success", + awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + RequestInfo sent = server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + assertTrue(sent.getBody().contains("kept-0")); + assertTrue(sent.getBody().contains("kept-1")); + assertFalse(sent.getBody().contains("torn")); + assertFalse(damagedFile.exists()); + + // Reported once: the next flush has nothing of it to answer for. + eventProcessor.recordCustomEvent(CONTEXT, "after", LDValue.ofNull(), null); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + } finally { + eventProcessor.close(); + } + } + } + + /** Closes what a test expects to be exactly one batch, as anything short of a failed write leaves. */ + private static EventStore.Batch closeOne(EventStore store) { + List closed = store.closeBatch(); + assertEquals(1, closed.size()); + return closed.get(0); + } + + private static File findFile(File directory, String name) { + File[] files = directory.listFiles(); + if (files == null) { + return null; + } + for (File file : files) { + if (file.getName().equals(name)) { + return file; + } + File found = file.isDirectory() ? findFile(file, name) : null; + if (found != null) { + return found; + } + } + return null; + } + + /** @return how many events of this kind the store holds, read as another process would read it */ + /** + * Every event in the environment's directory, read straight off the disk the way a process starting after a + * kill finds them: every log, whoever has it open, and every batch. + */ + private List payloadsOnDisk() throws IOException { + List payloads = new ArrayList<>(); + File[] files = EventDirectory.defaultDirectory(eventsDirectory.getRoot(), MOBILE_KEY).listFiles(); + if (files == null) { + return payloads; + } + for (File file : files) { + String name = file.getName(); + if (name.startsWith(DiskEventLog.PREFIX) || name.startsWith("ready-")) { + try { + payloads.addAll(EventLogFormat.eventPayloads(Files.readAllBytes(file.toPath()))); + } catch (NoSuchFileException e) { + // Closed off or delivered since the listing. + } + } + } + return payloads; + } + + private int kindsOnDisk(String kind) throws Exception { + int found = 0; + for (byte[] payload : payloadsOnDisk()) { + if (kind.equals(LDValue.parse(new String(payload, "UTF-8")).get("kind").stringValue())) { + found++; + } + } + return found; + } + + /** {@link #kindsOnDisk}, once at least one is there or five seconds have passed. */ + private int kindsOnDiskSoon(String kind) throws Exception { + long deadline = System.currentTimeMillis() + 5_000; + while (kindsOnDisk(kind) == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + return kindsOnDisk(kind); + } + + @Test + public void persistenceIsOffUnlessTheApplicationAsksForIt() throws Exception { + try (HttpServer server = startEventsServer()) { + // Deliberately not eventsBuilder(), which turns persistence on: this is what an application + // gets without configuring anything. + EventProcessor eventProcessor = makeEventProcessor(server, + Components.sendEvents() + .capacity(DEFAULT_CAPACITY) + .flushIntervalMillis(NO_PERIODIC_FLUSH_MILLIS), + true); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + assertTrue("an event reached the disk without being asked to", payloadsOnDisk().isEmpty()); + + // Held in memory instead, so turning persistence off costs durability and nothing else. + List events = flushAndCollect(eventProcessor, server); + requireEventOfKind(events, "identify"); + requireEventOfKind(events, "custom"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void identifyAndCustomEventsAreDelivered() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(2, events.size()); + requireEventOfKind(events, "identify"); + LDValue customEvent = requireEventOfKind(events, "custom"); + assertEquals(LDValue.of("an-event"), customEvent.get("key")); + assertEquals(LDValue.of("data"), customEvent.get("data")); + assertEquals(LDValue.of(2.5), customEvent.get("metricValue")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unserializableMetricDoesNotPoisonLaterDeliveries() throws Exception { + try (HttpServer server = startEventsServer()) { + // One slot makes this cover both recovery and release of the poisoned event's capacity. + EventProcessor eventProcessor = makeEventProcessor(server, 1); + try { + // This is the event produced by LDClient.trackMetric(..., Double.NaN). Gson's + // strict writer rejects the non-finite metric. + eventProcessor.recordCustomEvent( + CONTEXT, "poison", LDValue.ofNull(), Double.NaN); + eventProcessor.blockingFlush(); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + + eventProcessor.recordCustomEvent( + CONTEXT, "after-poison", LDValue.ofNull(), 1.0); + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, events.size()); + assertEquals(LDValue.of("after-poison"), requireEventOfKind(events, "custom").get("key")); + logging.assertErrorLogged("Dropping unserializable"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unserializableMetricDoesNotDropSiblingEvents() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent( + CONTEXT, "poison", LDValue.ofNull(), Double.NaN); + eventProcessor.recordCustomEvent( + CONTEXT, "kept", LDValue.ofNull(), 1.0); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, events.size()); + assertEquals(LDValue.of("kept"), requireEventOfKind(events, "custom").get("key")); + logging.assertErrorLogged("Dropping unserializable"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unserializableSummaryDoesNotPoisonLaterDeliveries() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + // Java callers can violate boolVariation's @NonNull contract. A null flag key can + // enter the summarizer, but Gson cannot use it as a JSON object member name. + eventProcessor.recordEvaluationEvent(CONTEXT, null, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + eventProcessor.blockingFlush(); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + + recordEvaluation(eventProcessor, false, null); + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, events.size()); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + logging.assertErrorLogged("Dropping unserializable"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unserializableSummaryDoesNotDropSiblingEvents() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordEvaluationEvent(CONTEXT, null, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + eventProcessor.recordIdentifyEvent(CONTEXT); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, countEventsOfKind(events, "identify")); + assertEquals(0, countEventsOfKind(events, "summary")); + logging.assertErrorLogged("Dropping unserializable"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void capacityLimitsFullEventsButNotSummaries() throws Exception { + try (HttpServer server = startEventsServer()) { + int capacity = 3; + EventProcessor eventProcessor = makeEventProcessor(server, capacity); + try { + for (int i = 0; i < capacity + 2; i++) { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + } + // Evaluations well past capacity still have to be counted in full, because a + // counter does not occupy a buffer slot. + int evaluations = capacity * 100; + for (int i = 0; i < evaluations; i++) { + recordEvaluation(eventProcessor, false, null); + } + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(capacity, countEventsOfKind(events, "custom")); + assertEquals(evaluations, summaryCountFor(events, FLAG_KEY)); + logging.assertWarnLogged("Exceeded event queue capacity"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aNonPositiveCapacityStillReportsRatherThanGoingSilent() throws Exception { + // Nought and negatives both mean one, and the processor and the buffer have to agree on + // that: disagreeing leaves the SDK holding an event it will never summarize, or summarizing + // for a buffer that will never send. Disabling events entirely is what noEvents() is for. + for (int capacity : new int[] { 0, -5 }) { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, capacity); + try { + recordEvaluation(eventProcessor, false, null); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals("capacity " + capacity + " summarized nothing", + 1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + } + + @Test + public void closeDeliversBufferedEventsWithoutAnExplicitFlush() throws Exception { + // This is the case the SDK previously lost: a short session that records something and + // then shuts down before the periodic flush comes around. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + recordEvaluation(eventProcessor, false, null); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + eventProcessor.close(); + + List events = collectDelivered(server); + assertEquals(1, countEventsOfKind(events, "custom")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } + } + + @Test + public void closeNeverDeliversASummaryWithoutTheFeatureEventItCounted() throws Exception { + // An evaluation of a tracked flag writes a summary counter and a feature event. close() has + // to take both or neither. If it can take the counter and then refuse the event, it reports + // an evaluation that no event describes -- for experimentation traffic, a data point that + // disappears while the summary insists it happened. + // + // The window is the gap between those two writes, so this races them rather than asserting + // on a single ordering. Nothing here is timing-tolerant: the invariant holds for every + // interleaving, so any trial that breaks it is a real defect. + int featureEventsSeen = 0; + for (int trial = 0; trial < RACE_TRIALS; trial++) { + try (HttpServer server = startEventsServer()) { + // Deliberately small. The recorders outrun the buffer, so capacity is the only thing + // bounding the payload, and the test server records the body with a single unlooped + // read -- so anything past one socket read's worth comes back truncated, at a point + // that moves from run to run. A few kilobytes stays well clear of that. + DirectEventProcessor eventProcessor = + (DirectEventProcessor) makeEventProcessor(server, RACE_CAPACITY); + // Guarantees the final delivery has something in it, so collectDelivered always has + // a request to read even if every recorder loses the race. + eventProcessor.recordIdentifyEvent(CONTEXT); + + AtomicInteger recorded = new AtomicInteger(); + List recorders = new ArrayList<>(); + for (int i = 0; i < RACE_RECORDERS; i++) { + Thread recorder = new Thread(() -> { + for (int n = 0; n < EVALUATIONS_PER_RACE_RECORDER; n++) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, true, null); + recorded.incrementAndGet(); + } + }); + recorders.add(recorder); + recorder.start(); + } + // Close in the middle of the burst rather than at the edge of it. Closing before the + // recorders are going would resolve the race trivially every time. + while (recorded.get() < EVALUATIONS_BEFORE_CLOSE) { + Thread.yield(); + } + eventProcessor.close(); + for (Thread recorder : recorders) { + recorder.join(); + } + + long dropped = eventProcessor.getAndClearDroppedCount(); + List events = collectDelivered(server); + int featureEvents = countEventsOfKind(events, "feature"); + // Every evaluation is accounted for one of three ways: delivered in full, dropped + // for capacity, or refused outright before anything was written. Only the first two + // may leave a counter behind, so a counter that matches neither is one that was + // taken from a half-recorded evaluation. + assertEquals("trial " + trial + ": a summary counted an evaluation whose feature" + + " event was neither delivered nor dropped", + featureEvents + dropped, summaryCounters(events, FLAG_KEY)); + featureEventsSeen += featureEvents; + } + } + // Guards against the whole thing passing because nothing ever got as far as being delivered. + assertTrue("no evaluation survived to be delivered, so nothing was actually compared", + featureEventsSeen > 0); + } + + /** + * Like {@code summaryCountFor}, but returns zero rather than failing when no summary was + * delivered. Here a trial in which close() beat every recorder is a legitimate outcome. + */ + private int summaryCounters(List events, String flagKey) { + int total = 0; + for (LDValue event : events) { + if (!"summary".equals(event.get("kind").stringValue())) { + continue; + } + for (LDValue counter : event.get("features").get(flagKey).get("counters").values()) { + total += counter.get("count").intValue(); + } + } + return total; + } + + @Test + public void eventsRecordedWhileOfflineAreRetainedAndSentOnceOnline() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + recordEvaluation(eventProcessor, false, null); + + eventProcessor.blockingFlush(); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + + eventProcessor.setOffline(false); + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, countEventsOfKind(events, "custom")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void closingWhileOfflineStaysOffTheNetwork() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + recordEvaluation(eventProcessor, false, null); + + eventProcessor.close(); + + server.getRecorder().requireNoRequests(500, TimeUnit.MILLISECONDS); + } + } + + @Test + public void goingOnlineForTheFirstTimeDoesNotDeliverOnItsOwn() throws Exception { + // What LDClient does at startup: the initial identify is recorded while the processor is + // still offline, and initialization then turns it on. The identify waits to be batched with + // what follows rather than going out alone. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY), true); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + + eventProcessor.setOffline(false); + + server.getRecorder().requireNoRequests(500, TimeUnit.MILLISECONDS); + List events = flushAndCollect(eventProcessor, server); + assertEquals(1, countEventsOfKind(events, "identify")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void closeCancelsThePeriodicFlushAndNothingAfterItSchedulesAnother() throws Exception { + List> scheduled = Collections.synchronizedList(new ArrayList<>()); + ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1) { + @Override + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, + long delay, TimeUnit unit) { + ScheduledFuture future = super.scheduleWithFixedDelay(command, initialDelay, delay, unit); + scheduled.add(future); + return future; + } + }; + // Otherwise shutting the executor down would cancel the task for close(), and this could not + // tell whether close() did. + scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(true); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + eventProcessor.setOffline(false); + eventProcessor.close(); + eventProcessor.setOffline(true); + eventProcessor.setOffline(false); + eventProcessor.setInBackground(true); + eventProcessor.setInBackground(false); + + assertEquals(1, scheduled.size()); + assertTrue("close() left the periodic flush running", scheduled.get(0).isCancelled()); + } finally { + scheduler.shutdownNow(); + } + } + + @Test + public void anOutageDoesNotRestartTheFlushInterval() throws Exception { + // Restarting it on every reconnect would let a run of brief outages hold events back for + // far longer than one interval. + AtomicInteger scheduled = new AtomicInteger(); + ScheduledExecutorService scheduler = new ScheduledThreadPoolExecutor(1) { + @Override + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, + long delay, TimeUnit unit) { + scheduled.incrementAndGet(); + return super.scheduleWithFixedDelay(command, initialDelay, delay, unit); + } + }; + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + eventProcessor.setOffline(false); + for (int i = 0; i < 5; i++) { + eventProcessor.setOffline(true); + eventProcessor.setOffline(false); + } + + assertEquals(1, scheduled.get()); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void eventsOnTheirWayIntoTheStoreStillCountAgainstCapacity() throws Exception { + // A commit takes the pending events and encodes them before staging, and for that long they + // are in neither place. Counting only the two would let an event recorded then past capacity. + CountDownLatch staging = new CountDownLatch(1); + CountDownLatch releaseStaging = new CountDownLatch(1); + // Persisting, since without it a commit point queues no commit at all. + EventStore store = new EventStore(eventsDirectory.newFolder(), 1, true, + logging.logger) { + @Override + boolean stageReserved(byte[] serializedEvent) { + staging.countDown(); + awaitQuietly(releaseStaging, 5, TimeUnit.SECONDS); + return super.stageReserved(serializedEvent); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, 1, scheduler); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + assertTrue("the commit never started staging", staging.await(2, TimeUnit.SECONDS)); + + eventProcessor.recordIdentifyEvent(CONTEXT); + releaseStaging.countDown(); + + assertEquals(1, eventProcessor.getAndClearDroppedCount()); + } finally { + releaseStaging.countDown(); + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void anEventAlreadyStagedIsNotCountedTwiceAgainstCapacity() throws Exception { + // Paused just after the first event reached the store, with the commit that staged it still + // running. Counted once, it leaves room for a second event under a capacity of two; counted both + // as staged and as on its way, it would not. + CountDownLatch staged = new CountDownLatch(1); + CountDownLatch releaseCommit = new CountDownLatch(1); + // Persisting, for the reason the test above is. + EventStore store = new EventStore(eventsDirectory.newFolder(), 2, true, + logging.logger) { + @Override + boolean stageReserved(byte[] serializedEvent) { + boolean result = super.stageReserved(serializedEvent); + staged.countDown(); + awaitQuietly(releaseCommit, 5, TimeUnit.SECONDS); + return result; + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, 2, scheduler); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + assertTrue("the commit never staged the event", staged.await(2, TimeUnit.SECONDS)); + + eventProcessor.recordIdentifyEvent(CONTEXT); + releaseCommit.countDown(); + + assertEquals(0, eventProcessor.getAndClearDroppedCount()); + } finally { + releaseCommit.countDown(); + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void storedEventsPastTheMaximumAgeAreDiscardedAsDroppedWithoutFailingAFlush() throws Exception { + assumePersisting(); + EventStore previousRun = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, + DEFAULT_CAPACITY, true, logging.logger); + previousRun.stage(LDValue.buildObject().put("kind", "custom").put("key", "old-1").build() + .toJsonString().getBytes(StandardCharsets.UTF_8)); + previousRun.stage(LDValue.buildObject().put("kind", "custom").put("key", "old-2").build() + .toJsonString().getBytes(StandardCharsets.UTF_8)); + previousRun.closeBatch(); + previousRun.close(); + File[] batches = previousRun.getDirectory().listFiles((dir, name) -> name.startsWith("ready-")); + assertEquals(1, batches.length); + assertTrue(batches[0].setLastModified(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(8))); + + try (HttpServer server = startEventsServer()) { + DirectEventProcessor eventProcessor = (DirectEventProcessor) makeEventProcessor(server, + DEFAULT_CAPACITY); + try { + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + assertEquals(0, server.getRecorder().count()); + assertEquals(2, eventProcessor.getAndClearDroppedCount()); + assertFalse(batches[0].exists()); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void anEventWithNoBytesLeftForItIsDroppedBeforeItIsEncoded() throws Exception { + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, 1000, + EventProcessorBuilder.DEFAULT_MAX_AGE_MILLIS, true, logging.logger); + byte[] filler = new byte[800 - EventLogFormat.FRAME_HEADER_SIZE]; + Arrays.fill(filler, (byte) 'x'); + assertTrue(store.stage(filler)); + assertFalse(store.hasRoomForEventBytes()); + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, scheduler); + try { + eventProcessor.recordCustomEvent(CONTEXT, "refused", LDValue.ofNull(), null); + eventProcessor.flush(); + + assertEquals(1, eventProcessor.getAndClearDroppedCount()); + assertEquals(1, store.getPendingEventCount()); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void aFlushFailsWhenAnAcceptedEventIsTooLargeForTheBytesLeft() throws Exception { + assumePersisting(); + StringBuilder data = new StringBuilder(); + for (int i = 0; i < 500; i++) { + data.append('x'); + } + try (HttpServer server = startEventsServer()) { + DirectEventProcessor eventProcessor = (DirectEventProcessor) makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).capacityBytes(400), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "too-large", LDValue.of(data.toString()), null); + + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + logging.assertErrorLogged("no room"); + assertEquals(0, eventProcessor.getAndClearDroppedCount()); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void theStoreAsksThePlatformWhereItLivesOffTheCallersThread() throws Exception { + // LDClient.init builds the processor, usually on the main thread, and answering touches the disk: + // the platform creates the no-backup directory when asked for it. + Queue askedOn = new ConcurrentLinkedQueue<>(); + MockPlatformState platformState = new MockPlatformState() { + @Override + public File getNoBackupFilesDir() { + askedOn.add(Thread.currentThread()); + return eventsDirectory.getRoot(); + } + }; + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, + Components.sendEvents().eventPersistence(EventPersistence.DEFERRED) + .flushIntervalMillis(NO_PERIODIC_FLUSH_MILLIS), + true, platformState); + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.flush(); + eventProcessor.blockingFlush(); + eventProcessor.close(); + + assertFalse("the platform was never asked, so this proves nothing", askedOn.isEmpty()); + assertFalse("the platform was asked on the caller's thread", + askedOn.contains(Thread.currentThread())); + } + } + + @Test + public void withoutPersistenceTheStoreNeverAsksThePlatformWhereItLives() throws Exception { + // Asking is itself a filesystem operation, so a store that never asks never touches the disk: + // not for a previous run's events at startup, and not on every delivery after. + AtomicInteger asked = new AtomicInteger(); + MockPlatformState platformState = new MockPlatformState() { + @Override + public File getNoBackupFilesDir() { + asked.incrementAndGet(); + return eventsDirectory.getRoot(); + } + }; + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, + Components.sendEvents().eventPersistence(EventPersistence.DISABLED) + .flushIntervalMillis(NO_PERIODIC_FLUSH_MILLIS), + true, platformState); + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.flush(); + eventProcessor.blockingFlush(); + eventProcessor.close(); + + assertEquals("the event was never delivered, so this proves nothing", + 1, countEventsOfKind(collectDelivered(server), "custom")); + assertEquals("a store without persistence looked for its directory", 0, asked.get()); + } + } + + @Test + public void concurrentImmediateTracksShareCommitsAndEachIsDurableOnReturn() throws Exception { + final int trackers = 8; + Queue staged = new ConcurrentLinkedQueue<>(); + Queue durable = new ConcurrentLinkedQueue<>(); + AtomicInteger commits = new AtomicInteger(); + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, true, + logging.logger) { + @Override + boolean stageReserved(byte[] serializedEvent) { + staged.add(new String(serializedEvent, StandardCharsets.UTF_8)); + return super.stageReserved(serializedEvent); + } + + @Override + void commit() { + commits.incrementAndGet(); + // A slow disk, so that the other tracks arrive while this write is under way. + sleepQuietly(50); + // Everything staged so far was staged under the commit lock this commit holds. + List writing = new ArrayList<>(staged); + super.commit(); + durable.addAll(writing); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), UNUSED_EVENTS_URI, + null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, scheduler, diagnosticExecutor, + store, DEFAULT_CAPACITY, true); + try { + CountDownLatch start = new CountDownLatch(1); + AtomicInteger notDurableOnReturn = new AtomicInteger(); + List threads = new ArrayList<>(); + for (int i = 0; i < trackers; i++) { + String key = "event-" + i; + Thread thread = new Thread(() -> { + awaitQuietly(start, 5, TimeUnit.SECONDS); + eventProcessor.recordCustomEvent(CONTEXT, key, LDValue.ofNull(), null); + boolean found = false; + for (String event : durable) { + found |= event.contains("\"key\":\"" + key + "\""); + } + if (!found) { + notDurableOnReturn.incrementAndGet(); + } + }); + thread.start(); + threads.add(thread); + } + start.countDown(); + for (Thread thread : threads) { + thread.join(10_000); + } + + assertEquals("a track returned before its event was on disk", 0, notDurableOnReturn.get()); + assertTrue("each track committed on its own: " + commits.get() + " commits for " + + trackers + " tracks", commits.get() < trackers); + } finally { + eventProcessor.close(); + } + } + + private static void sleepQuietly(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Test + public void deferredPersistenceWritesOnTheCallersThreadOnlyForAFlush() throws Exception { + Queue committedOn = new ConcurrentLinkedQueue<>(); + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, true, + logging.logger) { + @Override + void commit() { + committedOn.add(Thread.currentThread()); + super.commit(); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, scheduler); + CountDownLatch commitsResume = new CountDownLatch(1); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.recordCustomEvent(CONTEXT, "another", LDValue.ofNull(), null); + lastCommitExecutor.submit(() -> { }).get(2, TimeUnit.SECONDS); + + assertFalse("nothing was ever committed, so this proves nothing", committedOn.isEmpty()); + assertFalse("a track committed on the caller's thread", + committedOn.contains(Thread.currentThread())); + + // Held, because the track below queues a commit of its own. Were it to run first it would cover + // everything the flush needs, leaving the flush nothing to write on this thread. + lastCommitExecutor.submit(() -> awaitQuietly(commitsResume, 10, TimeUnit.SECONDS)); + eventProcessor.recordCustomEvent(CONTEXT, "before-exit", LDValue.ofNull(), null); + eventProcessor.flush(); + + assertTrue("a flush left its commit to another thread", + committedOn.contains(Thread.currentThread())); + } finally { + commitsResume.countDown(); + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void aFullPendingRunIsCommittedWherePersistenceIsOn() throws Exception { + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, true, + logging.logger); + + assertTrue("the run was never committed", pendingEventsInStoreAfterAFullRun(store) > 0); + } + + @Test + public void aFullPendingRunWaitsForTheFlushWherePersistenceIsOff() throws Exception { + // Committing it would make nothing durable, and would put the encode in among the evaluations. + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, false, + logging.logger); + + assertEquals(0, pendingEventsInStoreAfterAFullRun(store)); + } + + @Test + public void anOfflineFlushEncodesNothingWherePersistenceIsOff() throws Exception { + AtomicInteger encoded = new AtomicInteger(); + EventStore store = encodingCountingStore(encoded); + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(server, store, scheduler); + try { + eventProcessor.setOffline(true); + // Past the pending threshold, which schedules a commit only where one would write. + for (int i = 0; i < 40; i++) { + recordEvaluation(eventProcessor, true, null); + } + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + assertFalse(awaitFlush(eventProcessor, 5, TimeUnit.SECONDS)); + eventProcessor.blockingFlush(); + eventProcessor.flush(); + scheduler.submit(() -> { }).get(5, TimeUnit.SECONDS); + lastCommitExecutor.submit(() -> { }).get(5, TimeUnit.SECONDS); + + assertEquals("an offline flush encoded events it could not send", 0, encoded.get()); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + + eventProcessor.setOffline(false); + eventProcessor.blockingFlush(); + + // One delivery, carrying everything recorded while offline. + RequestInfo sent = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + List events = new ArrayList<>(); + for (LDValue event : LDValue.parse(sent.getBody()).values()) { + events.add(event); + } + assertEquals(40, countEventsOfKind(events, "feature")); + assertEquals(1, countEventsOfKind(events, "custom")); + assertEquals(40, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void closingOfflineEncodesNothingWherePersistenceIsOff() throws Exception { + AtomicInteger encoded = new AtomicInteger(); + EventStore store = encodingCountingStore(encoded); + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(server, store, scheduler); + try { + eventProcessor.setOffline(true); + recordEvaluation(eventProcessor, true, null); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + eventProcessor.close(); + assertTrue(scheduler.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(lastCommitExecutor.awaitTermination(5, TimeUnit.SECONDS)); + + assertEquals("closing offline encoded events it could not send", 0, encoded.get()); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + /** A store that does not persist, counting every encoded event or summary staged into it. */ + private EventStore encodingCountingStore(AtomicInteger encoded) throws Exception { + return new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, false, logging.logger) { + @Override + boolean stage(byte[] serializedEvent, boolean bypassingCapacity) { + encoded.incrementAndGet(); + return super.stage(serializedEvent, bypassingCapacity); + } + + @Override + boolean stageReserved(byte[] serializedEvent) { + encoded.incrementAndGet(); + return super.stageReserved(serializedEvent); + } + }; + } + + private DirectEventProcessor makeEventProcessor(HttpServer server, EventStore store, + ScheduledExecutorService scheduler) { + return makeEventProcessor(server, store, DEFAULT_CAPACITY, scheduler); + } + + private DirectEventProcessor makeEventProcessor(HttpServer server, EventStore store, int capacity, + ScheduledExecutorService scheduler) { + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + return makeEventProcessor(new StubEventSender(), server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, + 60_000, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, scheduler, diagnosticExecutor, + store, capacity); + } + + @Test + public void aBatchDoesNotCountWhileItsRequestIsOutAndCountsAgainIfTheSendIsToBeRetried() + throws Exception { + for (boolean persistEvents : new boolean[] { true, false }) { + EventStore store = new EventStore(eventsDirectory.newFolder(), 1, persistEvents, + logging.logger); + assertTrue(store.stage(serialized("first"))); + EventStore.Batch batch = closeOne(store); + + store.startedSending(batch); + assertEquals(0, store.getPendingEventCount()); + assertTrue(store.stage(serialized("while-sending"))); + + store.stoppedSending(batch); + assertEquals(2, store.getPendingEventCount()); + assertFalse(store.stage(serialized("after-retryable"))); + } + } + + @Test + public void aSentBatchLeavesTheCountOnlyOnceWhenItIsDelivered() throws Exception { + for (boolean persistEvents : new boolean[] { true, false }) { + EventStore store = new EventStore(eventsDirectory.newFolder(), 2, persistEvents, + logging.logger); + assertTrue(store.stage(serialized("first"))); + EventStore.Batch batch = closeOne(store); + assertTrue(store.stage(serialized("second"))); + + store.startedSending(batch); + store.remove(batch); + assertEquals(1, store.getPendingEventCount()); + store.stoppedSending(batch); + assertEquals(1, store.getPendingEventCount()); + store.pendingBatches(); + assertEquals(1, store.getPendingEventCount()); + } + } + + @Test + public void aBatchAlreadyRemovedIsNotLeftOutOfTheCountWhenItsSendStarts() throws Exception { + // As when another process delivered it and a listing dropped it before this one got to it: leaving + // it out would take its events from the batches still counted, and admit that many more. + for (boolean persistEvents : new boolean[] { true, false }) { + EventStore store = new EventStore(eventsDirectory.newFolder(), 2, persistEvents, + logging.logger); + assertTrue(store.stage(serialized("first"))); + EventStore.Batch removed = closeOne(store); + assertTrue(store.stage(serialized("second"))); + closeOne(store); + store.remove(removed); + + store.startedSending(removed); + assertEquals(1, store.getPendingEventCount()); + assertTrue(store.stage(serialized("third"))); + assertFalse(store.stage(serialized("fourth"))); + + store.stoppedSending(removed); + assertEquals(2, store.getPendingEventCount()); + } + } + + @Test + public void aBatchAnotherProcessDeliveredIsNotLeftOutOfTheCountWhenItsSendStarts() throws Exception { + File directory = eventsDirectory.newFolder(); + EventStore store = new EventStore(directory, 2, true, logging.logger); + assertTrue(store.stage(serialized("first"))); + EventStore.Batch delivered = closeOne(store); + assertTrue(store.stage(serialized("second"))); + closeOne(store); + EventStore other = new EventStore(directory, 2, true, logging.logger); + other.pendingBatches(); + other.remove(delivered); + store.pendingBatches(); + + store.startedSending(delivered); + assertEquals(1, store.getPendingEventCount()); + assertTrue(store.stage(serialized("third"))); + assertFalse(store.stage(serialized("fourth"))); + } + + /** + * What the contract tests' "buffer is reset after flush" does: they have the payload as soon as it is + * sent, and record the next event before the SDK has heard back. + */ + @Test + public void aFullStoreMakesRoomAsSoonAsItsBatchIsSent() throws Exception { + CountDownLatch received = new CountDownLatch(1); + CountDownLatch respond = new CountDownLatch(1); + try (HttpServer server = HttpServer.start(Handlers.sequential( + context -> { + received.countDown(); + awaitQuietly(respond, 5, TimeUnit.SECONDS); + context.setStatus(202); + }, + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, 2); + try { + eventProcessor.recordCustomEvent(CONTEXT, "first", LDValue.ofNull(), null); + eventProcessor.recordCustomEvent(CONTEXT, "second", LDValue.ofNull(), null); + eventProcessor.flush(); + assertTrue("the first batch was never sent", received.await(5, TimeUnit.SECONDS)); + + eventProcessor.recordCustomEvent(CONTEXT, "one-more", LDValue.ofNull(), null); + respond.countDown(); + eventProcessor.flush(); + + List keys = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + RequestInfo request = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + for (LDValue event : LDValue.parse(request.getBody()).values()) { + keys.add(event.get("key").stringValue()); + } + } + assertEquals(Arrays.asList("first", "second", "one-more"), keys); + } finally { + eventProcessor.close(); + } + } + } + + private static byte[] serialized(String key) { + return ("{\"kind\":\"custom\",\"key\":\"" + key + "\"}").getBytes(StandardCharsets.UTF_8); + } + + /** + * Records more tracked evaluations than a pending run holds, lets the commit thread run whatever that + * queued, and reports how many events reached the store. + */ + private int pendingEventsInStoreAfterAFullRun(EventStore store) throws Exception { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, scheduler); + try { + for (int i = 0; i < 40; i++) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + } + lastCommitExecutor.submit(() -> { }).get(2, TimeUnit.SECONDS); + return store.getPendingEventCount(); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void closeStillWritesWhenTheEventsThreadNeverGetsToIt() throws Exception { + // The events thread is single-threaded, so a delivery that hangs would hold a final commit queued + // there for longer than close() waits. The write is close()'s promise, so it does not run there. + AtomicBoolean closing = new AtomicBoolean(false); + Queue committedDuringClose = new ConcurrentLinkedQueue<>(); + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, true, + logging.logger) { + @Override + void commit() { + if (closing.get()) { + committedDuringClose.add(Thread.currentThread()); + } + super.commit(); + } + }; + CountDownLatch releaseEventsThread = new CountDownLatch(1); + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + UNUSED_EVENTS_URI, null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY); + try { + Thread[] eventsThread = new Thread[1]; + CountDownLatch eventsThreadHeld = new CountDownLatch(1); + scheduler.submit(() -> { + eventsThread[0] = Thread.currentThread(); + eventsThreadHeld.countDown(); + awaitQuietly(releaseEventsThread, 5, TimeUnit.SECONDS); + }); + assertTrue(eventsThreadHeld.await(5, TimeUnit.SECONDS)); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + closing.set(true); + eventProcessor.close(); + + assertFalse("close() returned without writing the event down", + committedDuringClose.isEmpty()); + assertFalse("the final commit waited on the events thread", + committedDuringClose.contains(eventsThread[0])); + assertEquals(1, store.getPendingEventCount()); + } finally { + releaseEventsThread.countDown(); + scheduler.shutdownNow(); + } + } + + @Test + public void eventsHeldDuringAnOutageGoOutWithTheNextPeriodicFlush() throws Exception { + // Analytics go out through AnalyticsEventSender rather than the injectable one, so the + // delivery has to be watched at the server rather than at the stub. + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, 50, 60_000, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, + scheduler); + try { + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + server.getRecorder().requireNoRequests(300, TimeUnit.MILLISECONDS); + + eventProcessor.setOffline(false); + + List events = collectDelivered(server); + assertEquals(1, countEventsOfKind(events, "custom")); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void debugEventIsSentWhileDebuggingIsActive() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + long debugUntil = System.currentTimeMillis() + 3_600_000; + recordEvaluation(eventProcessor, false, debugUntil); + + List events = flushAndCollect(eventProcessor, server); + + LDValue debugEvent = requireEventOfKind(events, "debug"); + assertEquals(LDValue.of(FLAG_KEY), debugEvent.get("key")); + // A debug event carries the full context rather than just its keys. + assertEquals(LDValue.of(CONTEXT.getKey()), debugEvent.get("context").get("key")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void debugEventIsNotSentOnceDebuggingHasExpired() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + long debugUntil = System.currentTimeMillis() - 3_600_000; + recordEvaluation(eventProcessor, false, debugUntil); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(0, countEventsOfKind(events, "debug")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void recordingAfterCloseIsIgnored() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + eventProcessor.recordCustomEvent(CONTEXT, "before-close", LDValue.ofNull(), null); + eventProcessor.close(); + collectDelivered(server); + + eventProcessor.recordCustomEvent(CONTEXT, "after-close", LDValue.ofNull(), null); + eventProcessor.flush(); + + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } + } + + @Test + public void diagnosticInitEventIsSentWhenComingOnline() throws Exception { + try (HttpServer server = startEventsServer()) { + // makeEventProcessor takes the processor online, which is what triggers the init event. + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY, false); + try { + RequestInfo request = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + + assertEquals("/mobile/events/diagnostic", request.getPath()); + LDValue body = LDValue.parse(request.getBody()); + assertEquals(LDValue.of("diagnostic-init"), body.get("kind")); + // Only one, even though going online and coming to the foreground both ask for it. + returnToTheForegroundWithTheDiagnosticsThreadFree(eventProcessor); + server.getRecorder().requireNoRequests(500, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void diagnosticInitEventIsNotSentWhileOffline() throws Exception { + // Built offline and left that way. Waiting for the init first, as this used to, means the + // only thing stopping a second one is that the first already went -- so the test passes just + // as happily against a processor that sends init events while offline. + BlockingQueue posted = new LinkedBlockingQueue<>(); + EventSender sender = new StubEventSender() { + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + posted.add(LDValue.parse(new String(data, StandardCharsets.UTF_8))); + return new Result(true, false, null); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(sender, makeDiagnosticStore(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + assertNull("a diagnostic event was sent while offline", + posted.poll(500, TimeUnit.MILLISECONDS)); + + // Proof that the silence above was the offline state and not a fixture that could never + // have sent anything. + eventProcessor.setOffline(false); + assertEquals(LDValue.of("diagnostic-init"), requirePosted(posted).get("kind")); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void flushWithTimeoutReportsDeliveredEvents() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + assertEquals(1, countEventsOfKind(collectDelivered(server), "custom")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aCommitNeverSplitsAnEvaluationAcrossTwoCommits() throws Exception { + // The other half of the atomicity invariant. close() only ever commits once, so it can show + // an evaluation being stranded but not one being split: a counter staged by commit N with its + // feature event following in N+1. Payloads cannot show it here, because a delivery reads the + // store and may cut or join commits for its own reasons, so this looks at what each commit + // stages instead, while recording continues. + // + // Every evaluation is tracked and the capacity is far beyond what the run produces, so within + // a commit the counter for the flag and the number of feature events are the same number. + // + // Persistence is off so that the store never commits on its own; every commit() is one of the + // processor's. Where nothing persists only a delivery commits, so the processor is online and + // each flush commits and delivers. + // + // Repeated because the window is narrow -- the two writes are adjacent, and a commit has to + // land between them. + int commitsWithCounters = 0; + for (int trial = 0; trial < SPLIT_TRIALS; trial++) { + List> commits = new ArrayList<>(); + EventStore store = new EventStore(eventsDirectory.newFolder(), NO_DROP_CAPACITY, + false, logging.logger) { + private List staged = new ArrayList<>(); + + @Override + synchronized boolean stage(byte[] serializedEvent, boolean bypassingCapacity) { + staged.add(LDValue.parse(new String(serializedEvent, StandardCharsets.UTF_8))); + return super.stage(serializedEvent, bypassingCapacity); + } + + @Override + synchronized boolean stageReserved(byte[] serializedEvent) { + staged.add(LDValue.parse(new String(serializedEvent, StandardCharsets.UTF_8))); + return super.stageReserved(serializedEvent); + } + + @Override + synchronized void commit() { + commits.add(staged); + staged = new ArrayList<>(); + super.commit(); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + HttpServer server = startEventsServer(); + DirectEventProcessor eventProcessor = makeEventProcessor(server, store, NO_DROP_CAPACITY, + scheduler); + try { + eventProcessor.setOffline(false); + AtomicInteger recorded = new AtomicInteger(); + List recorders = new ArrayList<>(); + for (int i = 0; i < RACE_RECORDERS; i++) { + Thread recorder = new Thread(() -> { + for (int n = 0; n < EVALUATIONS_PER_RACE_RECORDER; n++) { + recordEvaluation(eventProcessor, true, null); + recorded.incrementAndGet(); + } + }); + recorders.add(recorder); + recorder.start(); + } + int target = RACE_RECORDERS * EVALUATIONS_PER_RACE_RECORDER; + while (recorded.get() < target) { + eventProcessor.blockingFlush(); + } + for (Thread recorder : recorders) { + recorder.join(); + } + eventProcessor.blockingFlush(); + + assertEquals("trial " + trial + ": events were dropped, so a commit may be short" + + " for that reason instead", + 0, eventProcessor.getAndClearDroppedCount()); + int counted = 0; + synchronized (store) { + for (List commit : commits) { + int counters = summaryCounters(commit, FLAG_KEY); + assertEquals("trial " + trial + ": a commit counted evaluations whose feature" + + " events were staged separately", + countEventsOfKind(commit, "feature"), counters); + counted += counters; + if (counters > 0) { + commitsWithCounters++; + } + } + } + assertEquals("trial " + trial + ": some evaluations were never committed", + target, counted); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + server.close(); + } + } + // Otherwise a single commit per trial would satisfy everything above without a commit ever + // having overlapped a recording. + assertTrue("every evaluation went out in one commit, so nothing was interleaved", + commitsWithCounters > SPLIT_TRIALS); + } + + @Test + public void periodicTaskSurvivesErrorFromSender() throws Exception { + // An Error (not Exception) from a scheduled run used to cancel the repeating future with + // nothing logged, after which enableOrDisableTask kept returning that dead future forever. + // Driven through diagnostics, because that is the periodic task the injectable EventSender + // still carries once analytics go out through the store and AnalyticsEventSender. + CountDownLatch firstPeriodicAttempt = new CountDownLatch(1); + BlockingQueue delivered = new LinkedBlockingQueue<>(); + EventSender sender = new StubEventSender() { + private final AtomicInteger periodicAttempts = new AtomicInteger(); + + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + String kind = LDValue.parse(new String(data, StandardCharsets.UTF_8)) + .get("kind").stringValue(); + if ("diagnostic-init".equals(kind)) { + return new Result(true, false, null); // one-shot, not the series under test + } + if (periodicAttempts.getAndIncrement() == 0) { + firstPeriodicAttempt.countDown(); + throw new Error("periodic diagnostics"); + } + delivered.add(data); + return new Result(true, false, null); + } + }; + + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(sender, + URI.create("https://events.example"), makeDiagnosticStore(), + NO_PERIODIC_FLUSH_MILLIS, 40, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, + scheduler); + try { + // Coming online is the only thing that schedules the series. Nothing after this point + // touches the processor's state, so it has to stay alive on its own. + eventProcessor.setOffline(false); + assertTrue("sender never saw the first periodic diagnostic", + firstPeriodicAttempt.await(2, TimeUnit.SECONDS)); + + assertNotNull("periodic diagnostics did not run again after Error", + delivered.poll(2, TimeUnit.SECONDS)); + logging.assertErrorLogged("Unexpected error in event processor"); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void flushWithTimeoutReportsSuccessWhenThereIsNothingToSend() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + // Nothing was recorded, so the caller's events are not waiting anywhere. + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unexpectedRecordingErrorDoesNotBubbleToCallerAndLogs() throws Exception { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + // Must not throw if record throws: + eventProcessor.record(new Event(System.currentTimeMillis(), CONTEXT) { + @Override + public long getSamplingRatio() { + throw new RuntimeException("simulated record crash"); + } + }); + logging.assertErrorLogged("Unexpected error in event processor: java.lang.RuntimeException: simulated record crash"); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void anErrorWhileRecordingStillReachesTheCaller() throws Exception { + // Only exceptions are the SDK's to absorb. An Error such as OutOfMemoryError belongs to the + // application's crash reporting, and swallowing it on the caller's thread would hide it. + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + eventProcessor.record(new Event(System.currentTimeMillis(), CONTEXT) { + @Override + public long getSamplingRatio() { + throw new StackOverflowError("simulated"); + } + }); + fail("the Error was swallowed"); + } catch (StackOverflowError expected) { + // what the application's handler would see + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void aFailedCommitOnTheCallersThreadDoesNotReachTheCaller() throws Exception { + // With IMMEDIATE the commit runs inside track, identify and flush, after record() has + // returned, so it needs its own guard. + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, true, + logging.logger) { + @Override + void commit() { + throw new IllegalStateException("simulated commit failure"); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + UNUSED_EVENTS_URI, null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.recordIdentifyEvent(CONTEXT); + eventProcessor.flush(); + // Its own last-chance commit runs on the caller's thread when the events thread's fails. + eventProcessor.close(); + + logging.assertErrorLogged( + "Unexpected error in event processor: java.lang.IllegalStateException: simulated commit failure"); + } finally { + scheduler.shutdownNow(); + } + } + + @Test + public void closeGivesUpWaitingOnAStalledDelivery() throws Exception { + // close() runs on the caller's thread, usually the main one, so a send that never comes back + // used to park the application there for as long as the HTTP timeouts allowed. + Semaphore letResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.all(Handlers.waitFor(letResponseFinish), + Handlers.status(202)))) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "stalled", LDValue.ofNull(), null); + + long startedAtNanos = System.nanoTime(); + eventProcessor.close(); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAtNanos); + + assertNotNull("the sender was never asked to send anything", + server.getRecorder().requireRequest(2, TimeUnit.SECONDS)); + assertTrue("close() waited " + elapsedMillis + "ms on a budget of " + + CLOSE_BUDGET_MILLIS + "ms", elapsedMillis < CLOSE_BUDGET_MILLIS * 5); + logging.assertWarnLogged("Gave up waiting for the final event delivery"); + } finally { + letResponseFinish.release(Integer.MAX_VALUE); + scheduler.shutdownNow(); + } + } + } + + @Test + public void closeGivesUpWaitingOnAStalledDiskWrite() throws Exception { + // A commit stalled on the disk holds commitLock, and the store's I/O lock, for as long as the + // write takes. close() used to fall back to committing on the caller's thread once its budget + // ran out, which then waited on those locks with no bound at all. + CountDownLatch writeStarted = new CountDownLatch(1); + Semaphore letWriteFinish = new Semaphore(0); + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, true, + logging.logger) { + @Override + FileOutputStream openForAppending(File file) throws IOException { + writeStarted.countDown(); + awaitQuietly(letWriteFinish); + return super.openForAppending(file); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + UNUSED_EVENTS_URI, null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, false); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + // A commit point, so a commit is queued and stalls on the disk, holding the locks. + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + assertTrue("the commit never reached the disk", writeStarted.await(5, TimeUnit.SECONDS)); + + long startedAtNanos = System.nanoTime(); + // On a thread of its own, so that a close() that does hang fails this test rather than + // hanging the suite. + Future closing = caller.submit(() -> { + eventProcessor.close(); + return null; + }); + closing.get(CLOSE_BUDGET_MILLIS * 5, TimeUnit.MILLISECONDS); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAtNanos); + + assertTrue("close() waited " + elapsedMillis + "ms on a budget of " + + CLOSE_BUDGET_MILLIS + "ms", elapsedMillis < CLOSE_BUDGET_MILLIS * 3); + logging.assertWarnLogged("Gave up waiting for the final event commit"); + } finally { + letWriteFinish.release(Integer.MAX_VALUE); + caller.shutdownNow(); + scheduler.shutdownNow(); + } + } + + private static void awaitQuietly(Semaphore semaphore) { + try { + if (semaphore.tryAcquire(30, TimeUnit.SECONDS)) { + semaphore.release(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Test + public void flushWithTimeoutReportsFailureWhileOffline() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + // The events are still buffered rather than delivered, and no amount of waiting + // changes that, so the caller is told so instead of being told they are safe. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushesArrivingWhileADeliveryRunsShareOneFollowUpDelivery() throws Exception { + // Otherwise a flush called faster than a post completes queues a post per call, and the + // flush that matters -- the one at shutdown, with a deadline -- waits behind all of them. + Semaphore letFirstResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.sequential( + Handlers.all(Handlers.waitFor(letFirstResponseFinish), Handlers.status(202)), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "first", LDValue.ofNull(), null); + Future first = eventProcessor.flushAsync(); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + + // The delivery thread is inside that post, so none of these can start, and each of + // them has to be answered by the one delivery that is queued behind it. + eventProcessor.recordCustomEvent(CONTEXT, "second", LDValue.ofNull(), null); + Future queued = eventProcessor.flushAsync(); + for (int i = 0; i < 50; i++) { + assertSame(queued, eventProcessor.flushAsync()); + } + + letFirstResponseFinish.release(1); + assertTrue(first.get(5, TimeUnit.SECONDS)); + assertTrue(queued.get(5, TimeUnit.SECONDS)); + + // One post for the running delivery and one for the 51 that joined, and no more. + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } finally { + letFirstResponseFinish.release(Integer.MAX_VALUE); + eventProcessor.close(); + } + } + } + + @Test + public void aFlushJoiningADeliveryStillCoversWhatTheCallerRecorded() throws Exception { + // Joining is only sound while the delivery it joins has not taken the buffer yet, so what + // the joining caller recorded has to come back in that delivery's payload. + Semaphore letFirstResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.sequential( + Handlers.all(Handlers.waitFor(letFirstResponseFinish), Handlers.status(202)), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "first", LDValue.ofNull(), null); + Future first = eventProcessor.flushAsync(); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + + eventProcessor.recordCustomEvent(CONTEXT, "joined", LDValue.ofNull(), null); + Future queued = eventProcessor.flushAsync(); + + letFirstResponseFinish.release(1); + assertTrue(first.get(5, TimeUnit.SECONDS)); + assertTrue(queued.get(5, TimeUnit.SECONDS)); + + RequestInfo second = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue("the joining caller's event was left behind", + second.getBody().contains("\"key\":\"joined\"")); + } finally { + letFirstResponseFinish.release(Integer.MAX_VALUE); + eventProcessor.close(); + } + } + } + + @Test + public void flushWithTimeoutReportsFailureWhenTheTimeoutExpiresFirst() throws Exception { + Semaphore letResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.all(Handlers.waitFor(letResponseFinish), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + Future delivery = eventProcessor.flushAsync(); + + try { + delivery.get(100, TimeUnit.MILLISECONDS); + fail("the delivery finished while its response was still being held"); + } catch (TimeoutException expected) { + // The caller gives up here, as flushAndWait does when its budget runs out. + } + + // Left running rather than canceled, the delivery still gets its events through + // once the service answers, and a canceled one could not report that. + letResponseFinish.release(1); + assertTrue(delivery.get(5, TimeUnit.SECONDS)); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + } finally { + // Released before closing, so that the delivery still in flight can finish rather + // than hold up the shutdown that close() waits on. + letResponseFinish.drainPermits(); + letResponseFinish.release(Integer.MAX_VALUE); + eventProcessor.close(); + } + } + } + + @Test + public void flushReportsFailureWhenTheServiceRefusesTheEventsForNow() throws Exception { + // 503 is a failure that may pass: the processor retries it a second later, and a caller waiting on + // the flush is answered by that retry. The batch is kept after it; see the tests below. + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushAndWaitDoesNotReportDeliveryForEventsAnEarlierFailedFlushDrained() throws Exception { + // Every request fails recoverably (503): each delivery attempts, retries once, and keeps the batch. + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + + // Stands in for the periodic flush or the SDK's flush-on-background: it closes the + // buffer into a batch and its delivery fails. The batch stays in the store. + eventProcessor.blockingFlush(); + server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + + // The buffer is empty, but the event is still waiting in its batch, not delivered. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchAnUnheardDeliveryCouldNotSendGoesOutOnceTheServiceRecovers() throws Exception { + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(503)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilderWithoutBackoff(DEFAULT_CAPACITY), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + eventProcessor.blockingFlush(); + // The attempt and its one retry: the service only recovers once both have failed. + String payloadId = server.getRecorder().requireRequest(10, TimeUnit.SECONDS) + .getHeader(PAYLOAD_ID_HEADER); + server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + service.setTarget(Handlers.status(202)); + + // Nobody heard the failure, and the next flush still sends what it refused. + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo resent = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + assertTrue(resent.getBody().contains("\"key\":\"kept-event\"")); + assertEquals("the same delivery, so the service can discard a repeat", + payloadId, resent.getHeader(PAYLOAD_ID_HEADER)); + + // Delivered once: the next flush has only its own events to send. + eventProcessor.recordCustomEvent(CONTEXT, "later-event", LDValue.ofNull(), null); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo later = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + assertTrue(later.getBody().contains("\"key\":\"later-event\"")); + assertFalse(later.getBody().contains("kept-event")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchIsKeptThroughRepeatedRefusalsAndDeliveredOnceTheServiceRecovers() throws Exception { + assertKeptThroughRefusalsAndDeliveredOnRecovery(503, eventsBuilderWithoutBackoff(DEFAULT_CAPACITY), 3); + } + + @Test + public void aBatchHeldInMemoryIsKeptThroughRepeatedRefusalsWherePersistenceIsOff() throws Exception { + // Without a disk the store holds batches in memory, and a refusal must not cost them either. + assertKeptThroughRefusalsAndDeliveredOnRecovery(503, + eventsBuilderWithoutBackoff(DEFAULT_CAPACITY).eventPersistence(EventPersistence.DISABLED), 3); + } + + @Test + public void flushesDuringAnOutageCloseNoBatchesBehindTheOneBlocked() throws Exception { + // A batch per flush would grow without bound for as long as an outage lasts: files on the disk, + // or batches held in memory, each with a summary of its own. + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(503)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilderWithoutBackoff(DEFAULT_CAPACITY), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "blocked", LDValue.ofNull(), null); + for (int flush = 0; flush < 3; flush++) { + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + // The attempt and its retry, both of the batch at the head. + for (int attempt = 0; attempt < 2; attempt++) { + RequestInfo refused = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(refused.getBody().contains("\"key\":\"blocked\"")); + } + eventProcessor.recordCustomEvent(CONTEXT, "during-" + flush, LDValue.ofNull(), null); + } + service.setTarget(Handlers.status(202)); + + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo head = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(head.getBody().contains("\"key\":\"blocked\"")); + RequestInfo since = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + for (int flush = 0; flush < 3; flush++) { + assertTrue("during-" + flush, since.getBody().contains("\"key\":\"during-" + flush + "\"")); + } + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushFailsWhileTheStoredEventsCannotBeListed() throws Exception { + // A directory that cannot be listed is not an empty one: the batch the flush closes is on the disk. + assumePersisting(); + try (HttpServer server = HttpServer.start(Handlers.status(202))) { + EventProcessor eventProcessor = makeEventProcessor(server, eventsBuilder(DEFAULT_CAPACITY), true); + File directory = EventDirectory.defaultDirectory(eventsDirectory.getRoot(), MOBILE_KEY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "first", LDValue.ofNull(), null); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(directory.isDirectory()); + + eventProcessor.recordCustomEvent(CONTEXT, "unlisted", LDValue.ofNull(), null); + assertTrue(directory.setReadable(false)); + try { + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + } finally { + assertTrue(directory.setReadable(true)); + } + + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo request = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(request.getBody().contains("\"key\":\"unlisted\"")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushFailsWhileTheEventLogWillNotClose() throws Exception { + // The events are still in the log, for the next close to move, and this flush did not send them. + File directory = eventsDirectory.newFolder(); + AtomicBoolean closesFail = new AtomicBoolean(true); + EventStore store = new EventStore(directory, DEFAULT_CAPACITY, true, logging.logger) { + @Override + List closeBatch() { + if (!closesFail.get()) { + return super.closeBatch(); + } + // Read-only only for the close, so the log is written and then cannot be renamed. + assertTrue(directory.setWritable(false)); + try { + return super.closeBatch(); + } finally { + assertTrue(directory.setWritable(true)); + } + } + }; + try (HttpServer server = HttpServer.start(Handlers.status(202))) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "unclosed", LDValue.ofNull(), null); + + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + assertTrue(store.holdsUnclosedEvents()); + + closesFail.set(false); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo request = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(request.getBody().contains("\"key\":\"unclosed\"")); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void theSummariesAnOutageWritesLeaveRoomForEvents() throws Exception { + // Every flush of an outage is a commit point, and each writes a summary of the evaluations since + // the last. Counted as events, they would leave a long outage refusing every track. + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(503)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, eventsBuilderWithoutBackoff(2), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "blocked", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + for (int flush = 0; flush < 3; flush++) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, false, null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + } + + eventProcessor.recordCustomEvent(CONTEXT, "after-the-summaries", LDValue.ofNull(), null); + service.setTarget(Handlers.status(202)); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + boolean sent = false; + // Past the requests the outage refused, two for each flush. + for (int request = 0; request < 10 && !sent; request++) { + sent = server.getRecorder().requireRequest(5, TimeUnit.SECONDS).getBody() + .contains("\"key\":\"after-the-summaries\""); + } + assertTrue("refused for room the summaries took", sent); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void everyFailingStatusKeepsTheBatchPastTheRetry() throws Exception { + // Each has to keep the batch, not just 503: a refusal such as a 401 only waits longer. Not a 400, + // which drops the batch when it comes twice in a row; see the tests below. + for (int status : new int[] { 401, 403, 404, 408, 413, 429, 500, 502, 504 }) { + assertKeptThroughRefusalsAndDeliveredOnRecovery(status, eventsBuilderWithoutBackoff(DEFAULT_CAPACITY), + 2); + } + } + + @Test + public void eventsRecordedDuringAnOutageAreDeliveredWithTheBatchesRefusedBeforeThem() throws Exception { + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(503)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilderWithoutBackoff(DEFAULT_CAPACITY), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "first-event", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + eventProcessor.recordCustomEvent(CONTEXT, "second-event", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + // Two flushes, each an attempt and a retry, all at the first batch: delivery stops at + // the oldest batch that fails in a way that may pass. + for (int i = 0; i < 4; i++) { + assertTrue(server.getRecorder().requireRequest(5, TimeUnit.SECONDS).getBody() + .contains("\"key\":\"first-event\"")); + } + service.setTarget(Handlers.status(202)); + + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + // Oldest first, each exactly once. + RequestInfo first = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + RequestInfo second = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + assertTrue(first.getBody().contains("\"key\":\"first-event\"")); + assertFalse(first.getBody().contains("second-event")); + assertTrue(second.getBody().contains("\"key\":\"second-event\"")); + assertFalse(second.getBody().contains("first-event")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchStillRefusedAtCloseIsDeliveredByTheNextRun() throws Exception { + assumePersisting(); + String payloadId; + try (HttpServer refusing = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(refusing, DEFAULT_CAPACITY); + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + payloadId = refusing.getRecorder().requireRequest(1, TimeUnit.SECONDS) + .getHeader(PAYLOAD_ID_HEADER); + eventProcessor.close(); + } + + // The same directory, as the application's next launch would find it. + try (HttpServer server = startEventsServer()) { + EventProcessor nextRun = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + assertTrue(awaitFlush(nextRun, 10, TimeUnit.SECONDS)); + RequestInfo resent = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + assertTrue(resent.getBody().contains("\"key\":\"kept-event\"")); + assertEquals(payloadId, resent.getHeader(PAYLOAD_ID_HEADER)); + } finally { + nextRun.close(); + } + } + } + + @Test + public void aBatchRefusedWith400TwiceInARowIsDroppedAndNoLongerHoldsTheOnesBehindIt() throws Exception { + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(400)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilderWithoutBackoff(DEFAULT_CAPACITY), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "malformed", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + String payloadId = server.getRecorder().requireRequest(1, TimeUnit.SECONDS) + .getHeader(PAYLOAD_ID_HEADER); + assertEquals(payloadId, server.getRecorder().requireRequest(1, TimeUnit.SECONDS) + .getHeader(PAYLOAD_ID_HEADER)); + server.getRecorder().requireNoRequests(300, TimeUnit.MILLISECONDS); + service.setTarget(Handlers.status(202)); + + // What was recorded since goes out alone, and that flush is the first a caller waits on + // since the loss, so it hears about it. + eventProcessor.recordCustomEvent(CONTEXT, "next", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo next = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(next.getBody().contains("\"key\":\"next\"")); + assertFalse(next.getBody().contains("malformed")); + server.getRecorder().requireNoRequests(300, TimeUnit.MILLISECONDS); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchIsDroppedOnlyForTwo400sInARow() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.sequential(Handlers.status(400), + Handlers.status(503), Handlers.status(400), Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilderWithoutBackoff(DEFAULT_CAPACITY), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "kept", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + String payloadId = null; + for (int post = 0; post < 4; post++) { + RequestInfo request = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(request.getBody().contains("\"key\":\"kept\"")); + if (payloadId == null) { + payloadId = request.getHeader(PAYLOAD_ID_HEADER); + } + assertEquals(payloadId, request.getHeader(PAYLOAD_ID_HEADER)); + } + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchRefusedWith503IsPostedOnlyAsTheBackoffAllows() throws Exception { + assertPostedOnlyAsTheBackoffAllows(503); + } + + /** + * Kept is not the same as resent on its own: a batch the service always refuses is posted a second + * after the first failure and then twice as long after each one, and a flush in between posts nothing, + * or it would be posted in a loop. + */ + private void assertPostedOnlyAsTheBackoffAllows(int status) throws Exception { + long second = DirectEventProcessor.RETRY_DELAY_MILLIS; + try (HttpServer server = HttpServer.start(Handlers.status(status))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "refused", LDValue.ofNull(), null); + // The attempt and the retry a second later, which the flush waits for. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + + // Two seconds now, which a flush neither posts in nor waits out. + long start = System.nanoTime(); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + assertTrue("the flush waited out the backoff", + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) < second); + server.getRecorder().requireNoRequests(second, TimeUnit.MILLISECONDS); + + // The retry that flush queued, once the two seconds are up, and then four seconds of nothing. + server.getRecorder().requireRequest(3 * second, TimeUnit.MILLISECONDS); + server.getRecorder().requireNoRequests(3 * second, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aRefusedBatchIsKeptAndPostedAgainOnceTheLongerWaitIsOver() throws Exception { + // A 401 for a valid key happens during an incident and passes in minutes, so the batch is kept and + // the SDK waits longer before trying again, rather than deleting it and stopping. Cut to a second + // and a half here from the eight and a half minutes the SDK waits. + long refusalWaitMillis = 1_500; + long second = DirectEventProcessor.RETRY_DELAY_MILLIS; + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(401)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, + ((ComponentsImpl.EventProcessorBuilderImpl) eventsBuilder(DEFAULT_CAPACITY)) + .retryDelaysForTesting(second, second, refusalWaitMillis), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "refused", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + String payloadId = server.getRecorder().requireRequest(1, TimeUnit.SECONDS) + .getHeader(PAYLOAD_ID_HEADER); + + // Recording goes on, and a flush during the wait says it failed without posting. + eventProcessor.recordCustomEvent(CONTEXT, "recorded-meanwhile", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireNoRequests(refusalWaitMillis / 2, TimeUnit.MILLISECONDS); + service.setTarget(Handlers.status(202)); + + // The retry the refusal queued: the refused batch under the ID it was refused under, then what + // was recorded since. + RequestInfo resent = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(resent.getBody().contains("\"key\":\"refused\"")); + assertEquals(payloadId, resent.getHeader(PAYLOAD_ID_HEADER)); + RequestInfo since = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(since.getBody().contains("\"key\":\"recorded-meanwhile\"")); + + // One accepted delivery puts the backoff back: the next failure waits only the second a flush + // sits out, so the flush hears that its retry worked. + service.setTarget(Handlers.sequential(Handlers.status(503), Handlers.status(202))); + eventProcessor.recordCustomEvent(CONTEXT, "after", LDValue.ofNull(), null); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aRetryAfterSetsTheWaitBeforeTheNextPost() throws Exception { + // Four seconds, jittered down to no less than two, which is past the second a flush waits out. + try (HttpServer server = HttpServer.start(Handlers.sequential( + Handlers.all(Handlers.header("Retry-After", "4"), Handlers.status(503)), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(1_800, TimeUnit.MILLISECONDS); + assertTrue(server.getRecorder().requireRequest(5, TimeUnit.SECONDS).getBody() + .contains("\"key\":\"an-event\"")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushesDuringARefusedDeliveryShareOneDeliveryAndOneRetry() throws Exception { + Semaphore letFirstResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.all(Handlers.waitFor(letFirstResponseFinish), + Handlers.status(503)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "refused", LDValue.ofNull(), null); + List> flushes = new ArrayList<>(); + flushes.add(eventProcessor.flushAsync()); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + + // Held on the first post, every one of these joins the single delivery queued behind it. + for (int i = 0; i < 20; i++) { + flushes.add(eventProcessor.flushAsync()); + } + letFirstResponseFinish.release(100); + + for (Future flush : flushes) { + assertFalse(flush.get(10, TimeUnit.SECONDS)); + } + // The second delivery runs inside the wait the first failure started and posts nothing; the + // one retry both share does. + server.getRecorder().requireRequest(2, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(DirectEventProcessor.RETRY_DELAY_MILLIS * 3, + TimeUnit.MILLISECONDS); + } finally { + letFirstResponseFinish.release(100); + eventProcessor.close(); + } + } + } + + @Test + public void thePeriodicFlushPostsARefusedBatchNoMoreOftenThanTheBackoffAllows() throws Exception { + int intervalMillis = 200; + long windowMillis = 2_000; + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).flushIntervalMillis(intervalMillis), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "refused", LDValue.ofNull(), null); + Thread.sleep(windowMillis); + + // At most one post a second, the backoff's ceiling where the interval is shorter; a loop + // would be in the hundreds by now. + int posts = server.getRecorder().count(); + long most = windowMillis / DirectEventProcessor.RETRY_DELAY_MILLIS + 1; + assertTrue(posts + " posts in " + windowMillis + "ms", posts >= 2 && posts <= most); + + // Offline stops it entirely, kept batch or not. + eventProcessor.setOffline(true); + Thread.sleep(DirectEventProcessor.RETRY_DELAY_MILLIS + intervalMillis); + int whenOffline = server.getRecorder().count(); + Thread.sleep(intervalMillis * 3); + assertEquals(whenOffline, server.getRecorder().count()); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchAPreviousRunLeftIsTriedOnceAtStartAndThenWaitsWhileStillRefused() throws Exception { + assumePersisting(); + try (HttpServer refusing = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(refusing, DEFAULT_CAPACITY); + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + eventProcessor.close(); + } + + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + EventProcessor nextRun = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + // The delivery at start and its retry, and nothing after them until a flush asks. + assertTrue(server.getRecorder().requireRequest(5, TimeUnit.SECONDS).getBody() + .contains("\"key\":\"kept-event\"")); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(DirectEventProcessor.RETRY_DELAY_MILLIS * 3, + TimeUnit.MILLISECONDS); + } finally { + nextRun.close(); + } + } + } + + /** + * Refuses a batch with {@code status} for several flushes in a row, each of which attempts it and + * retries it once, then lets the service recover and checks that the batch arrives exactly once, + * under the payload ID it was first sent with. + */ + private void assertKeptThroughRefusalsAndDeliveredOnRecovery(int status, EventProcessorBuilder events, + int refusedFlushes) throws Exception { + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(status)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, events, true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + String payloadId = null; + for (int flush = 0; flush < refusedFlushes; flush++) { + assertFalse("HTTP " + status + ", flush " + flush, + awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + for (int attempt = 0; attempt < 2; attempt++) { + RequestInfo refused = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue("HTTP " + status + ": the batch was dropped before flush " + flush, + refused.getBody().contains("\"key\":\"kept-event\"")); + if (payloadId == null) { + payloadId = refused.getHeader(PAYLOAD_ID_HEADER); + } + assertEquals(payloadId, refused.getHeader(PAYLOAD_ID_HEADER)); + } + } + service.setTarget(Handlers.status(202)); + + assertTrue("HTTP " + status + ": not delivered once the service recovered", + awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo delivered = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(delivered.getBody().contains("\"key\":\"kept-event\"")); + assertEquals(payloadId, delivered.getHeader(PAYLOAD_ID_HEADER)); + + // Gone once accepted: nothing is left to resend. + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushIsNotToldEventsArrivedThatCouldNotBeSerialized() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "poison", LDValue.ofNull(), Double.NaN); + + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushIsNotToldEventsArrivedWhenSomeOfThemCouldNotBeSerialized() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "poison", LDValue.ofNull(), Double.NaN); + eventProcessor.recordCustomEvent(CONTEXT, "fine", LDValue.ofNull(), 1.0); + + // The post succeeds, and still not everything the caller recorded is in it. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + List events = collectDelivered(server); + assertEquals(LDValue.of("fine"), requireEventOfKind(events, "custom").get("key")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushIsNotToldEvaluationsArrivedWhoseSummaryCouldNotBeSerialized() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + // A null flag key, as a Java caller can pass, which Gson cannot write as a member name. + eventProcessor.recordEvaluationEvent(CONTEXT, null, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + recordEvaluation(eventProcessor, false, null); + eventProcessor.recordIdentifyEvent(CONTEXT); + + // The summary is dropped whole, the valid flag's counter with it, and the flush says so. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + List events = collectDelivered(server); + assertEquals(1, countEventsOfKind(events, "identify")); + assertEquals(0, countEventsOfKind(events, "summary")); + logging.assertErrorLogged("Dropping unserializable summary event"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushIsNotToldEventsArrivedWhenOneWasTooLargeToStore() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + char[] data = new char[EventLogFormat.MAX_FRAME_SIZE]; + Arrays.fill(data, 'x'); + eventProcessor.recordCustomEvent(CONTEXT, "too-large", LDValue.of(new String(data)), null); + eventProcessor.recordCustomEvent(CONTEXT, "fine", LDValue.ofNull(), null); + + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + List events = collectDelivered(server); + assertEquals(1, countEventsOfKind(events, "custom")); + assertEquals(LDValue.of("fine"), requireEventOfKind(events, "custom").get("key")); + logging.assertErrorLogged("too large to store"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushOfMoreThanABatchSendsEveryEventOnceInRequestsNoLargerThanABatch() throws Exception { + // Small enough that a batch closed at the limit is within one event of it, so a body that is + // short of the limit by more than that shows the split happened somewhere else. + int dataChars = EventLogFormat.MAX_BATCH_SIZE / 32; + int eventCount = 100; + char[] data = new char[dataChars]; + Arrays.fill(data, 'x'); + LDValue padding = LDValue.of(new String(data)); + // MockWebServer rather than test-helpers' HttpServer, which keeps only what one read of the + // socket returns and so cuts a body this large short. + try (MockWebServer server = new MockWebServer()) { + server.setDispatcher(new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + return new MockResponse().setResponseCode(202); + } + }); + EventProcessor eventProcessor = makeEventProcessor(server.url("/").uri(), + eventsBuilder(eventCount), true); + try { + for (int i = 1; i <= eventCount; i++) { + eventProcessor.recordCustomEvent(CONTEXT, "event-" + i, padding, null); + } + + assertTrue(awaitFlush(eventProcessor, 30, TimeUnit.SECONDS)); + + List bodySizes = new ArrayList<>(); + Set payloadIds = new HashSet<>(); + List keys = new ArrayList<>(); + for (int i = server.getRequestCount(); i > 0; i--) { + RecordedRequest request = server.takeRequest(10, TimeUnit.SECONDS); + assertNotNull(request); + String body = request.getBody().readUtf8(); + bodySizes.add((int) request.getBodySize()); + payloadIds.add(request.getHeader(PAYLOAD_ID_HEADER)); + for (LDValue event : LDValue.parse(body).values()) { + keys.add(event.get("key").stringValue()); + } + } + + assertTrue("one request carried everything: " + bodySizes, bodySizes.size() > 1); + Collections.sort(bodySizes); + assertTrue("a request was larger than a batch: " + bodySizes, + bodySizes.get(bodySizes.size() - 1) <= EventLogFormat.MAX_BATCH_SIZE); + // Only the batch the events ran out in may be short of full. + for (int size : bodySizes.subList(1, bodySizes.size())) { + assertTrue("a batch closed short of the limit: " + bodySizes, + size > EventLogFormat.MAX_BATCH_SIZE - dataChars - 4096); + } + assertEquals(bodySizes.size(), payloadIds.size()); + List expected = new ArrayList<>(); + for (int i = 1; i <= eventCount; i++) { + expected.add("event-" + i); + } + Collections.sort(keys); + Collections.sort(expected); + assertEquals(expected, keys); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aLossAfterAFlushClosedItsBatchIsReportedToTheNextFlushInstead() throws Exception { + assumePersisting(); + // A flush answers for the batch it closed. One that met a retryable failure answers only when + // its retry does, which is long enough for a loss to happen in between, and that loss belongs + // to the flush that closes the batch after it rather than to this one. + char[] data = new char[EventLogFormat.MAX_FRAME_SIZE]; + Arrays.fill(data, 'x'); + LDValue tooLarge = LDValue.of(new String(data)); + Semaphore letFirstResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.sequential( + Handlers.all(Handlers.waitFor(letFirstResponseFinish), Handlers.status(503)), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + Future firstFlush = eventProcessor.flushAsync(); + // The batch is closed and on its way by the time the request lands, so the loss below + // happens after this flush took everything it can speak for. + assertNotNull(server.getRecorder().requireRequest(5, TimeUnit.SECONDS)); + + eventProcessor.recordCustomEvent(CONTEXT, "too-large", tooLarge, null); + logging.assertErrorLogged("too large to store"); + // One permit, not every permit: only this first response is held, and the finally + // below has to be able to release it again without overflowing the count. + letFirstResponseFinish.release(); + + // The retry sends the batch this flush closed, and everything in it arrives. + assertTrue("told about a loss that happened after it closed its batch", + Boolean.TRUE.equals(firstFlush.get(10, TimeUnit.SECONDS))); + // And the loss is still owed to somebody, so the next caller is the one to hear it. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + } finally { + letFirstResponseFinish.release(); + eventProcessor.close(); + } + } + } + + @Test + public void eventsHeldInMemoryWhenAWriteFailsAreDeliveredWithThoseWrittenBeforeIt() throws Exception { + AtomicInteger writes = new AtomicInteger(); + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, true, + logging.logger) { + @Override + FileOutputStream openForAppending(File file) throws IOException { + return new FileOutputStream(file, true) { + @Override + public void write(byte[] bytes) throws IOException { + // The file header and the first event land; the disk is full from then on. + if (writes.incrementAndGet() > 2) { + throw new IOException("No space left on device"); + } + super.write(bytes); + } + }; + } + }; + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, true); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "written", LDValue.ofNull(), null); + // Its write fails, so persistence is given up with "written" in the log and this in memory. + eventProcessor.recordCustomEvent(CONTEXT, "held", LDValue.ofNull(), null); + logging.assertWarnLogged("Giving up on persisting events"); + + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + List keys = new ArrayList<>(); + for (LDValue event : collectDelivered(server)) { + keys.add(event.get("key").stringValue()); + } + assertEquals("oldest first, and neither left behind", + Arrays.asList("written", "held"), keys); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void aFlushIsToldOfABatchADeliveryNobodyWaitedOnDroppedAsUnreadable() throws Exception { + File directory = eventsDirectory.newFolder(); + EventStore previousRun = new EventStore(directory, DEFAULT_CAPACITY, true, logging.logger); + assertTrue(previousRun.stage(serialized("damaged"))); + EventStore.Batch damaged = closeOne(previousRun); + previousRun.close(); + // Its name still counts it, so it is listed without being read; the damage is only found on sending. + File damagedFile = new File(directory, "ready-" + damaged.payloadId + "." + damaged.eventCount); + try (FileOutputStream out = new FileOutputStream(damagedFile)) { + out.write("not an event log".getBytes(StandardCharsets.UTF_8)); + } + + EventStore store = new EventStore(directory, DEFAULT_CAPACITY, true, logging.logger); + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, true); + try { + // Going online delivers what the previous run left, with no caller waiting on it. + eventProcessor.setOffline(false); + long deadline = System.currentTimeMillis() + 5_000; + while (damagedFile.exists() && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + assertFalse("the damaged batch was never delivered", damagedFile.exists()); + logging.assertErrorLogged("this version cannot read"); + + eventProcessor.recordCustomEvent(CONTEXT, "after", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + eventProcessor.recordCustomEvent(CONTEXT, "later", LDValue.ofNull(), null); + assertTrue("the loss was reported twice", awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void aBatchThatCannotBeReadIsKeptAndDeliveredOnceItCanBe() throws Exception { + AtomicBoolean failing = new AtomicBoolean(false); + EventStore store = new EventStore(eventsDirectory.newFolder(), DEFAULT_CAPACITY, true, + logging.logger) { + @Override + InputStream openForReading(File file) throws IOException { + if (failing.get()) { + // What a process out of file descriptors gets for a file that is there. + throw new FileNotFoundException(file + " (Too many open files)"); + } + return super.openForReading(file); + } + }; + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, true); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "kept", LDValue.ofNull(), null); + + failing.set(true); + assertFalse("a batch that could not be read was reported as delivered", + awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + logging.assertWarnLogged("Could not read stored events"); + + failing.set(false); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + List events = collectDelivered(server); + assertEquals(1, countEventsOfKind(events, "custom")); + assertEquals(LDValue.of("kept"), requireEventOfKind(events, "custom").get("key")); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void closeReleasesTheStoreAndSenderOnlyAfterTheLastDeliveryFinishes() throws Exception { + // Giving up on the wait must not turn into pulling the store or the HTTP client out from + // under the delivery we just decided not to wait for. Neither can be observed being closed + // from here, but the consequence can: a delivery that lost either would fail once released + // and leave its batch on disk instead of deleting it. + Semaphore letResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.all(Handlers.waitFor(letResponseFinish), + Handlers.status(202)))) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "stalled", LDValue.ofNull(), null); + + eventProcessor.close(); // returns on its budget with the post still stalled + letResponseFinish.release(Integer.MAX_VALUE); + + assertTrue("the stalled delivery never completed, so its batch is still on disk", + awaitNoPendingEvents(5, TimeUnit.SECONDS)); + } finally { + // Drained first, because the release above may already have happened and topping a + // semaphore up twice from Integer.MAX_VALUE overflows its permit count. + letResponseFinish.drainPermits(); + letResponseFinish.release(Integer.MAX_VALUE); + scheduler.shutdownNow(); + } + } + } + + private boolean awaitNoPendingEvents(long timeout, TimeUnit unit) throws Exception { + long deadlineNanos = System.nanoTime() + unit.toNanos(timeout); + do { + if (payloadsOnDisk().isEmpty()) { + return true; + } + Thread.sleep(25); + } while (System.nanoTime() < deadlineNanos); + return false; + } + + @Test + public void flushAfterCloseDoesNotDeliverThroughReleasedResources() throws Exception { + // close() hands the store and the analytics sender back on the delivery thread and then stops + // that thread taking work, so nothing it accepts afterwards can find them already released. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + eventProcessor.recordCustomEvent(CONTEXT, "before-close", LDValue.ofNull(), null); + // Leaves the buffer empty, so any request after this is one close should not have made. + flushAndCollect(eventProcessor, server); + + eventProcessor.close(); + + eventProcessor.recordCustomEvent(CONTEXT, "after-close", LDValue.ofNull(), null); + eventProcessor.flush(); + eventProcessor.blockingFlush(); + + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + } + } + + @Test + public void stalledDiagnosticPostDoesNotHoldUpAnalyticsDelivery() throws Exception { + // Diagnostics used to share the one thread that delivers analytics, so a post against a + // network that accepts connections and never answers stalled every flush behind it, and a + // buffer that is not being drained fills up and drops what the application asked to send. + CountDownLatch diagnosticStarted = new CountDownLatch(1); + CountDownLatch releaseDiagnostic = new CountDownLatch(1); + EventSender sender = new StubEventSender() { + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + diagnosticStarted.countDown(); + awaitQuietly(releaseDiagnostic, 5, TimeUnit.SECONDS); + return new Result(true, false, null); + } + }; + + // Analytics go out through AnalyticsEventSender rather than the injectable one, so the + // delivery has to be watched at the server rather than at the stub. + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(sender, server.getUri(), + makeDiagnosticStore(), NO_PERIODIC_FLUSH_MILLIS, 60_000, + DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, scheduler); + try { + // Coming online posts the diagnostic init event, which then never comes back. + eventProcessor.setOffline(false); + assertTrue("the diagnostic event was never posted", + diagnosticStarted.await(2, TimeUnit.SECONDS)); + + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.flush(); + + RequestInfo request = null; + try { + request = server.getRecorder().requireRequest(2, TimeUnit.SECONDS); + } catch (Exception timedOut) { + // Reported by the assertion below, which can say what it means. + } + assertNotNull("analytics delivery was stuck behind the diagnostic post", request); + assertTrue(request.getBody().contains("an-event")); + } finally { + releaseDiagnostic.countDown(); + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void diagnosticEventIsDroppedRatherThanQueuedBehindOneStillPosting() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + AtomicInteger posts = new AtomicInteger(); + EventSender sender = new StubEventSender() { + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + posts.incrementAndGet(); + firstStarted.countDown(); + awaitQuietly(releaseFirst, 5, TimeUnit.SECONDS); + return new Result(true, false, null); + } + }; + + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + // A diagnostic interval short enough that the periodic task fires repeatedly while the init + // event is still stuck on the posting thread. + DirectEventProcessor eventProcessor = makeEventProcessor(sender, makeDiagnosticStore(), + NO_PERIODIC_FLUSH_MILLIS, 20, scheduler); + try { + eventProcessor.setOffline(false); + assertTrue("the diagnostic event was never posted", + firstStarted.await(2, TimeUnit.SECONDS)); + + // Long enough for many periodic runs, every one of which has to be turned away rather + // than left on the posting thread's queue. Counting posts is not enough on its own: a + // queued one would not have started yet either, so the skips are what distinguishes + // being dropped from merely waiting. + Thread.sleep(300); + assertTrue("no diagnostic event was turned away, so they were queueing up instead", + countLogged("Skipped a diagnostic event") > 0); + assertEquals("more than one diagnostic event reached the sender", 1, posts.get()); + } finally { + releaseFirst.countDown(); + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + private static final URI UNUSED_EVENTS_URI = URI.create("https://events.example"); + + @Test + public void goingToTheBackgroundDefersADiagnosticPeriodRatherThanDestroyingIt() throws Exception { + // createEventAndReset hands the period's statistics back and clears them in the same call, so + // whoever consumes it owns it. The post reaches the diagnostics thread through a queue it may + // have to wait in, and the SDK can go offline or to the background in the meantime. If that is + // noticed after the event was built, the period is gone: nothing retries it, and the next + // event reports a window starting after the reset, so the statistics are not merely late but + // absent. + BlockingQueue posted = new LinkedBlockingQueue<>(); + EventSender sender = new StubEventSender() { + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + posted.add(LDValue.parse(new String(data, StandardCharsets.UTF_8))); + return new Result(true, false, null); + } + }; + + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + // Short enough that the periodic task fires while the diagnostics thread is held. + DirectEventProcessor eventProcessor = makeEventProcessor(sender, UNUSED_EVENTS_URI, + makeDiagnosticStore(), NO_PERIODIC_FLUSH_MILLIS, 20, + DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, scheduler, diagnosticExecutor); + try { + eventProcessor.setOffline(false); + assertEquals(LDValue.of("diagnostic-init"), + requirePosted(posted).get("kind")); + + // Something for the period to have in it. Capacity is what makes these countable, and the + // dropped count is carried by the periodic event and nothing else. Far past capacity so + // that drops happen whatever else is draining the buffer. + for (int i = 0; i < DEFAULT_CAPACITY * 10; i++) { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + } + logging.assertWarnLogged("Exceeded event queue capacity"); + + // Holds the diagnostics thread so the next periodic post has to queue behind it, which is + // the window the state can change in. + CountDownLatch releaseThread = new CountDownLatch(1); + diagnosticExecutor.submit(() -> awaitQuietly(releaseThread, 5, TimeUnit.SECONDS)); + // A later run being turned away is proof that an earlier one took the claim and is sitting + // on the queue, which is what we need before changing the state underneath it. + awaitLogged("Skipped a diagnostic event"); + + eventProcessor.setInBackground(true); + releaseThread.countDown(); + // The queued post now runs suspended. Nothing should reach the sender. + assertNull("a diagnostic event was posted from the background", + posted.poll(300, TimeUnit.MILLISECONDS)); + + eventProcessor.setInBackground(false); + LDValue statistics = requirePosted(posted); + + assertEquals(LDValue.of("diagnostic"), statistics.get("kind")); + // Nothing is recorded after the suspension, so every drop this could report happened + // before it. A destroyed period therefore reports exactly zero, which is what separates + // the two outcomes. The exact figure is not asserted because it is not the same on every + // tier: where events are staged to a store, reaching it frees capacity as we go. + assertTrue("the suspended period was destroyed rather than carried forward", + statistics.get("droppedEvents").longValue() > 0); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + private LDValue requirePosted(BlockingQueue posted) throws InterruptedException { + LDValue event = posted.poll(5, TimeUnit.SECONDS); + assertNotNull("no diagnostic event was posted", event); + return event; + } + + /** + * Goes to the background and back, leaving the processor at the point where it decides whether + * to send a second init event. + *

+ * Two things have to be true for that decision to be reached, and only one of them is under the + * test's control. Going to the foreground has to be a real transition, because setInBackground + * returns immediately when the value is unchanged. And the diagnostics thread has to be free: + * the first init event holds a claim on it until its post returns, which is after the request + * reaches the server, so a transition right after the request arrives is turned away before it + * gets anywhere near the decision. Being turned away is logged, which is what this waits out. + */ + private void returnToTheForegroundWithTheDiagnosticsThreadFree(EventProcessor eventProcessor) + throws InterruptedException { + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + long skipsBefore = countLogged("Skipped a diagnostic event"); + eventProcessor.setInBackground(true); + eventProcessor.setInBackground(false); + // Logged by the claim itself, on this thread, so it is already there if it happened. + if (countLogged("Skipped a diagnostic event") == skipsBefore) { + return; + } + Thread.sleep(10); + } + throw new AssertionError("the diagnostics thread never came free"); + } + + private void awaitLogged(String messageSubstring) throws InterruptedException { + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (countLogged(messageSubstring) > 0) { + return; + } + Thread.sleep(10); + } + throw new AssertionError("never logged: " + messageSubstring); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + long flushIntervalMillis, + ScheduledExecutorService scheduler) { + return makeEventProcessor(diagnosticSender, UNUSED_EVENTS_URI, null, flushIntervalMillis, + 60_000, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, scheduler); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + ScheduledExecutorService scheduler) { + return makeEventProcessor(diagnosticSender, UNUSED_EVENTS_URI, diagnosticStore, flushIntervalMillis, + 60_000, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, scheduler); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + ScheduledExecutorService scheduler) { + return makeEventProcessor(diagnosticSender, UNUSED_EVENTS_URI, diagnosticStore, flushIntervalMillis, + diagnosticIntervalMillis, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, + scheduler); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + URI eventsUri, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + long closeBudgetMillis, + ScheduledExecutorService scheduler) { + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + return makeEventProcessor(diagnosticSender, eventsUri, diagnosticStore, flushIntervalMillis, + diagnosticIntervalMillis, closeBudgetMillis, scheduler, diagnosticExecutor); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + URI eventsUri, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + long closeBudgetMillis, + ScheduledExecutorService scheduler, + ExecutorService diagnosticExecutor) { + return makeEventProcessor(diagnosticSender, eventsUri, diagnosticStore, flushIntervalMillis, + diagnosticIntervalMillis, closeBudgetMillis, scheduler, diagnosticExecutor, + EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, + DEFAULT_CAPACITY, true, logging.logger), + DEFAULT_CAPACITY); + } + + private DirectEventProcessor makeEventProcessor(EventStore store, int capacity, + ScheduledExecutorService scheduler) { + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + return makeEventProcessor(new StubEventSender(), UNUSED_EVENTS_URI, null, + NO_PERIODIC_FLUSH_MILLIS, 60_000, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, capacity); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + URI eventsUri, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + long closeBudgetMillis, + ScheduledExecutorService scheduler, + ExecutorService diagnosticExecutor, + EventStore store, + int capacity) { + return makeEventProcessor(diagnosticSender, eventsUri, diagnosticStore, flushIntervalMillis, + diagnosticIntervalMillis, closeBudgetMillis, scheduler, diagnosticExecutor, store, + capacity, false); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + URI eventsUri, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + long closeBudgetMillis, + ScheduledExecutorService scheduler, + ExecutorService diagnosticExecutor, + EventStore store, + int capacity, + boolean commitOnCallerThread) { + ExecutorService commitExecutor = EventUtil.makeEventsCommitExecutor(); + diagnosticExecutors.add(commitExecutor); + lastCommitExecutor = commitExecutor; + return new DirectEventProcessor( + new OutboundEventBuffer(false, Collections.emptyList(), true, capacity, + logging.logger), + store, + diagnosticSender, + new AnalyticsEventSender(LDUtil.makeHttpProperties( + new HttpConfiguration(2000, Collections.emptyMap(), null, false)), + logging.logger), + new DeliveryRetryState(flushIntervalMillis), + eventsUri, + diagnosticStore, + capacity, + commitOnCallerThread, + flushIntervalMillis, + diagnosticIntervalMillis, + closeBudgetMillis, + false, + true, // initiallyOffline, as the SDK builds it + scheduler, + commitExecutor, + diagnosticExecutor, + logging.logger); + } + + private long countLogged(String messageSubstring) { + long count = 0; + for (String message : logging.logCapture.getMessageStrings()) { + if (message.contains(messageSubstring)) { + count++; + } + } + return count; + } + + private DiagnosticStore makeDiagnosticStore() { + return new DiagnosticStore(new DiagnosticStore.SdkDiagnosticParams(MOBILE_KEY, + "android-client-sdk", "0.0.0", "Android", null, Collections.emptyMap(), null)); + } + + /** + * Flushes and waits for the outcome the way {@code LDClient.flushAndWait} does, which is the + * only place a timeout belongs. + */ + private static boolean awaitFlush(EventProcessor eventProcessor, long timeout, TimeUnit unit) + throws Exception { + try { + return Boolean.TRUE.equals(eventProcessor.flushAsync().get(timeout, unit)); + } catch (TimeoutException e) { + return false; + } + } + + private static void awaitQuietly(CountDownLatch latch, long timeout, TimeUnit unit) { + try { + latch.await(timeout, unit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** Accepts everything, so that a test overrides only the one method it is about. */ + private static class StubEventSender implements EventSender { + @Override + public Result sendAnalyticsEvents(byte[] data, int eventCount, URI eventsBaseUri) { + return new Result(true, false, null); + } + + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + return new Result(true, false, null); + } + + @Override + public void close() {} + } + + private void recordEvaluation(EventProcessor eventProcessor, boolean requireFullEvent, + Long debugEventsUntilDate) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, FLAG_VALUE, + null, DEFAULT_VALUE, requireFullEvent, debugEventsUntilDate); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorWithoutPersistenceTest.java new file mode 100644 index 000000000..744440aee --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorWithoutPersistenceTest.java @@ -0,0 +1,11 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.sdk.android.integrations.EventPersistence; + +/** {@link DirectEventProcessorTest} with persistence off, which is what an application gets by default. */ +public class DirectEventProcessorWithoutPersistenceTest extends DirectEventProcessorTest { + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventLogFormatTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventLogFormatTest.java new file mode 100644 index 000000000..c78ecbd1c --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventLogFormatTest.java @@ -0,0 +1,180 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.LDValue; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * The frame checksum: a frame the storage changed after it was written ends the walk, as a torn tail does. + */ +public class EventLogFormatTest { + private static byte[] event(String key) { + return ("{\"kind\":\"custom\",\"key\":\"" + key + "\"}").getBytes(StandardCharsets.UTF_8); + } + + private static byte[] log(String... keys) { + ByteArrayOutputStream frames = new ByteArrayOutputStream(); + for (String key : keys) { + EventLogFormat.writeFrame(frames, event(key)); + } + return EventLogFormat.withFileHeader(frames.toByteArray()); + } + + /** Where each frame of a log written by this version starts. */ + private static List frameOffsets(byte[] log) { + List offsets = new ArrayList<>(); + int cursor = EventLogFormat.FILE_HEADER_SIZE; + while (cursor + EventLogFormat.FRAME_HEADER_SIZE <= log.length) { + offsets.add(cursor); + cursor += EventLogFormat.FRAME_HEADER_SIZE + EventLogFormat.readInt32(log, cursor + 2); + } + return offsets; + } + + private static List keys(byte[] log) { + List keys = new ArrayList<>(); + for (byte[] payload : EventLogFormat.eventPayloads(log)) { + keys.add(LDValue.parse(new String(payload, StandardCharsets.UTF_8)).get("key").stringValue()); + } + return keys; + } + + @Test + public void writesTheFrameTheIosSdkWrites() { + // The checksum is zlib's CRC-32, so these bytes are what the iOS SDK writes for the same event too. + ByteArrayOutputStream frame = new ByteArrayOutputStream(); + EventLogFormat.writeFrame(frame, "{\"kind\":\"custom\",\"key\":\"k\"}".getBytes(StandardCharsets.UTF_8)); + + byte[] header = Arrays.copyOf(frame.toByteArray(), EventLogFormat.FRAME_HEADER_SIZE); + assertArrayEquals(new byte[] {0x00, 0x01, 0x00, 0x00, 0x00, 0x1B, (byte) 0xCB, 0x2B, 0x5E, 0x6F}, header); + } + + @Test + public void readsBackEveryFrameItWrote() { + byte[] log = log("event-0", "event-1", "event-2"); + + assertEquals(Arrays.asList("event-0", "event-1", "event-2"), keys(log)); + assertFalse(EventLogFormat.isDamaged(log)); + } + + @Test + public void keepsEveryEventBeforeAPayloadWithAFlippedBit() { + byte[] log = log("event-0", "event-1", "event-2"); + int second = frameOffsets(log).get(1); + log[second + EventLogFormat.FRAME_HEADER_SIZE + 3] ^= 0x04; + + assertEquals(Arrays.asList("event-0"), keys(log)); + assertEquals(1, EventLogFormat.eventCount(log)); + assertTrue(EventLogFormat.isDamaged(log)); + } + + @Test + public void keepsEveryEventBeforeAZeroFilledPayload() { + byte[] log = log("event-0", "event-1", "event-2"); + int second = frameOffsets(log).get(1); + int size = EventLogFormat.readInt32(log, second + 2); + Arrays.fill(log, second + EventLogFormat.FRAME_HEADER_SIZE, + second + EventLogFormat.FRAME_HEADER_SIZE + size, (byte) 0); + + assertEquals(Arrays.asList("event-0"), keys(log)); + } + + @Test + public void keepsEveryEventBeforeASizeThatStillFitsInTheFile() { + // A size that is merely wrong rather than impossible passes every bound a torn tail is caught by, + // and would have the reader start the next frame partway through this one's payload. + byte[] log = log("event-0", "event-1", "event-2"); + int second = frameOffsets(log).get(1); + log[second + 5] -= 1; + + assertEquals(Arrays.asList("event-0"), keys(log)); + } + + @Test + public void assemblesTheEventsExactlyAsTheyWereRecordedIntoOneArray() { + String expected = "[" + new String(event("event-0"), StandardCharsets.UTF_8) + "," + + new String(event("event-1"), StandardCharsets.UTF_8) + "]"; + + assertArrayEquals(expected.getBytes(StandardCharsets.UTF_8), + EventLogFormat.assembleBody(log("event-0", "event-1"))); + assertArrayEquals(("[" + new String(event("only"), StandardCharsets.UTF_8) + "]") + .getBytes(StandardCharsets.UTF_8), EventLogFormat.assembleBody(log("only"))); + } + + @Test + public void assemblesThePayloadAtTheStartOfTheLogsOwnArray() { + byte[] log = log("event-0", "event-1", "event-2"); + byte[] expected = EventLogFormat.assembleBody(log); + + EventLogFormat.Payload payload = EventLogFormat.assemblePayloadInPlace(log, Integer.MAX_VALUE); + + assertSame(log, payload.bytes); + assertArrayEquals(expected, Arrays.copyOf(log, payload.length)); + assertEquals(3, payload.eventCount); + } + + @Test + public void saysHowManyEventsMadeItIntoThePayloadOfALogDamagedAfterItsFirstFrame() { + byte[] log = log("event-0", "event-1", "event-2"); + log[log.length - 1] ^= 0x01; + + EventLogFormat.Payload payload = EventLogFormat.assemblePayloadInPlace(log, Integer.MAX_VALUE); + + assertEquals(2, payload.eventCount); + assertEquals(LDValue.parse("[" + new String(event("event-0"), StandardCharsets.UTF_8) + "," + + new String(event("event-1"), StandardCharsets.UTF_8) + "]"), + LDValue.parse(new String(payload.toByteArray(), StandardCharsets.UTF_8))); + } + + @Test + public void leavesALogWithNoEventsAsItWasWhenAssemblingInPlace() { + byte[] log = log("damaged"); + log[EventLogFormat.FILE_HEADER_SIZE + EventLogFormat.FRAME_HEADER_SIZE] ^= 0x01; + byte[] before = log.clone(); + + assertNull(EventLogFormat.assemblePayloadInPlace(log, Integer.MAX_VALUE)); + assertArrayEquals(before, log); + } + + @Test + public void assemblesNoMoreEventsThanItIsAskedFor() { + byte[] log = log("event-0", "event-1", "event-2"); + + assertEquals(LDValue.parse("[" + new String(event("event-0"), StandardCharsets.UTF_8) + "," + + new String(event("event-1"), StandardCharsets.UTF_8) + "]"), + LDValue.parse(new String(EventLogFormat.assembleBody(log, 2), StandardCharsets.UTF_8))); + assertEquals(null, EventLogFormat.assembleBody(log, 0)); + } + + @Test + public void hasNoBodyWhenTheFirstFrameIsDamaged() { + byte[] log = log("damaged"); + log[EventLogFormat.FILE_HEADER_SIZE + EventLogFormat.FRAME_HEADER_SIZE] ^= 0x01; + + assertEquals(null, EventLogFormat.assembleBody(log)); + assertEquals(0, EventLogFormat.eventCount(log)); + assertTrue(EventLogFormat.isDamaged(log)); + } + + @Test + public void refusesAVersionItDoesNotKnow() { + byte[] log = log("event-0"); + log[5] = 2; + + assertEquals(-1, EventLogFormat.eventCount(log)); + assertFalse(EventLogFormat.isDamaged(log)); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java new file mode 100644 index 000000000..7d1fd2d30 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java @@ -0,0 +1,623 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assume.assumeTrue; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.AttributeRef; +import com.launchdarkly.sdk.ContextKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.IntConsumer; + +/** + * The Android counterpart of the iOS {@code EventPersistenceBenchmark}: what it costs to record an + * analytics event, so the encoding research can be argued from measurements on this platform rather + * than from the ones taken on iOS. + *

+ * This exists because the two platforms are not obviously comparable. iOS encodes with {@code Codable}, + * Android encodes with {@code EventOutputFormatter} in java-sdk-internal, and a JIT with escape + * analysis behaves nothing like ahead-of-time compiled Swift. The iOS finding -- that the cost is in + * the serializer's per-value machinery rather than in walking and redacting the context -- is a claim + * about {@code Codable}, and nothing about it transfers here by assumption. + *

+ * Skipped unless {@code LD_EVENT_BENCH=1}, because nothing here asserts anything a regression would + * trip and the timing loops take far longer than a unit test should: + * 

+ *   LD_EVENT_BENCH=1 ./gradlew :launchdarkly-android-client-sdk:testDebugUnitTest \
+ *       --tests '*EventPersistenceBenchmark*'
+ *
+ *

+ * Read the numbers as ratios, not absolutes. Three reasons. This runs on a desktop JVM rather + * than on ART on a device, which is the same caveat the iOS figures carry about running on a Mac. It + * is a hand-rolled timing loop rather than JMH, so it defends against the two failure modes that + * matter -- an unwarmed JIT and dead-code elimination -- and against nothing else. And the JVM's + * escape analysis can delete allocations here that a real caller would pay for, which flatters any + * row whose result is discarded. + */ +public class EventPersistenceBenchmark { + private static final String FLAG_KEY = "benchmark-flag"; + private static final int FLAG_VERSION = 7; + private static final int VARIATION = 1; + + /** + * Fixed rather than {@code System.currentTimeMillis()}, so that repeated runs encode identical + * bytes and the clock call is not charged to the encoder. The event-construction row below covers + * what the real path pays around it. + */ + private static final long CREATION_DATE = 1_700_000_000_000L; + + /** + * One already serialized event, byte for byte what the iOS benchmark stages, so the store figures on the + * two platforms are measuring the same amount of work reaching the same kind of file. + */ + private static final String SERIALIZED_EVENT = + "{\"kind\":\"feature\",\"key\":\"benchmark-flag\",\"value\":true,\"default\":false," + + "\"variation\":1,\"version\":7,\"creationDate\":1740000000000}"; + + /** How many times each timed loop is repeated; the fastest round is reported. */ + private static final int ROUNDS = 5; + + /** + * The sizes the commit-point figures are swept over. The same three are used by the iOS benchmark, so + * the two platforms' tables line up row for row. + */ + private static final int[] BULK_SIZES = {100, 1_000, 10_000}; + + /** + * Consumes results so that the JIT cannot prove the work is unused and delete it. Never read for + * its value. + */ + private static long blackhole; + + /** + * Holds an allocation so that escape analysis cannot scalar-replace it. Assigning to a static field + * is what makes the object genuinely exist, which is the difference between measuring an allocation + * and measuring nothing at all. + */ + private static Object escaped; + + @Rule + public TemporaryFolder eventsDirectory = new TemporaryFolder(); + + private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); + + @Before + public void requireBenchmarking() { + assumeTrue("Set LD_EVENT_BENCH=1 to measure event recording", + "1".equals(System.getenv("LD_EVENT_BENCH"))); + } + + /** + * M1 and M2 for Android: where the time goes when one feature event is serialized, across the + * context shapes and privacy settings that change how much work redaction has to do. + *

+ * The shape of the result is the interesting part. If the cost is dominated by walking and + * redacting the context, it should grow with the number of attributes and grow again when private + * attributes force the filtering path. If it is dominated by the serializer's fixed per-event + * machinery, it should be far flatter than the attribute counts suggest. + */ + @Test + public void encodingCostByContextShape() { + List building = new ArrayList<>(); + List serializing = new ArrayList<>(); + + for (ContextShape shape : ContextShape.values()) { + final LDContext context = makeContext(shape, "benchmark-key"); + + building.add(new Measurement(shape.label, measure(200_000, i -> { + escaped = featureEvent(context, true); + }))); + + for (PrivacyShape privacy : privacyShapes()) { + final OutboundEventBuffer buffer = makeBuffer(privacy); + + byte[] sample = buffer.serialize(featureEvent(context, true)); + assertNotNull("the corpus produced an event that will not serialize", sample); + + double nanos = measure(50_000, i -> { + byte[] out = buffer.serialize(featureEvent(context, true)); + blackhole += out == null ? 0 : out.length; + }); + serializing.add(new Measurement( + shape.label + ", " + privacy.name + ", " + sample.length + " bytes", nanos)); + } + } + + report("building one feature event, no serialization", building); + report("serializing one feature event", serializing); + } + + /** + * The figures a customer actually pays, mirroring what {@link DirectEventProcessor} does per call. + *

+ * The three rows are the three paths through {@code recordEvaluationEvent} and {@code recordCustomEvent}: + * an evaluation of an untracked flag is a counter increment; an evaluation of a tracked flag also + * serializes and stages a full event; and a {@code track} is a commit point, which additionally turns + * the counters into summary events and writes everything to the log. + *

+ * The sequence is reproduced here rather than driven through the processor itself, which would need + * an HTTP server, a scheduler and a {@code ClientContext} whose costs would swamp what is being + * measured. It is kept deliberately close to the original; if the processor changes, this should too. + */ + @Test + public void recordingCostPerCall() throws IOException { + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + final EventStore store = EventStore.create(eventsDirectory.getRoot(), "benchmark-key", + Integer.MAX_VALUE, true, logger); + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + + try { + List results = new ArrayList<>(); + + results.add(new Measurement("evaluation, summary only", measure(200_000, i -> { + buffer.summarize(featureEvent(context, false)); + }))); + + results.add(new Measurement("evaluation, full event", measure(50_000, i -> { + Event.FeatureRequest event = featureEvent(context, true); + buffer.summarize(event); + byte[] out = buffer.serialize(event); + if (out != null) { + store.stage(out); + blackhole += out.length; + } + }))); + + results.add(new Measurement("evaluation + track (commit point)", measure(20_000, i -> { + buffer.summarize(featureEvent(context, false)); + byte[] out = buffer.serialize( + new Event.Custom(CREATION_DATE, "benchmark-event", context, LDValue.ofNull(), null)); + if (out != null) { + store.stage(out); + blackhole += out.length; + } + for (byte[] summary : buffer.serializeSummariesAndReset()) { + store.stage(summary, true); + blackhole += summary.length; + } + store.commit(); + }))); + + report("recording, per call", results); + } finally { + store.close(); + } + } + + /** + * What a commit point costs when it goes badly, rather than on average. + *

+ * {@code track} and {@code identify} commit on the caller's thread by design -- that is the durable + * barrier -- so this is the one figure here an application can observe as a stall. The mean is the number + * that gets quoted; the tail is the number a user notices. + *

+ * The counterpart iOS test runs this against a SQLite store as well, where the tail is the point: a + * database in WAL mode folds its write-ahead log back into the file every thousand pages, on whichever + * thread happens to be committing, so one unlucky caller pays for all of them. An append has no such + * thing, and these rows are what that claim is measured against. + *

+ * Two rows per size. {@code track} commits where it stands, so its tail is the write. The other row + * encodes and stages without committing, and the store never writes on its own, so its tail is the + * encode and the copy; the gap between the two is what the write itself costs. + *

+ * p99.9 only separates from the worst at ten thousand samples. Below that the index falls on the + * last sample and the two columns are the same number by construction. They are left side by side so the + * table lines up across the sweep, and so the size at which they part is visible rather than asserted. + */ + @Test + public void commitPointOutliers() throws IOException { + final byte[] serialized = SERIALIZED_EVENT.getBytes(StandardCharsets.UTF_8); + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + + // A store of its own for each size and each row, so that a log inherited from an earlier measurement + // does not charge the next one for the file it had already grown. + for (int calls : BULK_SIZES) { + final EventStore committing = EventStore.create(eventsDirectory.newFolder(), "benchmark-key", + Integer.MAX_VALUE, true, logger); + final EventStore staging = EventStore.create(eventsDirectory.newFolder(), "benchmark-key", + Integer.MAX_VALUE, true, logger); + try { + List results = new ArrayList<>(); + + results.add(new DistributionMeasurement("track, commits on this thread", + measureDistribution(calls, i -> { + committing.stage(serialized); + committing.commit(); + }))); + + results.add(new DistributionMeasurement("evaluation, staged only", + measureDistribution(calls, i -> { + Event.FeatureRequest event = featureEvent(context, true); + buffer.summarize(event); + byte[] out = buffer.serialize(event); + if (out != null) { + staging.stage(out); + blackhole += out.length; + } + }))); + + reportDistribution("one call, " + count(calls) + " of them, on the caller's thread", results); + } finally { + committing.close(); + staging.close(); + } + } + } + + /** + * Whether O2 -- caching the encoded context and reusing it while the context has not changed -- + * could work here, which comes down to what the cache key would cost to check. + *

+ * Android differs from iOS in a way that matters. {@code LDContext} is an immutable object passed by + * reference, and nothing on the event path copies it, so an application that holds one context hands + * the same instance to every event and {@code equals} can settle on reference identity. iOS had to + * reach that fast path by not rebuilding a value type. The row that decides the question is the other + * one: an application that rebuilds its context between evaluations produces an equal object sharing + * nothing, and that comparison is the price of a cache hit in the worst case. + */ + @Test + public void contextComparisonCost() { + List comparisons = new ArrayList<>(); + List hashing = new ArrayList<>(); + + for (ContextShape shape : ContextShape.values()) { + final LDContext context = makeContext(shape, "benchmark-key"); + final LDContext sameInstance = context; + final LDContext rebuilt = makeContext(shape, "benchmark-key"); + + // Both must be equal, or the figures below measure an early exit rather than a comparison. + assertEquals(shape.label, context, sameInstance); + assertEquals(shape.label, context, rebuilt); + + comparisons.add(new Measurement("same instance, " + shape.label, measure(500_000, i -> { + blackhole += context.equals(sameInstance) ? 1 : 0; + }))); + comparisons.add(new Measurement("independently built, " + shape.label, measure(200_000, i -> { + blackhole += context.equals(rebuilt) ? 1 : 0; + }))); + hashing.add(new Measurement(shape.label, measure(500_000, i -> { + blackhole += context.hashCode(); + }))); + } + + report("comparing two equal contexts", comparisons); + report("hashing a context", hashing); + } + + /** + * What the deferred half of the bill costs: summaries are not serialized per evaluation but at + * commit points and deliveries, and the size of that payment depends on how many distinct flags the + * counters have accumulated. + *

+ * Counting and serializing are reported separately, because only the second is deferred -- the + * counter increment is already paid on every evaluation. + */ + @Test + public void summarySerializationCost() { + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + List results = new ArrayList<>(); + + for (final int flagCount : new int[] {1, 10, 50}) { + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + + double countingOnly = measure(20_000, i -> { + for (int flag = 0; flag < flagCount; flag++) { + buffer.summarize(summaryEvent(context, flag)); + } + }); + // Drained, so the counters left behind above do not inflate the first serialized round. + buffer.serializeSummariesAndReset(); + + double countingAndSerializing = measure(20_000, i -> { + for (int flag = 0; flag < flagCount; flag++) { + buffer.summarize(summaryEvent(context, flag)); + } + for (byte[] summary : buffer.serializeSummariesAndReset()) { + blackhole += summary.length; + } + }); + + results.add(new Measurement(flagCount + " flags: counting only", countingOnly)); + results.add(new Measurement(flagCount + " flags: counting + serializing", + countingAndSerializing)); + results.add(new Measurement(flagCount + " flags: serializing alone", + countingAndSerializing - countingOnly)); + } + + report("turning counters into a summary event", results); + } + + // MARK: Corpus + + /** + * The same five shapes the iOS benchmark uses, so the two sets of numbers describe the same + * contexts. + */ + private enum ContextShape { + KEY_ONLY("key only"), + STUB("stub, 9 attributes"), + WIDE("20 flat attributes"), + NESTED("nested attributes"), + MULTI("multi-context"); + + final String label; + + ContextShape(String label) { + this.label = label; + } + } + + private static LDContext makeContext(ContextShape shape, String key) { + switch (shape) { + case KEY_ONLY: + return LDContext.create(key); + case STUB: + return stubContext(key); + case WIDE: { + com.launchdarkly.sdk.ContextBuilder builder = LDContext.builder(key).name("Wide"); + for (int i = 0; i < 20; i++) { + builder.set("attribute" + i, "value" + i); + } + return builder.build(); + } + case NESTED: + return LDContext.builder(key) + .name("Nested") + .set("address", LDValue.buildObject() + .put("street", "1 Main St") + .put("city", "Springfield") + .put("geo", LDValue.buildObject().put("lat", 1.5).put("lon", -2.5).build()) + .build()) + .set("tags", LDValue.buildArray() + .add("a").add("b").add("c").add("d").add("e").build()) + .build(); + case MULTI: + return LDContext.createMulti( + stubContext(key), + LDContext.builder(ContextKind.of("device"), "device-" + key) + .set("os", LDValue.buildObject() + .put("name", "Android").put("version", 14).build()) + .build()); + default: + throw new IllegalArgumentException(shape.name()); + } + } + + /** Nine attributes, matching the shape iOS calls its stub context. */ + private static LDContext stubContext(String key) { + return LDContext.builder(key) + .name("Stub Context") + .set("firstName", "Ada") + .set("lastName", "Lovelace") + .set("email", "ada@example.com") + .set("country", "US") + .set("age", 36) + .set("score", 12.5) + .set("verified", true) + .set("plan", "enterprise") + .build(); + } + + /** The privacy settings that change what redaction has to do. */ + private static final class PrivacyShape { + final String name; + final boolean allAttributesPrivate; + final Collection privateAttributes; + + PrivacyShape(String name, boolean allAttributesPrivate, Collection privateAttributes) { + this.name = name; + this.allAttributesPrivate = allAttributesPrivate; + this.privateAttributes = privateAttributes; + } + } + + private static PrivacyShape noRedaction() { + return new PrivacyShape("no redaction", false, Collections.emptyList()); + } + + private static List privacyShapes() { + return Arrays.asList( + noRedaction(), + new PrivacyShape("1 global private", false, + Collections.singletonList(AttributeRef.fromLiteral("email"))), + new PrivacyShape("all private", true, Collections.emptyList())); + } + + private OutboundEventBuffer makeBuffer(PrivacyShape privacy) { + // Unbounded cardinality, so a benchmark that sweeps context counts measures the encoder + // rather than the point at which the summarizer starts turning contexts away. + return new OutboundEventBuffer(privacy.allAttributesPrivate, privacy.privateAttributes, + true, Integer.MAX_VALUE, logger); + } + + private static Event.FeatureRequest featureEvent(LDContext context, boolean requireFullEvent) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY, context, FLAG_VERSION, VARIATION, + LDValue.of(true), LDValue.of(false), null, null, requireFullEvent, null, false); + } + + private static Event.FeatureRequest summaryEvent(LDContext context, int flagIndex) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY + "-" + flagIndex, context, + FLAG_VERSION, VARIATION, LDValue.of(true), LDValue.of(false), null, null, false, null, + false); + } + + // MARK: Harness + + /** + * Times {@code body}, returning nanoseconds per iteration. + *

+ * Warmed first, because a JVM measured cold reports the interpreter rather than the compiled code. + * Then run several times with the fastest round reported: a garbage collection or a descheduled + * thread can only ever make a round slower, so the minimum is the closest estimate of the work + * itself. That is also why these numbers should not be read as what a busy device would see. + */ + private double measure(int iterations, IntConsumer body) { + int warmup = Math.max(10_000, iterations / 4); + for (int i = 0; i < warmup; i++) { + body.accept(i); + } + + double best = Double.MAX_VALUE; + for (int round = 0; round < ROUNDS; round++) { + long start = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + body.accept(i); + } + long elapsed = System.nanoTime() - start; + best = Math.min(best, (double) elapsed / iterations); + } + return best; + } + + /** How long one call took, across many of them, when the tail is the point. */ + private static final class Distribution { + final double median; + final double mean; + final double p999; + final double worst; + + Distribution(double median, double mean, double p999, double worst) { + this.median = median; + this.mean = mean; + this.p999 = p999; + this.worst = worst; + } + } + + /** + * Times every call rather than the enclosing loop, so the tail survives to be reported. + *

+ * One round, and the median rather than the fastest. Every other figure here reports the cheapest of + * several rounds, on the reasoning that noise can only make a round slower -- but here an outlier is the + * measurement, and that aggregation would throw away the answer. + */ + private Distribution measureDistribution(int iterations, IntConsumer body) { + int warmup = Math.max(10_000, iterations / 4); + for (int i = 0; i < warmup; i++) { + body.accept(i); + } + + long[] samples = new long[iterations]; + for (int i = 0; i < iterations; i++) { + long start = System.nanoTime(); + body.accept(i); + samples[i] = System.nanoTime() - start; + } + + long total = 0; + for (long sample : samples) { + total += sample; + } + Arrays.sort(samples); + return new Distribution(samples[iterations / 2], + (double) total / iterations, + samples[Math.min(iterations - 1, (int) ((long) iterations * 999 / 1000))], + samples[iterations - 1]); + } + + private static final class DistributionMeasurement { + final String name; + final Distribution distribution; + + DistributionMeasurement(String name, Distribution distribution) { + this.name = name; + this.distribution = distribution; + } + } + + private static final class Measurement { + final String name; + final double nanosPerOp; + + Measurement(String name, double nanosPerOp) { + this.name = name; + this.nanosPerOp = nanosPerOp; + } + } + + private static void report(String title, List results) { + int width = 0; + for (Measurement result : results) { + width = Math.max(width, result.name.length()); + } + StringBuilder out = new StringBuilder("\n").append(title).append('\n'); + for (Measurement result : results) { + out.append(" ").append(pad(result.name, width)).append(" ") + .append(format(result.nanosPerOp)).append('\n'); + } + System.out.print(out); + System.out.flush(); + } + + private static void reportDistribution(String title, List results) { + int width = 0; + for (DistributionMeasurement result : results) { + width = Math.max(width, result.name.length()); + } + StringBuilder out = new StringBuilder("\n").append(title).append('\n') + .append(" ").append(pad("", width)) + .append(" ").append(column("median")) + .append(" ").append(column("mean")) + .append(" ").append(column("p99.9")) + .append(" ").append(column("worst")).append('\n'); + for (DistributionMeasurement result : results) { + out.append(" ").append(pad(result.name, width)) + .append(" ").append(format(result.distribution.median)) + .append(" ").append(format(result.distribution.mean)) + .append(" ").append(format(result.distribution.p999)) + .append(" ").append(format(result.distribution.worst)).append('\n'); + } + System.out.print(out); + System.out.flush(); + } + + /** Renders a count with thousands separators, so a table title reads as a quantity, not a code literal. */ + private static String count(int value) { + return String.format("%,d", value); + } + + /** Right-aligns a column heading over the fixed width {@link #format} produces. */ + private static String column(String title) { + StringBuilder padded = new StringBuilder(); + for (int i = title.length(); i < 11; i++) { + padded.append(' '); + } + return padded.append(title).toString(); + } + + private static String pad(String value, int width) { + StringBuilder padded = new StringBuilder(value); + while (padded.length() < width) { + padded.append(' '); + } + return padded.toString(); + } + + private static String format(double nanoseconds) { + if (nanoseconds >= 1_000_000) { + return String.format("%8.2f ms", nanoseconds / 1_000_000); + } + if (nanoseconds >= 1_000) { + return String.format("%8.2f \u00b5s", nanoseconds / 1_000); + } + return String.format("%8.1f ns", nanoseconds); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingTest.java new file mode 100644 index 000000000..8d9ee9e0d --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingTest.java @@ -0,0 +1,339 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.testhelpers.httptest.HttpServer; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * Tests for what the event processor is allowed to drop when flag evaluations arrive faster than + * it can consume them. + *

+ * The processor used to be a thin wrapper around java-sdk-internal's + * {@code DefaultEventProcessor}, which buffers in two stages: every recorded event is first + * offered to a bounded "inbox" queue, and only after a background dispatcher thread picks it up is + * it either summarized or placed in the outbox that a flush actually drains. The inbox is sized to + * the configured event capacity, and it carries the processor's own FLUSH and SHUTDOWN control + * messages as well as events. So a burst of evaluations could fill it and cause silent drops even + * though those evaluations were only ever going to contribute a counter to a summary event, and + * even though unrelated custom events were competing for the same slots. + *

+ * These tests pin the behavior we want instead: summarization happens when the evaluation is + * recorded, so an evaluation never occupies a queue slot, and capacity only ever limits + * full-fidelity events. With summarization ahead of buffering there is no queue in this path, so + * the counts asserted below are exact rather than approximate. + */ +public class EventProcessorBufferingTest extends EventProcessorTestBase { + private static final String FLAG_KEY = "burst-flag"; + private static final int FLAG_VERSION = 10; + private static final int VARIATION = 1; + private static final LDValue FLAG_VALUE = LDValue.of(true); + private static final LDValue DEFAULT_VALUE = LDValue.of(false); + + // Deliberately far smaller than the number of evaluations each test records, so that any + // capacity-limited queue in the evaluation path would be guaranteed to overflow. + private static final int CAPACITY = 100; + + // For the tests that fill the summarizer to its context limit. Each counted context becomes its + // own summary event, so the flush is roughly this many summaries; at CAPACITY that came to about + // 28 KB, past what the test server reliably records with its single unlooped read of the body. + private static final int CONTEXT_LIMIT = 10; + + private static final int BURST_THREADS = 4; + private static final int EVALUATIONS_PER_THREAD = 25_000; + private static final int TOTAL_EVALUATIONS = BURST_THREADS * EVALUATIONS_PER_THREAD; + + // How many full-fidelity events the interleaving tests space through the burst. Comfortably + // under CAPACITY, so the buffer is never legitimately entitled to drop any of them. + private static final int INTERLEAVED_EVENTS = 50; + + @Test + public void summaryCountsSurviveEvaluationBurst() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CAPACITY); + try { + new EvaluationBurst(eventProcessor).join(); + + List events = flushAndCollect(eventProcessor, server); + + // Every evaluation contributes to the summary regardless of capacity, because a + // summary counter is not a buffered event. + assertEquals(TOTAL_EVALUATIONS, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void customEventsSurviveEvaluationBurst() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CAPACITY); + try { + // Record custom events spaced evenly through the burst. These are full-fidelity + // events and there are far fewer of them than the configured capacity, so none of + // them may be lost no matter how many evaluations are happening alongside them. + EvaluationBurst burst = new EvaluationBurst(eventProcessor); + for (int i = 1; i <= INTERLEAVED_EVENTS; i++) { + burst.awaitFraction(i, INTERLEAVED_EVENTS + 1); + eventProcessor.recordCustomEvent(CONTEXT, "burst-custom", LDValue.ofNull(), null); + } + burst.join(); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(INTERLEAVED_EVENTS, countEventsOfKind(events, "custom")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void fullFidelityEvaluationEventsSurviveEvaluationBurst() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CAPACITY); + try { + // A flag with trackEvents on produces a full feature event in addition to its + // summary counter. Evaluations of untracked flags must not crowd these out. + EvaluationBurst burst = new EvaluationBurst(eventProcessor); + for (int i = 1; i <= INTERLEAVED_EVENTS; i++) { + burst.awaitFraction(i, INTERLEAVED_EVENTS + 1); + eventProcessor.recordEvaluationEvent(CONTEXT, "tracked-flag", FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, true, null); + } + burst.join(); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(INTERLEAVED_EVENTS, countEventsOfKind(events, "feature")); + // The tracked evaluations are summarized too, alongside the burst. + assertEquals(INTERLEAVED_EVENTS, summaryCountFor(events, "tracked-flag")); + assertEquals(TOTAL_EVALUATIONS, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void distinctSummarizedContextsAreBoundedByCapacity() throws Exception { + // A summary counter costs a retained context and a set of counters per distinct context, and + // nothing drains it while the client is offline. Capacity has to bound that, or an application + // that goes on identifying through an outage grows the summarizer for as long as it lasts. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CONTEXT_LIMIT); + try { + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + + List events = flushAndCollect(eventProcessor, server); + + // One summary event per context that was counted, so this is the cardinality the + // summarizer was holding. + assertEquals(CONTEXT_LIMIT, countEventsOfKind(events, "summary")); + logging.assertWarnLogged("Exceeded the number of contexts"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void exceedingContextLimitIsWarnedOncePerFlushRun() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CONTEXT_LIMIT); + try { + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + flushAndCollect(eventProcessor, server); + + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(100 + i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + flushAndCollect(eventProcessor, server); + + int warnCount = 0; + for (String msg : logging.logCapture.getMessageStrings()) { + if (msg.contains("Exceeded the number of contexts")) { + warnCount++; + } + } + assertEquals(2, warnCount); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aContextAlreadyBeingCountedKeepsCountingOnceTheLimitIsReached() throws Exception { + // The limit is on how many contexts are held, not on how many evaluations are counted. An + // application evaluating against one context must not start losing counts because some other + // part of it churned through contexts. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CONTEXT_LIMIT); + try { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + int evaluationsAfterTheLimit = 500; + for (int i = 0; i < evaluationsAfterTheLimit; i++) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(evaluationsAfterTheLimit + 1, summaryCountForContext(events, CONTEXT)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void theLimitLiftsOnceTheSummariesHaveBeenDelivered() throws Exception { + // The bound is on how many contexts are held at once, not on how many an application may ever + // use, so a delivery has to make room for the next set. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CONTEXT_LIMIT); + try { + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + flushAndCollect(eventProcessor, server); + + for (int i = CONTEXT_LIMIT * 3; i < CONTEXT_LIMIT * 4; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + List events = flushAndCollect(eventProcessor, server); + + assertEquals(CONTEXT_LIMIT, countEventsOfKind(events, "summary")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aCommitPointMakesRoomInTheSummarizer() throws Exception { + assumePersisting(); + // Counters are materialized into summary events and reset at every commit point, so the + // summarizer only ever holds the contexts seen since the last one. That reset is what keeps + // the cardinality limit from binding: without it an application identifying its way through + // more contexts than capacity would start losing counts partway through. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CAPACITY); + try { + int contexts = CAPACITY * 3; + for (int i = 0; i < contexts; i++) { + LDContext context = contextNumber(i); + eventProcessor.recordEvaluationEvent(context, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + eventProcessor.recordIdentifyEvent(context); // a commit point + } + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(contexts, countEventsOfKind(events, "summary")); + assertFalse("evaluations were turned away even though every one of them followed a" + + " commit point that should have made room", + logged("Exceeded the number of contexts")); + } finally { + eventProcessor.close(); + } + } + } + + private boolean logged(String messageSubstring) { + for (String message : logging.logCapture.getMessageStrings()) { + if (message.contains(messageSubstring)) { + return true; + } + } + return false; + } + + private static LDContext contextNumber(int i) { + return LDContext.builder("burst-user-" + i).set("email", "user" + i + "@example.com").build(); + } + + /** @return how many evaluations the summary for this context counted, across every flag in it */ + private static int summaryCountForContext(List events, LDContext context) { + int total = 0; + for (LDValue event : events) { + if (!"summary".equals(event.get("kind").stringValue()) + || !context.getKey().equals(event.get("context").get("key").stringValue())) { + continue; + } + LDValue features = event.get("features"); + for (String flagKey : features.keys()) { + for (LDValue counter : features.get(flagKey).get("counters").values()) { + total += counter.get("count").intValue(); + } + } + } + return total; + } + + /** + * Records evaluations of an untracked flag from several threads at once, faster than any + * single consumer thread could drain them. Tests that need to interleave other events use + * {@link #awaitFraction} rather than sleeping, so the interleaved events land at the same + * points in the burst regardless of how fast the machine running the test is. + */ + private final class EvaluationBurst { + private final List workers = new ArrayList<>(); + private volatile int progress; + + EvaluationBurst(EventProcessor eventProcessor) { + for (int i = 0; i < BURST_THREADS; i++) { + boolean reportsProgress = i == 0; + Thread worker = new Thread(() -> { + for (int j = 0; j < EVALUATIONS_PER_THREAD; j++) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + if (reportsProgress) { + progress = j + 1; + } + } + }); + worker.start(); + workers.add(worker); + } + } + + /** Blocks until the burst is {@code numerator/denominator} of the way through. */ + void awaitFraction(int numerator, int denominator) { + int target = (int) ((long) EVALUATIONS_PER_THREAD * numerator / denominator); + while (progress < target && workers.get(0).isAlive()) { + Thread.yield(); + } + } + + void join() throws InterruptedException { + for (Thread worker : workers) { + worker.join(); + } + } + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingWithoutPersistenceTest.java new file mode 100644 index 000000000..4429958cd --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingWithoutPersistenceTest.java @@ -0,0 +1,11 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.sdk.android.integrations.EventPersistence; + +/** {@link EventProcessorBufferingTest} with persistence off, which is what an application gets by default. */ +public class EventProcessorBufferingWithoutPersistenceTest extends EventProcessorBufferingTest { + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsTest.java new file mode 100644 index 000000000..31f66e523 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsTest.java @@ -0,0 +1,176 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.testhelpers.httptest.HttpServer; + +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Tests for the two per-event flags the SDK is expected to honour: {@code excludeFromSummaries}, + * which keeps an evaluation out of the counters, and {@code samplingRatio}, which decides whether a + * full event is kept at all. + *

+ * The two are enforced at different stages, because that is where each one's subject matter lives. + * The counters are in {@link OutboundEventBuffer}, so the exclusion is checked there. The run of + * full events and the capacity counted against it are in {@link DirectEventProcessor}, so sampling + * is checked there, ahead of the capacity check that would otherwise record a sampled-out event as + * a drop. + *

+ * Neither flag can be reached through the SDK's own recording path. {@link DirectEventProcessor} + * builds its feature events with the constructor overload that fixes them at {@code false} and + * {@code 1}, so every evaluation an application performs leaves both at their defaults. That is why + * these tests hand an event straight to the stage that enforces the flag rather than going through + * {@code recordEvaluationEvent}: those are the only seams at which the flags can vary. + *

+ * They are worth pinning even so. java-sdk-internal's {@code DefaultEventProcessor}, which this + * path replaced, honoured both, and anything that later constructs feature events through the + * fourteen-argument constructor — adopting sampling is the obvious candidate — makes both live at + * once. The failure would be silent and would show up as wrong analytics rather than as an error, + * since an event carries its sampling ratio on the wire so the receiver can scale by it. + *

+ * Both ratios used here are deterministic rather than probabilistic: {@code Sampler.shouldSample} + * short-circuits to {@code true} at 1 and to {@code false} at 0, so nothing below is flaky. + */ +public class EventProcessorFlagsTest extends EventProcessorTestBase { + private static final String FLAG_KEY = "flags-test-flag"; + private static final int FLAG_VERSION = 3; + private static final int VARIATION = 1; + private static final LDValue FLAG_VALUE = LDValue.of(true); + private static final LDValue DEFAULT_VALUE = LDValue.of(false); + private static final long CREATION_DATE = 1_700_000_000_000L; + + // Far more than any test here records, so that capacity can never be what discarded an event. + private static final int CAPACITY = 100; + + @Test + public void anEvaluationExcludedFromSummariesIsNotCounted() { + OutboundEventBuffer buffer = makeBuffer(); + + buffer.summarize(evaluation(true)); + + assertTrue("an excluded evaluation should leave nothing to send", + drainToEvents(buffer).isEmpty()); + } + + @Test + public void anEvaluationNotExcludedFromSummariesIsCounted() { + OutboundEventBuffer buffer = makeBuffer(); + + buffer.summarize(evaluation(false)); + + assertEquals(1, summaryCountFor(drainToEvents(buffer), FLAG_KEY)); + } + + @Test + public void excludingOneEvaluationLeavesTheOthersCounted() { + OutboundEventBuffer buffer = makeBuffer(); + + buffer.summarize(evaluation(false)); + buffer.summarize(evaluation(true)); + buffer.summarize(evaluation(false)); + + // The exclusion applies to the event that carried it and to nothing else, which is what + // distinguishes honouring the flag from dropping the whole summary. + assertEquals(2, summaryCountFor(drainToEvents(buffer), FLAG_KEY)); + } + + @Test + public void aFullEventSampledOutIsNotKept() throws Exception { + try (HttpServer server = startEventsServer()) { + DirectEventProcessor processor = processor(server); + try { + processor.record(evaluation(false, 0)); + // Something the SDK will certainly send, so the flush has a payload to look at. + // Without it there would be no request at all, and nothing to distinguish an event + // that was dropped from one that simply has not been sent yet. + processor.recordIdentifyEvent(CONTEXT); + + List events = flushAndCollect(processor, server); + + assertEquals(0, countEventsOfKind(events, "feature")); + assertEquals(1, countEventsOfKind(events, "identify")); + } finally { + processor.close(); + } + } + } + + @Test + public void aFullEventSampledInIsKept() throws Exception { + try (HttpServer server = startEventsServer()) { + DirectEventProcessor processor = processor(server); + try { + processor.record(evaluation(false, 1)); + + assertEquals(1, countEventsOfKind(flushAndCollect(processor, server), "feature")); + } finally { + processor.close(); + } + } + } + + @Test + public void stagingAnEventDoesNotDrawForSamplingASecondTime() throws Exception { + // The processor draws once, as the event is recorded, and hands on only what it kept. A second + // draw where the event is serialized would square the odds against anything with a ratio above + // one, and would drop it without counting it anywhere. A ratio of 0 stands in for a draw that + // comes out against the event, because Sampler.shouldSample refuses it every time. + OutboundEventBuffer buffer = makeBuffer(); + Event sampledOut = evaluation(false, 0); + + assertEquals(1, buffer.serializeAll(Collections.singletonList((Event) sampledOut)).size()); + assertNotNull(buffer.serialize(sampledOut)); + } + + @Test + public void aSampledOutEventIsNotCountedAsADrop() throws Exception { + try (HttpServer server = startEventsServer()) { + DirectEventProcessor processor = processor(server); + try { + processor.record(evaluation(false, 0)); + + // Sampling and capacity are different reasons not to keep an event, and only the + // second is a loss the SDK owes anyone a count of. + assertEquals(0, processor.getAndClearDroppedCount()); + } finally { + processor.close(); + } + } + } + + private DirectEventProcessor processor(HttpServer server) { + return (DirectEventProcessor) makeEventProcessor(server, CAPACITY); + } + + private OutboundEventBuffer makeBuffer() { + return new OutboundEventBuffer(false, Collections.emptyList(), false, 100, logging.logger); + } + + private Event.FeatureRequest evaluation(boolean excludeFromSummaries) { + return evaluation(excludeFromSummaries, 1); + } + + private Event.FeatureRequest evaluation(boolean excludeFromSummaries, long samplingRatio) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY, CONTEXT, FLAG_VERSION, VARIATION, + FLAG_VALUE, DEFAULT_VALUE, null, null, true, null, false, samplingRatio, + excludeFromSummaries); + } + + private List drainToEvents(OutboundEventBuffer buffer) { + List events = new ArrayList<>(); + for (byte[] summary : buffer.serializeSummariesAndReset()) { + events.add(LDValue.parse(new String(summary, StandardCharsets.UTF_8))); + } + return events; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsWithoutPersistenceTest.java new file mode 100644 index 000000000..a63a323f7 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsWithoutPersistenceTest.java @@ -0,0 +1,11 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.sdk.android.integrations.EventPersistence; + +/** {@link EventProcessorFlagsTest} with persistence off, which is what an application gets by default. */ +public class EventProcessorFlagsWithoutPersistenceTest extends EventProcessorFlagsTest { + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlushAsyncDefaultTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlushAsyncDefaultTest.java new file mode 100644 index 000000000..427fc39a9 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlushAsyncDefaultTest.java @@ -0,0 +1,90 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Covers what {@link EventProcessor#flushAsync()} does for an implementation that predates it and + * has only its unbounded {@link EventProcessor#blockingFlush()}. The SDK has to be able to put a + * deadline on such a flush, and must not tell the caller its events are safe without knowing. + */ +public class EventProcessorFlushAsyncDefaultTest { + @Rule + public Timeout globalTimeout = Timeout.seconds(30); + + @Test + public void theCallersDeadlineBoundsTheWaitAndNotTheFlush() throws Exception { + LegacyEventProcessor eventProcessor = new LegacyEventProcessor(); + Future delivery = eventProcessor.flushAsync(); + + try { + delivery.get(100, TimeUnit.MILLISECONDS); + fail("the wait outlived the deadline"); + } catch (TimeoutException expected) { + // The flush is still going, which is why this is what the caller is told. + } + assertFalse(eventProcessor.flushReturned.get()); + + eventProcessor.letFlushFinish.countDown(); + assertTrue("the flush's own outcome was not reported", + delivery.get(5, TimeUnit.SECONDS)); + assertTrue(eventProcessor.flushReturned.get()); + } + + /** An implementation written before {@code flushAsync} existed. */ + private static final class LegacyEventProcessor implements EventProcessor { + final CountDownLatch letFlushFinish = new CountDownLatch(1); + final AtomicBoolean flushReturned = new AtomicBoolean(false); + + @Override + public void blockingFlush() { + try { + letFlushFinish.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + flushReturned.set(true); + } + + @Override + public void flush() {} + + @Override + public void setInBackground(boolean inBackground) {} + + @Override + public void setOffline(boolean offline) {} + + @Override + public void close() {} + + @Override + public void recordEvaluationEvent(LDContext context, String flagKey, int flagVersion, + int variation, LDValue value, EvaluationReason reason, + LDValue defaultValue, boolean requireFullEvent, + Long debugEventsUntilDate) {} + + @Override + public void recordIdentifyEvent(LDContext context) {} + + @Override + public void recordCustomEvent(LDContext context, String eventKey, LDValue data, + Double metricValue) {} + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyTest.java new file mode 100644 index 000000000..5ff0bcba3 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyTest.java @@ -0,0 +1,269 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.ContextKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.testhelpers.httptest.HttpServer; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * Context attribute redaction, as observed in the payload the processor actually posts. + *

+ * Redaction is not reimplemented by the SDK's event processor - it reuses java-sdk-internal's + * context formatter through {@link OutboundEventBuffer}. These + * tests exist because that reuse is the whole argument for the current design, and nothing else in + * this repository asserts that an attribute an application marked private stays out of the wire + * format. Beyond the redaction itself they pin the two rules that are easy to get wrong when + * events are assembled somewhere new: which events inline a context at all, and which of them + * additionally redact the attributes of an anonymous context. + */ +public class EventProcessorPrivacyTest extends EventProcessorTestBase { + private static final String FLAG_KEY = "flag-key"; + private static final int DEFAULT_CAPACITY = 100; + + private static final LDContext PERSON = LDContext.builder("user-key") + .name("Sandy") + .set("email", "sandy@example.com") + .set("address", LDValue.buildObject() + .put("city", "Oakland") + .put("street", "123 Main St") + .build()) + .build(); + + @Test + public void namedPrivateAttributesAreRedactedAndReported() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("email"), true); + try { + eventProcessor.recordIdentifyEvent(PERSON); + + LDValue context = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + assertEquals(LDValue.of("user-key"), context.get("key")); + assertEquals("email must not appear in the payload", + LDValue.ofNull(), context.get("email")); + // Attributes that were not marked private are still sent. + assertEquals(LDValue.of("Sandy"), context.get("name")); + assertEquals(redacted("email"), redactedAttributes(context)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void allAttributesPrivateLeavesOnlyTheKey() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).allAttributesPrivate(true), true); + try { + eventProcessor.recordIdentifyEvent(PERSON); + + LDValue context = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + // The key and kind are identifiers rather than attributes, so they survive. + assertEquals(LDValue.of("user-key"), context.get("key")); + assertEquals(LDValue.of("user"), context.get("kind")); + assertEquals(LDValue.ofNull(), context.get("name")); + assertEquals(LDValue.ofNull(), context.get("email")); + assertEquals(LDValue.ofNull(), context.get("address")); + assertEquals(redacted("address", "email", "name"), redactedAttributes(context)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aPrivateSubAttributeRedactsOnlyThatProperty() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("/address/street"), true); + try { + eventProcessor.recordIdentifyEvent(PERSON); + + LDValue context = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + assertEquals(LDValue.of("Oakland"), context.get("address").get("city")); + assertEquals(LDValue.ofNull(), context.get("address").get("street")); + assertEquals(redacted("/address/street"), redactedAttributes(context)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void attributesMarkedPrivateOnTheContextItselfAreRedacted() throws Exception { + // An application can mark an attribute private per-context instead of globally; the + // processor is not configured with anything in this case. + LDContext context = LDContext.builder("user-key") + .name("Sandy") + .set("email", "sandy@example.com") + .privateAttributes("email") + .build(); + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordIdentifyEvent(context); + + LDValue delivered = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + assertEquals(LDValue.ofNull(), delivered.get("email")); + assertEquals(LDValue.of("Sandy"), delivered.get("name")); + assertEquals(redacted("email"), redactedAttributes(delivered)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void anonymousContextAttributesAreRedactedFromFeatureEventsButNotIdentifyEvents() throws Exception { + LDContext anonymous = LDContext.builder(ContextKind.DEFAULT, "anon-key") + .anonymous(true) + .set("email", "sandy@example.com") + .build(); + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordIdentifyEvent(anonymous); + eventProcessor.recordEvaluationEvent(anonymous, FLAG_KEY, 10, 1, LDValue.of(true), + null, LDValue.of(false), true, null); + + List events = flushAndCollect(eventProcessor, server); + + // An identify event is the application deliberately registering this context. + LDValue identifyContext = requireEventOfKind(events, "identify").get("context"); + assertEquals(LDValue.of("sandy@example.com"), identifyContext.get("email")); + + // A feature event redacts every attribute of an anonymous context, without the + // application having to mark anything private. + LDValue featureContext = requireEventOfKind(events, "feature").get("context"); + assertEquals(LDValue.of("anon-key"), featureContext.get("key")); + assertEquals(LDValue.ofNull(), featureContext.get("email")); + assertEquals(redacted("email"), redactedAttributes(featureContext)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void summaryEventsRedactPrivateAttributesOfTheContextTheyInline() throws Exception { + // Because this SDK summarizes per context, a summary event inlines the context it counted + // rather than just its key, and so it is subject to redaction like any other event. This + // is the case most easily missed, since summaries are produced by the buffer rather than + // by an explicit record call. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("email"), true); + try { + eventProcessor.recordEvaluationEvent(PERSON, FLAG_KEY, 10, 1, LDValue.of(true), + null, LDValue.of(false), false, null); + + LDValue summary = requireEventOfKind(flushAndCollect(eventProcessor, server), + "summary"); + + LDValue context = summary.get("context"); + assertEquals(LDValue.of("user-key"), context.get("key")); + assertEquals(LDValue.ofNull(), context.get("email")); + assertEquals(redacted("email"), redactedAttributes(context)); + // Attributes that were not marked private are inlined here, same as anywhere else. + assertEquals(LDValue.of("Oakland"), context.get("address").get("city")); + assertFalse("summary should not contain the email value anywhere", + summary.toJsonString().contains("sandy@example.com")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void redactionAppliesToEveryKindOfAMultiContext() throws Exception { + LDContext multi = LDContext.createMulti( + LDContext.builder(ContextKind.of("user"), "user-key") + .set("email", "sandy@example.com") + .build(), + LDContext.builder(ContextKind.of("org"), "org-key") + .set("email", "billing@example.com") + .build()); + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("email"), true); + try { + eventProcessor.recordIdentifyEvent(multi); + + LDValue delivered = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + assertEquals(LDValue.of("multi"), delivered.get("kind")); + assertEquals(LDValue.ofNull(), delivered.get("user").get("email")); + assertEquals(LDValue.ofNull(), delivered.get("org").get("email")); + assertEquals(redacted("email"), redactedAttributes(delivered.get("user"))); + assertEquals(redacted("email"), redactedAttributes(delivered.get("org"))); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void noPayloadContainsAPrivateValueAnywhere() throws Exception { + // A belt-and-braces check over the whole request body rather than one field, so that a + // private value cannot slip through in some event kind these tests do not name. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("email"), true); + try { + eventProcessor.recordIdentifyEvent(PERSON); + eventProcessor.recordCustomEvent(PERSON, "an-event", LDValue.ofNull(), null); + eventProcessor.recordEvaluationEvent(PERSON, FLAG_KEY, 10, 1, LDValue.of(true), + null, LDValue.of(false), true, null); + + List events = flushAndCollect(eventProcessor, server); + + assertTrue("expected identify, custom, feature and summary events", + events.size() >= 4); + for (LDValue event : events) { + assertFalse("private value leaked into " + event.get("kind") + ": " + event, + event.toJsonString().contains("sandy@example.com")); + } + } finally { + eventProcessor.close(); + } + } + } + + private static List redacted(String... names) { + List result = new ArrayList<>(); + for (String name : names) { + result.add(name); + } + return result; + } + + /** The names the payload itself reports as redacted, sorted so the comparison is stable. */ + private static List redactedAttributes(LDValue context) { + List names = new ArrayList<>(); + for (LDValue name : context.get("_meta").get("redactedAttributes").values()) { + names.add(name.stringValue()); + } + java.util.Collections.sort(names); + return names; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyWithoutPersistenceTest.java new file mode 100644 index 000000000..8d91eeecf --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyWithoutPersistenceTest.java @@ -0,0 +1,11 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.sdk.android.integrations.EventPersistence; + +/** {@link EventProcessorPrivacyTest} with persistence off, which is what an application gets by default. */ +public class EventProcessorPrivacyWithoutPersistenceTest extends EventProcessorPrivacyTest { + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorTestBase.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorTestBase.java new file mode 100644 index 000000000..33dd8f740 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorTestBase.java @@ -0,0 +1,211 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.LDConfig.Builder.AutoEnvAttributes; +import com.launchdarkly.sdk.android.env.EnvironmentReporterBuilder; +import com.launchdarkly.sdk.android.env.IEnvironmentReporter; +import com.launchdarkly.sdk.android.integrations.EventPersistence; +import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; +import com.launchdarkly.sdk.android.subsystems.ClientContext; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.testhelpers.httptest.Handlers; +import com.launchdarkly.testhelpers.httptest.HttpServer; +import com.launchdarkly.testhelpers.httptest.RequestInfo; + +import org.junit.Assume; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.Timeout; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Fixture for tests that drive the SDK's event processor directly and read back the analytics + * payload it posts. + */ +public abstract class EventProcessorTestBase { + protected static final String MOBILE_KEY = "test-mobile-key"; + protected static final LDContext CONTEXT = LDContext.create("user-key"); + + // Long enough that the only payload in a test is the one it asks for explicitly. + protected static final int NO_PERIODIC_FLUSH_MILLIS = 600_000; + + @Rule + public Timeout globalTimeout = Timeout.seconds(60); + @Rule + public LogCaptureRule logging = new LogCaptureRule(); + /** + * A directory of its own per test, so that the events one test persists are never found by the next. + */ + @Rule + public TemporaryFolder eventsDirectory = new TemporaryFolder(); + + private final IEnvironmentReporter environmentReporter = new EnvironmentReporterBuilder().build(); + + /** + * The processor persists events, so it needs somewhere to put them. Tests get a real directory + * rather than a stub, which is what makes them exercise the same path an application does. + */ + protected MockPlatformState platformState() { + MockPlatformState platformState = new MockPlatformState(); + platformState.setNoBackupFilesDir(eventsDirectory.getRoot()); + return platformState; + } + + protected HttpServer startEventsServer() { + return HttpServer.start(Handlers.status(202)); + } + + protected EventProcessor makeEventProcessor(HttpServer server, int capacity) { + return makeEventProcessor(server, capacity, true); + } + + protected EventProcessor makeEventProcessor(HttpServer server, int capacity, + boolean diagnosticOptOut) { + return makeEventProcessor(server, eventsBuilder(capacity), diagnosticOptOut); + } + + /** + * @return an events builder with the capacity set and periodic flushing effectively off, for + * tests that need to configure something else on top such as private attributes + */ + protected EventProcessorBuilder eventsBuilder(int capacity) { + return Components.sendEvents() + .capacity(capacity) + .eventPersistence(persistence()) + .flushIntervalMillis(NO_PERIODIC_FLUSH_MILLIS); + } + + /** + * {@link #eventsBuilder} without the backoff, so every flush posts and is retried at once, for tests + * about what a failure keeps rather than about how long it waits. + */ + protected EventProcessorBuilder eventsBuilderWithoutBackoff(int capacity) { + return ((ComponentsImpl.EventProcessorBuilderImpl) eventsBuilder(capacity)) + .retryDelaysForTesting(0, 0, 0); + } + + /** + * The persistence {@link #eventsBuilder} configures. IMMEDIATE here, which is what lets a test + * assert on the store straight after a track; each suite also runs as a subclass that answers + * DISABLED, because that is what an application gets unless it asks. + */ + protected EventPersistence persistence() { + return EventPersistence.IMMEDIATE; + } + + /** Skips a test whose subject is the disk, in the subclasses that run a suite without one. */ + protected void assumePersisting() { + Assume.assumeTrue("needs a disk", persistence() != EventPersistence.DISABLED); + } + + protected EventProcessor makeEventProcessor(HttpServer server, EventProcessorBuilder events, + boolean diagnosticOptOut) { + return makeEventProcessor(server.getUri(), events, diagnosticOptOut); + } + + protected EventProcessor makeEventProcessor(URI eventsUri, EventProcessorBuilder events, + boolean diagnosticOptOut) { + EventProcessor eventProcessor = buildOfflineEventProcessor(eventsUri, events, diagnosticOptOut, + platformState()); + // LDClient turns the processor on once initialization decides the SDK is not in offline mode. + eventProcessor.setOffline(false); + return eventProcessor; + } + + /** @return the processor as the SDK builds it, before initialization has turned it on */ + protected EventProcessor buildOfflineEventProcessor(HttpServer server, EventProcessorBuilder events, + boolean diagnosticOptOut) { + return buildOfflineEventProcessor(server, events, diagnosticOptOut, platformState()); + } + + protected EventProcessor buildOfflineEventProcessor(HttpServer server, EventProcessorBuilder events, + boolean diagnosticOptOut, + PlatformState platformState) { + return buildOfflineEventProcessor(server.getUri(), events, diagnosticOptOut, platformState); + } + + protected EventProcessor buildOfflineEventProcessor(URI eventsUri, EventProcessorBuilder events, + boolean diagnosticOptOut, + PlatformState platformState) { + LDConfig config = new LDConfig.Builder(AutoEnvAttributes.Disabled) + .mobileKey(MOBILE_KEY) + .diagnosticOptOut(diagnosticOptOut) + .events(events) + .serviceEndpoints(Components.serviceEndpoints().events(eventsUri)) + .build(); + ClientContext clientContext = ClientContextImpl.fromConfig(config, MOBILE_KEY, "", + null, null, CONTEXT, logging.logger, platformState, environmentReporter, null); + return config.events.build(clientContext); + } + + /** + * Flushes and returns every event the processor posted. {@code blockingFlush} does not return + * until delivery completes, so there is no settling delay here. + */ + protected List flushAndCollect(EventProcessor eventProcessor, HttpServer server) { + eventProcessor.blockingFlush(); + return collectDelivered(server); + } + + /** Returns every event posted so far, without asking for a flush. */ + protected List collectDelivered(HttpServer server) { + List events = new ArrayList<>(); + collectRequest(server.getRecorder().requireRequest(10, TimeUnit.SECONDS), events); + while (server.getRecorder().count() > 0) { + collectRequest(server.getRecorder().requireRequest(10, TimeUnit.SECONDS), events); + } + return events; + } + + private void collectRequest(RequestInfo request, List events) { + assertEquals("POST", request.getMethod()); + assertEquals("/mobile/events/bulk", request.getPath()); + for (LDValue event : LDValue.parse(request.getBody()).values()) { + events.add(event); + } + } + + protected int countEventsOfKind(List events, String kind) { + int count = 0; + for (LDValue event : events) { + if (kind.equals(event.get("kind").stringValue())) { + count++; + } + } + return count; + } + + protected LDValue requireEventOfKind(List events, String kind) { + for (LDValue event : events) { + if (kind.equals(event.get("kind").stringValue())) { + return event; + } + } + throw new AssertionError("no event of kind " + kind + " in " + events); + } + + protected int summaryCountFor(List events, String flagKey) { + int total = 0; + boolean sawSummary = false; + for (LDValue event : events) { + if (!"summary".equals(event.get("kind").stringValue())) { + continue; + } + sawSummary = true; + // Per-context summarization means a flag can have more than one counter entry. + for (LDValue counter : event.get("features").get(flagKey).get("counters").values()) { + total += counter.get("count").intValue(); + } + } + assertTrue("no summary event was delivered", sawSummary); + return total; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorWithoutPersistenceTest.java new file mode 100644 index 000000000..d6808993a --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorWithoutPersistenceTest.java @@ -0,0 +1,310 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.integrations.EventPersistence; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.testhelpers.httptest.Handler; +import com.launchdarkly.testhelpers.httptest.HttpServer; +import com.launchdarkly.testhelpers.httptest.RequestContext; + +import org.junit.Test; + +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntUnaryOperator; + +/** + * What an application gets by default: events held in memory and delivered from there, with nothing + * on the disk. These are the situations where holding events is hardest, which the suites shared with + * persistence (run here through their WithoutPersistence subclasses) do not set out to reach. + */ +public class EventProcessorWithoutPersistenceTest extends EventProcessorTestBase { + private static final String PAYLOAD_ID_HEADER = "X-LaunchDarkly-Payload-ID"; + + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } + + /** + * An events service whose answer to each post can change during a test, and which keeps what it took. + * It serves either test-helpers' {@link HttpServer} or OkHttp's {@link MockWebServer}. + */ + private static final class EventsService extends Dispatcher implements Handler { + final AtomicInteger posts = new AtomicInteger(); + /** The status for the nth post, counting from 1. */ + volatile IntUnaryOperator statusForPost = post -> 202; + /** Custom event keys of each post the service accepted, by payload ID, in the order accepted. */ + final List>> accepted = + Collections.synchronizedList(new ArrayList>>()); + + @Override + public void apply(RequestContext context) { + context.setStatus(respond(context.getRequest().getBody(), + context.getRequest().getHeader(PAYLOAD_ID_HEADER))); + } + + @Override + public MockResponse dispatch(RecordedRequest request) { + return new MockResponse().setResponseCode( + respond(request.getBody().readUtf8(), request.getHeader(PAYLOAD_ID_HEADER))); + } + + private int respond(String body, String payloadId) { + int status = statusForPost.applyAsInt(posts.incrementAndGet()); + if (status == 202) { + List keys = new ArrayList<>(); + for (LDValue event : LDValue.parse(body).values()) { + if ("custom".equals(event.get("kind").stringValue())) { + keys.add(event.get("key").stringValue()); + } + } + accepted.add(new HashMap.SimpleEntry<>(payloadId, keys)); + } + return status; + } + + /** Every accepted key once per payload ID, as the service counts them, in the order accepted. */ + List acceptedKeys() { + Set payloads = new HashSet<>(); + List keys = new ArrayList<>(); + synchronized (accepted) { + for (Map.Entry> post : accepted) { + if (payloads.add(post.getKey())) { + keys.addAll(post.getValue()); + } + } + } + return keys; + } + } + + private static boolean awaitFlush(EventProcessor eventProcessor) throws Exception { + try { + return Boolean.TRUE.equals(eventProcessor.flushAsync().get(10, TimeUnit.SECONDS)); + } catch (TimeoutException e) { + return false; + } + } + + private static List keys(String prefix, int from, int to) { + List keys = new ArrayList<>(); + for (int i = from; i < to; i++) { + keys.add(prefix + i); + } + return keys; + } + + private static void track(EventProcessor eventProcessor, String key) { + eventProcessor.recordCustomEvent(CONTEXT, key, LDValue.ofNull(), null); + } + + @Test + public void aSessionOfRefusalsRetriesAndAShutdownNeverTouchesTheFilesystem() throws Exception { + AtomicInteger asked = new AtomicInteger(); + MockPlatformState platformState = new MockPlatformState() { + @Override + public File getNoBackupFilesDir() { + asked.incrementAndGet(); + return eventsDirectory.getRoot(); + } + }; + EventsService service = new EventsService(); + // Refused and refused again on the retry; then the batch held from that and the next one are + // accepted; then the service says the key is no good, after which nothing is posted for a while. + // The waits are cut to milliseconds, which makes that while about four seconds. + service.statusForPost = post -> post <= 2 ? 503 : post <= 4 ? 202 : 401; + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, + ((ComponentsImpl.EventProcessorBuilderImpl) eventsBuilder(100)) + .retryDelaysForTesting(8, 8, TimeUnit.HOURS.toMillis(1)), + true, platformState); + eventProcessor.setOffline(false); + try { + track(eventProcessor, "refused"); + assertFalse(awaitFlush(eventProcessor)); + track(eventProcessor, "accepted"); + assertTrue(awaitFlush(eventProcessor)); + track(eventProcessor, "shut-down"); + awaitFlush(eventProcessor); + track(eventProcessor, "after-shut-down"); + } finally { + eventProcessor.close(); + } + + assertEquals(Arrays.asList("refused", "accepted"), service.acceptedKeys()); + assertEquals("posted after the service said the key is no good", 5, service.posts.get()); + } + assertEquals("a store without persistence looked for its directory", 0, asked.get()); + String[] written = eventsDirectory.getRoot().list(); + assertEquals("wrote to the disk", 0, written == null ? 0 : written.length); + } + + @Test + public void anOutageKeepsTheOldestEventsUpToCapacityAndDeliversEachOnceInOrder() throws Exception { + // Before persistence, a batch refused twice was dropped. Now it waits in memory, still counted + // against capacity, so an outage costs the newest events rather than the oldest and memory stays + // bounded however long it lasts. + int capacity = 20; + EventsService service = new EventsService(); + AtomicBoolean down = new AtomicBoolean(true); + service.statusForPost = post -> down.get() ? 503 : 202; + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, eventsBuilderWithoutBackoff(capacity), + true); + try { + for (String key : keys("e-", 0, 10)) { + track(eventProcessor, key); + } + assertFalse("delivered while the service was down", awaitFlush(eventProcessor)); + for (String key : keys("e-", 10, 100)) { + track(eventProcessor, key); + } + assertFalse("delivered while the service was down", awaitFlush(eventProcessor)); + + down.set(false); + assertTrue(awaitFlush(eventProcessor)); + + assertEquals(keys("e-", 0, capacity), service.acceptedKeys()); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void eventsRecordedAndFlushedFromManyThreadsThroughRefusalsAreEachDeliveredOnce() throws Exception { + int recorders = 4; + int eventsEach = 150; + EventsService service = new EventsService(); + // Every third post refused, so retries, in-memory batches waiting for the next delivery and + // flushes overlapping one another all happen while the recorders are still going. + service.statusForPost = post -> post % 3 == 0 ? 503 : 202; + // Not test-helpers' HttpServer, which reads a request body with a single read(): under load a + // large post arrives in pieces, the body it hands over is cut short, and the rest of it is taken + // as the next request on the connection, which the SDK is then refused for. + try (MockWebServer server = new MockWebServer()) { + server.setDispatcher(service); + server.start(); + EventProcessor eventProcessor = makeEventProcessor(server.url("/").uri(), eventsBuilder(100_000), + true); + try { + CountDownLatch start = new CountDownLatch(1); + CountDownLatch recorded = new CountDownLatch(recorders); + for (int r = 0; r < recorders; r++) { + String prefix = "r" + r + "-"; + new Thread(() -> { + try { + start.await(); + for (String key : keys(prefix, 0, eventsEach)) { + track(eventProcessor, key); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + recorded.countDown(); + } + }).start(); + } + Thread flusher = new Thread(() -> { + while (recorded.getCount() > 0) { + eventProcessor.flush(); + try { + Thread.sleep(2); + } catch (InterruptedException e) { + return; + } + } + }); + flusher.start(); + start.countDown(); + assertTrue(recorded.await(20, TimeUnit.SECONDS)); + flusher.join(5_000); + + service.statusForPost = post -> 202; + assertTrue("what was held back was never delivered", awaitFlush(eventProcessor)); + + List delivered = service.acceptedKeys(); + Set distinct = new HashSet<>(delivered); + assertEquals("an event arrived under two payload IDs, so the service counts it twice", + delivered.size(), distinct.size()); + Set expected = new HashSet<>(); + for (int r = 0; r < recorders; r++) { + expected.addAll(keys("r" + r + "-", 0, eventsEach)); + } + assertEquals(expected, distinct); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void eventsRecordedBeforeTheProcessorIsTurnedOnAreDeliveredOnceItIs() throws Exception { + EventsService service = new EventsService(); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, eventsBuilder(100), true); + try { + for (String key : keys("offline-", 0, 3)) { + track(eventProcessor, key); + } + eventProcessor.flush(); + + eventProcessor.setOffline(false); + assertTrue(awaitFlush(eventProcessor)); + + assertEquals(keys("offline-", 0, 3), service.acceptedKeys()); + assertEquals("posted while offline, or posted twice", 1, service.posts.get()); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void atMostCapacityContextsAreSummarizedBetweenDeliveries() throws Exception { + // Without a disk, a commit point does not drain the summarizer, so between two deliveries it + // counts evaluations for at most capacity contexts and turns the rest away. This pins that + // trade-off: before the bound, offline evaluation across many contexts grew memory without + // limit; with persistence on, each commit point makes room again. + int capacity = 10; + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, eventsBuilder(capacity), true); + try { + for (int round = 0; round < 2; round++) { + for (int i = 0; i < capacity * 3; i++) { + LDContext context = LDContext.create("round-" + round + "-context-" + i); + eventProcessor.recordEvaluationEvent(context, "flag", 1, 0, LDValue.of(true), null, + LDValue.of(false), false, null); + } + List events = flushAndCollect(eventProcessor, server); + assertEquals("round " + round, capacity, countEventsOfKind(events, "summary")); + } + } finally { + eventProcessor.close(); + } + } + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventStoreBoundsTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventStoreBoundsTest.java new file mode 100644 index 000000000..ab40111f8 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventStoreBoundsTest.java @@ -0,0 +1,619 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.Timeout; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * What bounds the store holds an application's events within: the byte capacity, with the share of it + * held back for summaries, and the maximum age past which stored events are discarded. + */ +public class EventStoreBoundsTest { + private static final String MOBILE_KEY = "mob-key"; + private static final String OTHER_MOBILE_KEY = "mob-other-key"; + private static final int CAPACITY = 100; + private static final long BYTE_CAPACITY = 1000; + private static final int FRAME = 100; + + @Rule + public Timeout globalTimeout = Timeout.seconds(30); + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); + + private EventStore boundedStore(boolean persistEvents) { + return new EventStore(new File(tempFolder.getRoot(), "events"), CAPACITY, BYTE_CAPACITY, + EventProcessorBuilder.DEFAULT_MAX_AGE_MILLIS, persistEvents, logger); + } + + private EventStore environmentStore(String mobileKey) { + return EventStore.create(tempFolder.getRoot(), mobileKey, CAPACITY, true, logger); + } + + /** An event whose frame in the log, header included, is exactly {@code frameBytes} long. */ + private static byte[] event(int frameBytes, String key) { + String bare = "{\"kind\":\"custom\",\"key\":\"" + key + "\",\"data\":\"\"}"; + int padding = frameBytes - EventLogFormat.FRAME_HEADER_SIZE - bare.length(); + StringBuilder data = new StringBuilder(); + for (int i = 0; i < padding; i++) { + data.append('x'); + } + return ("{\"kind\":\"custom\",\"key\":\"" + key + "\",\"data\":\"" + data + "\"}") + .getBytes(StandardCharsets.UTF_8); + } + + private static void stageFrames(EventStore store, int count) { + for (int i = 1; i <= count; i++) { + assertTrue(store.stage(event(FRAME, "event-" + i))); + } + } + + private static File onlyBatchFile(EventStore store) { + File[] files = store.getDirectory().listFiles((dir, name) -> name.startsWith("ready-")); + assertNotNull(files); + assertEquals(1, files.length); + return files[0]; + } + + private static void age(File file, int days) { + assertTrue(file.setLastModified(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(days))); + } + + private static void ageContents(File directory, int days) { + File[] files = directory.listFiles(); + assertNotNull(files); + for (File file : files) { + age(file, days); + } + } + + // byte capacity + + @Test + public void refusesFullEventsPastWhatItHoldsBackForSummaries() { + EventStore store = boundedStore(true); + stageFrames(store, 8); + + assertFalse(store.hasRoomForEventBytes()); + assertFalse(store.stage(event(FRAME, "event-9"))); + assertEquals(8, store.getPendingEventCount()); + } + + @Test + public void admitsAnEventOnlyWhileThereIsRoomByCountAndByBytes() { + EventStore store = boundedStore(true); + stageFrames(store, 7); + + assertTrue(store.admitsEvent(0, CAPACITY)); + assertTrue(store.admitsEvent(CAPACITY - 8, CAPACITY)); + assertFalse(store.admitsEvent(CAPACITY - 7, CAPACITY)); + + stageFrames(store, 1); + assertFalse(store.admitsEvent(0, CAPACITY)); + } + + @Test + public void takesSummariesIntoWhatItHoldsBackAndRefusesThemPastTheByteCapacity() { + EventStore store = boundedStore(true); + stageFrames(store, 8); + + assertTrue(store.stage(event(FRAME, "summary-1"), true)); + assertTrue(store.stage(event(FRAME, "summary-2"), true)); + assertFalse(store.stage(event(FRAME, "summary-3"), true)); + // Their bytes count, but they are not events. + assertEquals(8, store.getPendingEventCount()); + } + + @Test + public void leavesSummariesOutOfTheEventCountInEveryState() { + // Written at every commit point, summaries would otherwise fill capacity during an outage, and full + // events would be refused for the summaries of evaluations already counted. + for (boolean persist : new boolean[] {true, false}) { + String mode = "persisting: " + persist; + EventStore store = new EventStore(new File(tempFolder.getRoot(), "persist-" + persist), 2, + EventProcessorBuilder.DEFAULT_CAPACITY_BYTES, EventProcessorBuilder.DEFAULT_MAX_AGE_MILLIS, + persist, logger); + assertTrue(store.stage(event(FRAME, "summary-1"), true)); + assertTrue(store.stage(event(FRAME, "event-1"))); + assertEquals(mode + ", staged", 1, store.getPendingEventCount()); + store.commit(); + assertEquals(mode + ", committed", 1, store.getPendingEventCount()); + + List closed = store.closeBatch(); + assertEquals(mode, 1, closed.size()); + EventStore.Batch batch = closed.get(0); + assertEquals(mode + ": the batch holds both frames", 2, batch.eventCount); + assertEquals(mode + ", closed", 1, store.getPendingEventCount()); + assertEquals(mode, Arrays.asList(batch), store.pendingBatches()); + assertEquals(mode + ", listed", 1, store.getPendingEventCount()); + + assertTrue(store.stage(event(FRAME, "summary-2"), true)); + assertTrue(mode, store.stage(event(FRAME, "event-2"))); + assertFalse(mode + ": capacity still counts the events", store.stage(event(FRAME, "event-3"))); + + store.startedSending(batch); + assertEquals(mode + ", sending", 1, store.getPendingEventCount()); + store.stoppedSending(batch); + assertEquals(mode + ", sent and failed", 2, store.getPendingEventCount()); + store.remove(batch); + assertEquals(mode + ", delivered", 1, store.getPendingEventCount()); + } + } + + @Test + public void refusesAReservedEventThatTurnsOutNotToFitAndGivesUpItsReservation() { + EventStore store = boundedStore(true); + stageFrames(store, 7); + store.reserve(1); + + assertFalse(store.stageReserved(event(2 * FRAME, "too-big"))); + assertEquals(7, store.getPendingEventCount()); + assertTrue(store.stage(event(FRAME, "fits"))); + } + + @Test + public void makesRoomAgainOnceABatchIsDelivered() { + EventStore store = boundedStore(true); + stageFrames(store, 8); + List closed = store.closeBatch(); + assertEquals(1, closed.size()); + assertFalse(store.stage(event(FRAME, "while-held"))); + + store.startedSending(closed.get(0)); + assertTrue(store.stage(event(FRAME, "while-sending"))); + store.remove(closed.get(0)); + assertTrue(store.hasRoomForEventBytes()); + } + + @Test + public void countsCommittedBytesAsWellAsStagedOnes() { + EventStore store = boundedStore(true); + stageFrames(store, 8); + store.commit(); + + assertFalse(store.stage(event(FRAME, "after-commit"))); + } + + @Test + public void countsTheBytesOfABatchAnotherProcessClosedFromTheListingWithoutReadingIt() { + File directory = new File(tempFolder.getRoot(), "events"); + EventStore writer = new EventStore(directory, CAPACITY, true, logger); + stageFrames(writer, 8); + assertEquals(1, writer.closeBatch().size()); + + AtomicInteger reads = new AtomicInteger(); + EventStore other = new EventStore(directory, CAPACITY, BYTE_CAPACITY, + EventProcessorBuilder.DEFAULT_MAX_AGE_MILLIS, true, logger) { + @Override + InputStream openForReading(File file) throws IOException { + reads.incrementAndGet(); + return new FileInputStream(file); + } + }; + assertEquals(1, other.pendingBatches().size()); + + assertEquals(0, reads.get()); + assertFalse(other.hasRoomForEventBytes()); + assertFalse(other.stage(event(FRAME, "other"))); + } + + @Test + public void boundsTheBatchesItHoldsInMemoryOnceItHasGivenUpOnTheDisk() { + EventStore store = new EventStore(new File(tempFolder.getRoot(), "events"), CAPACITY, + BYTE_CAPACITY, EventProcessorBuilder.DEFAULT_MAX_AGE_MILLIS, true, logger) { + @Override + FileOutputStream openForAppending(File file) throws IOException { + throw new IOException("no space left on device"); + } + }; + stageFrames(store, 8); + store.commit(); + assertFalse(store.isPersisting()); + assertEquals(1, store.closeBatch().size()); + + assertFalse(store.stage(event(FRAME, "past-capacity"))); + } + + @Test + public void isNotBoundedByBytesWhereItKeepsEventsOffTheDisk() { + EventStore store = boundedStore(false); + stageFrames(store, 20); + + assertTrue(store.hasRoomForEventBytes()); + assertEquals(20, store.getPendingEventCount()); + } + + @Test + public void boundsSummariesByBytesEvenWhereItKeepsEventsOffTheDisk() { + // Nothing else bounds them: capacity never refuses a summary, and a delivery that keeps failing + // keeps every one it was given. + EventStore store = boundedStore(false); + stageFrames(store, 8); + assertTrue(store.stage(event(FRAME, "summary-1"), true)); + assertEquals(1, store.closeBatch().size()); + assertTrue(store.stage(event(FRAME, "summary-2"), true)); + + assertFalse(store.stage(event(FRAME, "summary-3"), true)); + // Full events are still bounded by count alone. + assertTrue(store.stage(event(FRAME, "event-9"))); + } + + // batch size + + private static final int QUARTER_BATCH = EventLogFormat.MAX_BATCH_SIZE / 4; + + private EventStore storeWithRoomForLargeBatches(boolean persistEvents) { + return new EventStore(new File(tempFolder.getRoot(), "events"), CAPACITY, + EventProcessorBuilder.DEFAULT_CAPACITY_BYTES, EventProcessorBuilder.DEFAULT_MAX_AGE_MILLIS, + persistEvents, logger); + } + + @Test + public void assemblesAStoredBatchInTheArrayItWasReadIntoRatherThanASecondOne() throws IOException { + EventStore store = environmentStore(MOBILE_KEY); + stageQuarterBatches(store, 1, 3); + EventStore.Batch batch = store.closeBatch().get(0); + long fileSize = onlyBatchFile(store).length(); + + EventLogFormat.Payload payload = store.payload(batch); + + assertEquals(fileSize, payload.bytes.length); + assertTrue(payload.length < payload.bytes.length); + assertEquals(3, LDValue.parse(new String(payload.bytes, 0, payload.length, StandardCharsets.UTF_8)) + .size()); + } + + private static void stageQuarterBatches(EventStore store, int from, int to) { + for (int i = from; i <= to; i++) { + assertTrue(store.stage(event(QUARTER_BATCH, "event-" + i))); + } + } + + private static List> keysByBatch(EventStore store) throws IOException { + List> batches = new ArrayList<>(); + for (EventStore.Batch batch : store.pendingBatches()) { + byte[] body = store.body(batch); + assertTrue(body.length <= EventLogFormat.MAX_BATCH_SIZE); + List keys = new ArrayList<>(); + for (LDValue event : LDValue.parse(new String(body, StandardCharsets.UTF_8)).values()) { + keys.add(event.get("key").stringValue()); + } + assertEquals(keys.size(), batch.eventCount); + batches.add(keys); + } + // Batches closed within the same millisecond list in either order. + Collections.sort(batches, (a, b) -> a.get(0).compareTo(b.get(0))); + return batches; + } + + private static void assertNoBatchFileIsLargerThanABatch(EventStore store) { + File[] files = store.getDirectory().listFiles((dir, name) -> name.startsWith("ready-")); + assertNotNull(files); + for (File file : files) { + assertTrue(file.length() <= EventLogFormat.MAX_BATCH_SIZE); + } + } + + @Test + public void closesALogACommitWouldTakePastTheBatchSizeAndCarriesOnInANewOne() throws IOException { + // Four quarters and the file header are just past the batch size, so three fit in each. + EventStore store = storeWithRoomForLargeBatches(true); + stageQuarterBatches(store, 1, 5); + store.commit(); + store.closeBatch(); + + assertEquals(Arrays.asList(Arrays.asList("event-1", "event-2", "event-3"), + Arrays.asList("event-4", "event-5")), keysByBatch(store)); + assertNoBatchFileIsLargerThanABatch(store); + assertEquals(5, store.getPendingEventCount()); + } + + @Test + public void closesALogALaterCommitWouldTakePastTheBatchSize() throws IOException { + EventStore store = storeWithRoomForLargeBatches(true); + stageQuarterBatches(store, 1, 2); + store.commit(); + stageQuarterBatches(store, 3, 5); + store.commit(); + store.closeBatch(); + + assertEquals(Arrays.asList(Arrays.asList("event-1", "event-2", "event-3"), + Arrays.asList("event-4", "event-5")), keysByBatch(store)); + assertNoBatchFileIsLargerThanABatch(store); + } + + @Test + public void splitsWhatItHoldsInMemoryAtTheBatchSizeToo() throws IOException { + EventStore store = storeWithRoomForLargeBatches(false); + stageQuarterBatches(store, 1, 5); + + assertEquals(2, store.closeBatch().size()); + assertEquals(Arrays.asList(Arrays.asList("event-1", "event-2", "event-3"), + Arrays.asList("event-4", "event-5")), keysByBatch(store)); + assertEquals(5, store.getPendingEventCount()); + } + + @Test + public void takesAnEventThatFillsABatchOnItsOwnAndRefusesAnyLarger() throws IOException { + EventStore store = storeWithRoomForLargeBatches(true); + int largest = EventLogFormat.MAX_FRAME_SIZE + EventLogFormat.FRAME_HEADER_SIZE; + assertFalse(store.stage(event(largest + 1, "too-large"))); + stageFrames(store, 1); + assertTrue(store.stage(event(largest, "largest"))); + store.closeBatch(); + + assertEquals(Arrays.asList(Arrays.asList("event-1"), Arrays.asList("largest")), keysByBatch(store)); + assertNoBatchFileIsLargerThanABatch(store); + } + + // expiry + + @Test + public void discardsABatchLastWrittenLongerAgoThanTheMaximumAgeAndStopsCountingIt() { + EventStore store = environmentStore(MOBILE_KEY); + stageFrames(store, 2); + store.closeBatch(); + age(onlyBatchFile(store), 8); + + assertEquals(2, store.discardExpired()); + + assertTrue(store.pendingBatches().isEmpty()); + assertEquals(0, store.getPendingEventCount()); + } + + @Test + public void keepsABatchYoungerThanTheMaximumAge() { + EventStore store = environmentStore(MOBILE_KEY); + stageFrames(store, 1); + store.closeBatch(); + age(onlyBatchFile(store), 6); + + assertEquals(0, store.discardExpired()); + assertEquals(1, store.pendingBatches().size()); + } + + @Test + public void leavesABatchWhoseRequestIsOutHoweverOldItIs() { + EventStore store = environmentStore(MOBILE_KEY); + stageFrames(store, 1); + EventStore.Batch batch = store.closeBatch().get(0); + store.startedSending(batch); + age(onlyBatchFile(store), 8); + + assertEquals(0, store.discardExpired()); + assertTrue(onlyBatchFile(store).exists()); + } + + @Test + public void keepsEveryBatchItCountedWhereTheDirectoryCannotBeListed() { + EventStore store = environmentStore(MOBILE_KEY); + stageFrames(store, 1); + EventStore.Batch older = store.closeBatch().get(0); + age(onlyBatchFile(store), 1); + assertEquals(1, store.pendingBatches().size()); + stageFrames(store, 2); + EventStore.Batch newer = store.closeBatch().get(0); + File directory = store.getDirectory(); + + assertTrue(directory.setReadable(false)); + try { + EventStore.Listing listing = store.listPendingBatches(); + + assertFalse(listing.complete); + assertEquals(2, listing.batches.size()); + assertEquals(older.payloadId, listing.batches.get(0).payloadId); + assertEquals(newer.payloadId, listing.batches.get(1).payloadId); + assertEquals("the batches stopped counting against capacity", 3, store.getPendingEventCount()); + } finally { + assertTrue(directory.setReadable(true)); + } + assertTrue(store.listPendingBatches().complete); + assertEquals(3, store.getPendingEventCount()); + } + + @Test + public void listsNothingAndCompletelyWhereTheDirectoryIsNotThere() { + EventStore store = environmentStore(MOBILE_KEY); + stageFrames(store, 1); + store.closeBatch(); + File batch = onlyBatchFile(store); + assertTrue(batch.delete()); + assertTrue(store.getDirectory().delete()); + + EventStore.Listing listing = store.listPendingBatches(); + + assertTrue(listing.complete); + assertTrue(listing.batches.isEmpty()); + assertEquals(0, store.getPendingEventCount()); + } + + @Test + public void leavesABatchAnotherStoreInThisProcessIsSendingUntilItStops() { + EventStore sender = environmentStore(MOBILE_KEY); + stageFrames(sender, 1); + EventStore.Batch batch = sender.closeBatch().get(0); + sender.startedSending(batch); + File file = onlyBatchFile(sender); + age(file, 8); + + EventStore other = environmentStore(MOBILE_KEY); + assertEquals(0, other.discardExpired()); + assertTrue(file.exists()); + + sender.stoppedSending(batch); + assertEquals(1, other.discardExpired()); + assertFalse(file.exists()); + } + + @Test + public void leavesABatchAnotherProcessIsSendingUntilItStops() throws Exception { + EventStore store = environmentStore(MOBILE_KEY); + stageFrames(store, 1); + store.closeBatch(); + File file = onlyBatchFile(store); + age(file, 8); + + Process sender = new ProcessBuilder( + new File(System.getProperty("java.home"), "bin/java").getPath(), + "-cp", System.getProperty("java.class.path"), + BatchLockHolder.class.getName(), file.getAbsolutePath()) + .redirectErrorStream(true) + .start(); + try { + assertEquals("locked", new BufferedReader( + new InputStreamReader(sender.getInputStream(), StandardCharsets.UTF_8)).readLine()); + + assertEquals(0, store.discardExpired()); + assertTrue("deleted while another process was sending it", file.exists()); + } finally { + sender.getOutputStream().close(); + assertTrue(sender.waitFor(10, TimeUnit.SECONDS)); + } + + // The kernel let go of the lock with the process. + assertEquals(1, store.discardExpired()); + assertFalse(file.exists()); + } + + @Test + public void leavesTheDirectoryOfAnotherEnvironmentOpenInThisProcess() { + // Open, so it may be about to write, or be sending a batch older than the cutoff; its own store + // expires its batches. + EventStore open = environmentStore(OTHER_MOBILE_KEY); + stageFrames(open, 3); + open.closeBatch(); + File openDirectory = open.getDirectory(); + ageContents(openDirectory, 8); + + EventStore store = environmentStore(MOBILE_KEY); + + assertEquals(0, store.discardExpired()); + assertEquals(1, openDirectory.listFiles().length); + } + + @Test + public void discardsTheStaleLogOfAProcessThatIsGoneButNeverOneALiveProcessHolds() throws IOException { + EventStore store = environmentStore(MOBILE_KEY); + stageFrames(store, 1); + store.commit(); + EventStore gone = environmentStore(MOBILE_KEY); + stageFrames(gone, 2); + gone.commit(); + gone.close(); + File opening = new File(store.getDirectory(), DiskEventLog.OPENING_PREFIX + "left-by-a-crash"); + assertTrue(opening.createNewFile()); + ageContents(store.getDirectory(), 8); + + assertEquals(2, store.discardExpired()); + + List remaining = new ArrayList<>(); + for (File file : store.getDirectory().listFiles()) { + remaining.add(file.getName()); + } + assertEquals(1, remaining.size()); + assertTrue(remaining.get(0).startsWith(DiskEventLog.PREFIX)); + assertTrue("the live store's log was discarded", store.pendingEventPayloads().size() == 1); + } + + @Test + public void removesTheDirectoryOfAnEnvironmentNothingHasWrittenToForTheMaximumAge() { + EventStore abandoned = environmentStore(OTHER_MOBILE_KEY); + stageFrames(abandoned, 3); + abandoned.closeBatch(); + abandoned.close(); + File abandonedDirectory = abandoned.getDirectory(); + ageContents(abandonedDirectory, 8); + + EventStore store = environmentStore(MOBILE_KEY); + + assertEquals(3, store.discardExpired()); + assertFalse(abandonedDirectory.exists()); + } + + @Test + public void keepsTheDirectoryOfAnEnvironmentWrittenToRecently() { + EventStore inUse = environmentStore(OTHER_MOBILE_KEY); + stageFrames(inUse, 1); + inUse.closeBatch(); + File inUseDirectory = inUse.getDirectory(); + + EventStore store = environmentStore(MOBILE_KEY); + + assertEquals(0, store.discardExpired()); + assertEquals(1, inUseDirectory.listFiles().length); + } + + @Test + public void looksAtOtherEnvironmentsOnlyOncePerStore() { + EventStore store = environmentStore(MOBILE_KEY); + assertEquals(0, store.discardExpired()); + + EventStore abandoned = environmentStore(OTHER_MOBILE_KEY); + stageFrames(abandoned, 1); + abandoned.closeBatch(); + abandoned.close(); + ageContents(abandoned.getDirectory(), 8); + + assertEquals(0, store.discardExpired()); + assertTrue(abandoned.getDirectory().exists()); + } + + @Test + public void leavesTheSiblingsOfADirectoryTheSdkDidNotChoose() { + File sibling = new File(tempFolder.getRoot(), "unrelated"); + EventStore neighbor = new EventStore(sibling, CAPACITY, true, logger); + stageFrames(neighbor, 1); + neighbor.closeBatch(); + neighbor.close(); + ageContents(sibling, 8); + + EventStore store = new EventStore(new File(tempFolder.getRoot(), "events"), CAPACITY, true, + logger); + stageFrames(store, 1); + store.commit(); + + assertEquals(0, store.discardExpired()); + assertEquals(1, sibling.listFiles().length); + } + + @Test + public void discardsNothingWhereItKeepsEventsOffTheDisk() { + EventStore abandoned = environmentStore(OTHER_MOBILE_KEY); + stageFrames(abandoned, 1); + abandoned.closeBatch(); + abandoned.close(); + ageContents(abandoned.getDirectory(), 8); + + EventStore store = EventStore.create(tempFolder.getRoot(), MOBILE_KEY, CAPACITY, false, logger); + + assertEquals(0, store.discardExpired()); + assertEquals(1, abandoned.getDirectory().listFiles().length); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventStoreMultiProcessTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventStoreMultiProcessTest.java new file mode 100644 index 000000000..dcb370d0c --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventStoreMultiProcessTest.java @@ -0,0 +1,1071 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDValue; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.Timeout; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.channels.FileLock; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * The event store's behavior when an application runs the SDK in more than one process. + *

+ * An Android application may declare components in another process, and each process gets its own copy of + * the SDK sharing nothing but the filesystem. From the store's point of view a second process is simply a + * second instance over the same directory, which is what these tests set up. That is not the same thing as a second OS process - it cannot reproduce a genuinely concurrent + * {@code O_APPEND} write from separate address spaces, which needs an instrumented test - but it does + * exercise every decision the design makes about sharing: which files a process may touch, which it must + * leave alone, and what happens when both reach for the same batch. + */ +public class EventStoreMultiProcessTest { + private static final int CAPACITY = 100; + + @Rule + public Timeout globalTimeout = Timeout.seconds(30); + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); + + private File sharedDirectory() { + return new File(tempFolder.getRoot(), "events"); + } + + /** A store over the shared directory: another process, or another run of one. */ + private EventStore storeFor() { + return storeFor(CAPACITY); + } + + private EventStore storeFor(int capacity) { + return new EventStore(sharedDirectory(), capacity, true, logger); + } + + private byte[] event(String key) { + return ("{\"kind\":\"custom\",\"key\":\"" + key + "\"}").getBytes(Charset.forName("UTF-8")); + } + + /** The event keys a store can currently see, which is its own log plus every shared batch. */ + private List visibleKeys(EventStore store) { + List keys = new ArrayList<>(); + for (byte[] payload : store.pendingEventPayloads()) { + keys.add(LDValue.parse(new String(payload, Charset.forName("UTF-8"))).get("key").stringValue()); + } + return keys; + } + + private void overwrite(File file, byte[] bytes) throws IOException { + try (FileOutputStream out = new FileOutputStream(file, false)) { + out.write(bytes); + } + } + + private byte[] readAll(File file) throws IOException { + return java.nio.file.Files.readAllBytes(file.toPath()); + } + + private List filesNamed(String prefix) { + List matching = new ArrayList<>(); + File[] files = sharedDirectory().listFiles(); + if (files != null) { + for (File file : files) { + if (file.getName().startsWith(prefix)) { + matching.add(file); + } + } + } + return matching; + } + + // MARK: each process writes only its own log + + @Test + public void eachProcessAppendsToItsOwnLog() { + EventStore main = storeFor(); + EventStore service = storeFor(); + + main.stage(event("main-1")); + main.commit(); + service.stage(event("service-1")); + service.commit(); + main.stage(event("main-2")); + main.commit(); + + // Two logs, not one. This is the property that makes a spliced frame impossible: only one + // process ever has the file open for writing. + assertEquals(2, filesNamed(DiskEventLog.PREFIX).size()); + assertEquals(Arrays.asList("main-1", "main-2"), visibleKeys(main)); + assertEquals(Arrays.asList("service-1"), visibleKeys(service)); + } + + @Test + public void aLogIsNamedForAUuid() { + EventStore main = storeFor(); + main.stage(event("main-1")); + main.commit(); + + List logs = filesNamed(DiskEventLog.PREFIX); + assertEquals(1, logs.size()); + String uuid = logs.get(0).getName().substring(DiskEventLog.PREFIX.length()); + assertEquals(uuid, UUID.fromString(uuid).toString()); + assertTrue("a log was left under the name it is opened under", filesNamed("new-").isEmpty()); + } + + // MARK: recovery takes what no live process holds + + /** + * What the kernel does to a store's log when its process dies: closes the descriptor, which releases the + * lock on it. Everything the store staged has been committed by then. + */ + private static void processDies(EventStore store) { + store.close(); + } + + @Test + public void recoveryLeavesAnotherProcessOpenLogAlone() { + EventStore service = storeFor(); + service.stage(event("service-1")); + service.commit(); + + // The main process starts while the service process is still alive and still appending. Its + // recovery must not take the log out from under it. + EventStore main = storeFor(); + main.preparePayloadsFromInterruptedLogs(); + + assertEquals("the other process's log was taken away", 1, filesNamed(DiskEventLog.PREFIX).size()); + assertTrue("another process's events were claimed as recovered", main.pendingBatches().isEmpty()); + + // And the service process can still append to it and read back both events. + service.stage(event("service-2")); + service.commit(); + assertEquals(Arrays.asList("service-1", "service-2"), visibleKeys(service)); + } + + @Test + public void recoveryLeavesALogWhoseLockIsHeldAlone() throws IOException { + EventStore service = storeFor(); + service.stage(event("service-1")); + service.commit(); + processDies(service); + File log = filesNamed(DiskEventLog.PREFIX).get(0); + + // A process this one knows nothing about, holding the lock on the log as its owner does. + try (RandomAccessFile owner = new RandomAccessFile(log, "rw"); + FileLock held = owner.getChannel().lock()) { + EventStore main = storeFor(); + main.preparePayloadsFromInterruptedLogs(); + assertTrue("a log a live process holds was recovered", main.pendingBatches().isEmpty()); + assertTrue(log.exists()); + } + + EventStore main = storeFor(); + main.preparePayloadsFromInterruptedLogs(); + assertEquals(1, main.pendingBatches().size()); + } + + @Test + public void anyProcessRecoversALogWhoseWriterIsGone() { + EventStore service = storeFor(); + service.stage(event("before-the-crash")); + service.commit(); + processDies(service); + + // Not the process that wrote it: a service process may never run again. + EventStore main = storeFor(); + main.preparePayloadsFromInterruptedLogs(); + + List recovered = main.pendingBatches(); + assertEquals(1, recovered.size()); + assertEquals(1, recovered.get(0).eventCount); + assertEquals(Arrays.asList("before-the-crash"), visibleKeys(main)); + assertTrue("the recovered log should no longer be open", filesNamed(DiskEventLog.PREFIX).isEmpty()); + } + + @Test + public void recoveryTakesEveryLogLeftBehind() { + EventStore firstRun = storeFor(); + firstRun.stage(event("first-run")); + firstRun.commit(); + processDies(firstRun); + EventStore service = storeFor(); + service.stage(event("service-1")); + service.stage(event("service-2")); + service.commit(); + processDies(service); + + EventStore secondRun = storeFor(); + secondRun.preparePayloadsFromInterruptedLogs(); + + assertEquals(2, secondRun.pendingBatches().size()); + assertEquals(3, secondRun.getPendingEventCount()); + assertTrue(filesNamed(DiskEventLog.PREFIX).isEmpty()); + } + + @Test + public void aCommitBeforeStartupRecoveryKeepsThisRunsEventsOutOfWhatItRecovers() { + // With IMMEDIATE the first commit happens on the caller's thread, and can beat the recovery the + // processor queues on its own thread at startup. + EventStore firstRun = storeFor(); + firstRun.stage(event("previous-run")); + firstRun.commit(); + processDies(firstRun); + + EventStore secondRun = storeFor(); + secondRun.stage(event("this-run")); + secondRun.commit(); + secondRun.preparePayloadsFromInterruptedLogs(); + + List recovered = secondRun.pendingBatches(); + assertEquals(1, recovered.size()); + assertEquals("this run's event was counted as a previous run's", 1, recovered.get(0).eventCount); + assertEquals(1, filesNamed(DiskEventLog.PREFIX).size()); + assertEquals(Arrays.asList("previous-run", "this-run"), visibleKeys(secondRun)); + } + + @Test + public void aStoreThatClosedItsLogAppendsToANewOne() { + EventStore store = storeFor(); + store.stage(event("before-close")); + store.commit(); + File first = filesNamed(DiskEventLog.PREFIX).get(0); + store.close(); + + // Unlocked once closed, and so another process's to recover from then on. + store.stage(event("after-close")); + store.commit(); + + assertEquals(2, filesNamed(DiskEventLog.PREFIX).size()); + assertTrue(first.exists()); + assertEquals(Arrays.asList("after-close"), visibleKeys(store)); + } + + @Test + public void aStoreWithoutPersistenceLeavesWhatAPersistentRunLeftBehindAlone() { + EventStore persistentRun = storeFor(); + persistentRun.stage(event("closed-batch")); + closeOne(persistentRun); + persistentRun.stage(event("open-log")); + persistentRun.commit(); + processDies(persistentRun); + + EventStore withoutPersistence = + new EventStore(sharedDirectory(), CAPACITY, false, logger); + withoutPersistence.preparePayloadsFromInterruptedLogs(); + assertTrue("a store without persistence picked up a previous run's events", + withoutPersistence.pendingBatches().isEmpty()); + assertTrue(visibleKeys(withoutPersistence).isEmpty()); + + withoutPersistence.stage(event("in-memory")); + EventStore.Batch own = closeOne(withoutPersistence); + assertNotNull(own); + assertEquals(Arrays.asList(own), withoutPersistence.pendingBatches()); + + assertEquals(1, filesNamed("ready-").size()); + assertEquals(1, filesNamed(DiskEventLog.PREFIX).size()); + + // Still there for the next run that has persistence turned on. + EventStore persistentAgain = storeFor(); + persistentAgain.preparePayloadsFromInterruptedLogs(); + assertEquals(2, persistentAgain.pendingBatches().size()); + } + + @Test + public void aStoreWithoutPersistenceAsksTheFilesystemNothingEvenAboutABatchItDoesNotHold() + throws IOException { + AtomicInteger reads = new AtomicInteger(); + EventStore store = new EventStore(sharedDirectory(), CAPACITY, false, logger) { + @Override + InputStream openForReading(File file) throws IOException { + reads.incrementAndGet(); + return super.openForReading(file); + } + }; + + store.stage(event("held")); + EventStore.Batch held = closeOne(store); + assertNotNull(held); + assertEquals(Arrays.asList(held), store.pendingBatches()); + assertEquals(LDValue.of("held"), + LDValue.parse(new String(store.body(held), Charset.forName("UTF-8"))).get(0).get("key")); + store.remove(held); + + // A batch this store never closed, which is what sends the paths that form a file name down the + // branch that would otherwise ask the filesystem about it. + EventStore.Batch stranger = new EventStore.Batch(UUID.randomUUID().toString(), 1); + assertNull(store.body(stranger)); + store.remove(stranger); + + assertEquals(0, reads.get()); + assertFalse("a store without persistence created its directory", sharedDirectory().exists()); + } + + @Test + public void oneProcessRecoveringDoesNotDisturbWhatAnotherHasAlreadyClosed() { + EventStore service = storeFor(); + service.stage(event("service-1")); + EventStore.Batch fromService = closeOne(service); + assertNotNull(fromService); + + // Recovery runs when a store is constructed, before that process has recorded anything of its own, + // which is the only time it is correct to go looking for a log to close. + EventStore main = storeFor(); + main.preparePayloadsFromInterruptedLogs(); + main.stage(event("main-1")); + main.commit(); + + // The service process's batch is untouched and still deliverable, by either of them. + assertEquals(1, filesNamed("ready-").size()); + assertTrue(main.pendingBatches().contains(fromService)); + } + + // MARK: closed batches are shared on purpose + + @Test + public void aBatchClosedByOneProcessIsDeliverableByAnother() throws IOException { + // The point of sharing: a service process that records events and is then killed may never run + // again, so a batch only it could see would never be delivered. + EventStore service = storeFor(); + service.stage(event("recorded-by-the-service")); + EventStore.Batch batch = closeOne(service); + assertNotNull(batch); + + EventStore main = storeFor(); + List pending = main.pendingBatches(); + + assertEquals(1, pending.size()); + EventStore.Batch seenByMain = pending.get(0); + // The same payload ID, which is what keeps two processes delivering this batch from counting the + // events twice upstream. + assertEquals(batch.payloadId, seenByMain.payloadId); + assertEquals(batch.eventCount, seenByMain.eventCount); + + byte[] body = main.body(seenByMain); + assertNotNull("the other process could not read the batch", body); + LDValue parsed = LDValue.parse(new String(body, Charset.forName("UTF-8"))); + assertEquals(1, parsed.size()); + assertEquals(LDValue.of("recorded-by-the-service"), parsed.get(0).get("key")); + } + + @Test + public void aClosedBatchIsCountedFromItsNameWithoutReadingItBack() throws IOException { + EventStore main = storeFor(); + main.stage(event("first")); + main.stage(event("second")); + EventStore.Batch batch = closeOne(main); + assertNotNull(batch); + + // Nothing appends to a batch once it is closed, so the count its name carries still holds and + // listing it has no reason to go to the disk. Damaging the file is how the test tells the two + // apart: a listing that read would refuse this batch. + List ready = filesNamed("ready-"); + assertEquals(1, ready.size()); + assertEquals("ready-" + batch.payloadId + ".2", ready.get(0).getName()); + overwrite(ready.get(0), "not an event log".getBytes(Charset.forName("UTF-8"))); + + // The process that closed it and one that did not both count it from the name alone. + for (EventStore store : Arrays.asList(main, storeFor())) { + List pending = store.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(batch.payloadId, pending.get(0).payloadId); + assertEquals(2, pending.get(0).eventCount); + } + + // The damage is found where the batch is read to be sent, and reads as a loss rather than as a + // failure to retry, so the delivery removes it instead of trying it forever, and says it lost events. + try { + main.body(batch); + fail("a damaged batch read as something to send or as nothing to lose"); + } catch (EventStore.UnreadableBatchException expected) { + // what a delivery takes as lost + } + } + + @Test + public void aBatchWhoseFirstFrameIsDamagedIsLostRatherThanEmpty() throws IOException { + EventStore main = storeFor(); + main.stage(event("damaged")); + EventStore.Batch batch = closeOne(main); + assertNotNull(batch); + + File file = filesNamed("ready-").get(0); + byte[] damaged = readAll(file); + damaged[EventLogFormat.FILE_HEADER_SIZE + EventLogFormat.FRAME_HEADER_SIZE] ^= 0x01; + overwrite(file, damaged); + + try { + main.body(batch); + fail("a batch with a damaged frame read as nothing to lose"); + } catch (EventStore.UnreadableBatchException expected) { + // what a delivery takes as lost + } + } + + /** + * Renames a batch to a name without a count, which is what a recovery that could not read a log + * leaves behind for a listing to count. + */ + private File asUncountedName(File batch) { + String name = batch.getName(); + File renamed = new File(batch.getParentFile(), name.substring(0, name.lastIndexOf('.'))); + assertTrue("could not rename " + name, batch.renameTo(renamed)); + return renamed; + } + + @Test + public void aBatchNamedWithoutACountIsReadOnceAndThenRemembered() throws IOException { + EventStore previousVersion = storeFor(); + previousVersion.stage(event("left-by-a-previous-version")); + closeOne(previousVersion); + assertEquals(1, filesNamed("ready-").size()); + asUncountedName(filesNamed("ready-").get(0)); + + AtomicInteger reads = new AtomicInteger(); + EventStore main = new EventStore(sharedDirectory(), CAPACITY, true, logger) { + @Override + InputStream openForReading(File file) throws IOException { + reads.incrementAndGet(); + return super.openForReading(file); + } + }; + + List pending = main.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(1, pending.get(0).eventCount); + assertEquals("a batch with no count in its name was not read", 1, reads.get()); + + // Counted once and remembered, rather than read again by every listing for the rest of the session. + assertEquals(pending, main.pendingBatches()); + assertEquals(1, reads.get()); + + // And still deliverable under the name it has, which is not the one its count would give it. + assertNotNull(main.body(pending.get(0))); + main.remove(pending.get(0)); + assertTrue(filesNamed("ready-").isEmpty()); + } + + @Test + public void aPayloadIdEndingInZeroLedDigitsIsNotTakenForACount() { + EventStore previousVersion = storeFor(); + previousVersion.stage(event("left-by-a-previous-version")); + closeOne(previousVersion); + File batch = filesNamed("ready-").get(0); + // Its last group fits an int once the leading zeros are dropped. + String payloadId = "6F9619FF-8B86-D011-B42D-000012345678"; + assertTrue(batch.renameTo(new File(batch.getParentFile(), "ready-" + payloadId))); + + List pending = storeFor().pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(payloadId, pending.get(0).payloadId); + assertEquals(1, pending.get(0).eventCount); + } + + @Test + public void aNameWithAFieldAfterTheCountIsReadByPosition() throws IOException { + EventStore later = storeFor(); + later.stage(event("first")); + later.stage(event("second")); + EventStore.Batch batch = closeOne(later); + File file = filesNamed("ready-").get(0); + // What a later version might add, which this one has no use for. + assertTrue(file.renameTo(new File(file.getParentFile(), file.getName() + ".v2-something"))); + + List pending = storeFor().pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(batch.payloadId, pending.get(0).payloadId); + assertEquals(2, pending.get(0).eventCount); + } + + @Test + public void aCountThatDoesNotParseIsCountedByReadingTheBatch() { + EventStore previous = storeFor(); + previous.stage(event("only")); + EventStore.Batch batch = closeOne(previous); + File file = asUncountedName(filesNamed("ready-").get(0)); + assertTrue(file.renameTo(new File(file.getParentFile(), file.getName() + ".not-a-count"))); + + List pending = storeFor().pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(batch.payloadId, pending.get(0).payloadId); + assertEquals(1, pending.get(0).eventCount); + } + + @Test + public void aBatchTheOtherProcessAlreadyDeliveredReadsAsNothingToSend() throws IOException { + EventStore service = storeFor(); + service.stage(event("delivered-once")); + EventStore.Batch batch = closeOne(service); + assertNotNull(batch); + + EventStore main = storeFor(); + main.remove(batch); // main delivered it first + + // The other process asking for the body gets nothing, which its caller treats as already + // delivered rather than as a failure to retry. + assertNull(service.body(batch)); + } + + @Test + public void bothProcessesRemovingTheSameBatchIsHarmless() { + EventStore service = storeFor(); + service.stage(event("delivered-twice")); + EventStore.Batch batch = closeOne(service); + assertNotNull(batch); + + EventStore main = storeFor(); + main.remove(batch); + service.remove(batch); // a remove of a file that is already gone + + assertTrue(filesNamed("ready-").isEmpty()); + assertEquals(0, main.pendingBatches().size()); + assertEquals(0, service.pendingBatches().size()); + } + + @Test + public void aProcessCountsAnotherProcessPendingBatchesAgainstItsOwnCapacity() { + // Deliberately conservative, and documented as such: the alternative is cross-process accounting, + // which would need cross-process locking to be correct. + EventStore service = storeFor(); + for (int i = 0; i < 3; i++) { + service.stage(event("service-" + i)); + } + closeOne(service); + + EventStore main = storeFor(); + main.pendingBatches(); // this is what reconciles the count with the directory + + assertEquals(3, main.getPendingEventCount()); + } + + // MARK: a read that fails is no reason to delete + + @Test + public void recoveryReadsWhatAPreviousRunLeftWithoutHoldingUpACommit() throws Exception { + // At IMMEDIATE a track commits on the caller's thread, and recovery runs at startup; a crash can + // leave many logs of up to a batch each. + EventStore firstRun = storeFor(); + firstRun.stage(event("before-the-crash")); + firstRun.commit(); + processDies(firstRun); + + CountDownLatch readStarted = new CountDownLatch(1); + CountDownLatch readResumes = new CountDownLatch(1); + EventStore secondRun = new EventStore(sharedDirectory(), CAPACITY, true, logger) { + @Override + InputStream openForReading(File file) throws IOException { + if (file.getName().startsWith(DiskEventLog.PREFIX)) { + readStarted.countDown(); + try { + readResumes.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + return super.openForReading(file); + } + }; + ExecutorService recovery = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + Future recovered = recovery.submit(secondRun::preparePayloadsFromInterruptedLogs); + assertTrue(readStarted.await(5, TimeUnit.SECONDS)); + + Future committed = caller.submit(() -> { + secondRun.stage(event("this-run")); + secondRun.commit(); + }); + committed.get(2, TimeUnit.SECONDS); + + readResumes.countDown(); + recovered.get(5, TimeUnit.SECONDS); + assertEquals(1, secondRun.pendingBatches().size()); + assertEquals(new HashSet<>(Arrays.asList("before-the-crash", "this-run")), + new HashSet<>(visibleKeys(secondRun))); + } finally { + readResumes.countDown(); + recovery.shutdownNow(); + caller.shutdownNow(); + } + } + + /** + * A store whose reads fail while {@code failing} is set, the way they do in a process that has run + * out of file descriptors: the files are intact and readable again once some are freed. + */ + private EventStore storeWithFailingReads(AtomicBoolean failing) { + return new EventStore(sharedDirectory(), CAPACITY, true, logger) { + @Override + InputStream openForReading(File file) throws IOException { + if (failing.get()) { + throw new FileNotFoundException(file + " (Too many open files)"); + } + return super.openForReading(file); + } + }; + } + + @Test + public void aBatchNamedWithoutACountThatCannotBeReadIsLeftForALaterListing() { + EventStore service = storeFor(); + service.stage(event("recorded-by-the-service")); + closeOne(service); + asUncountedName(filesNamed("ready-").get(0)); + + AtomicBoolean failing = new AtomicBoolean(true); + EventStore main = storeWithFailingReads(failing); + assertTrue(main.pendingBatches().isEmpty()); + assertEquals("a batch that could not be read was deleted", 1, filesNamed("ready-").size()); + + failing.set(false); + List pending = main.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(1, pending.get(0).eventCount); + } + + @Test + public void aBatchCountedByItsNameIsListedEvenWhileReadsAreFailing() { + EventStore service = storeFor(); + service.stage(event("recorded-by-the-service")); + closeOne(service); + + // The read a listing used to need is the one a process out of file descriptors cannot make. With + // the count in the name there is nothing to read until the batch is sent, and a body that cannot + // be read is a delivery to retry rather than a batch that is invisible until it can be. + AtomicBoolean failing = new AtomicBoolean(true); + EventStore main = storeWithFailingReads(failing); + List pending = main.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(1, pending.get(0).eventCount); + try { + main.body(pending.get(0)); + fail("a batch that could not be read was reported as having nothing to send"); + } catch (IOException expected) { + // what the processor retries on + } + } + + @Test + public void aBodyThatCannotBeReadIsAFailureToRetryRatherThanNothingToSend() throws IOException { + AtomicBoolean failing = new AtomicBoolean(false); + EventStore main = storeWithFailingReads(failing); + main.stage(event("kept")); + EventStore.Batch batch = closeOne(main); + assertNotNull(batch); + + failing.set(true); + try { + main.body(batch); + fail("a batch that could not be read was reported as having nothing to send"); + } catch (IOException expected) { + // what the processor retries on + } + assertEquals(1, filesNamed("ready-").size()); + + failing.set(false); + byte[] body = main.body(batch); + assertNotNull(body); + assertEquals(LDValue.of("kept"), + LDValue.parse(new String(body, Charset.forName("UTF-8"))).get(0).get("key")); + } + + @Test + public void aPreviousRunsLogThatCannotBeReadIsKeptForALaterListing() { + EventStore firstRun = storeFor(); + firstRun.stage(event("before-the-crash")); + firstRun.commit(); + processDies(firstRun); + + AtomicBoolean failing = new AtomicBoolean(true); + EventStore secondRun = storeWithFailingReads(failing); + secondRun.preparePayloadsFromInterruptedLogs(); + // Closed off uncounted for a listing to count, and not deleted. + assertTrue(filesNamed(DiskEventLog.PREFIX).isEmpty()); + assertEquals(1, filesNamed("ready-").size()); + + secondRun.stage(event("this-run")); + secondRun.commit(); + failing.set(false); + List pending = secondRun.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(1, pending.get(0).eventCount); + assertEquals(Arrays.asList("before-the-crash", "this-run"), visibleKeys(secondRun)); + } + + // MARK: a write that fails partway + + /** + * A store whose writes land whole until {@code whole} of them have, after which each gets all but + * the last few bytes onto the disk and fails, as one does when the device fills mid-write. + */ + private EventStore storeFailingPartwayAfter(int whole) { + AtomicInteger writes = new AtomicInteger(); + return new EventStore(sharedDirectory(), CAPACITY, true, logger) { + @Override + FileOutputStream openForAppending(File file) throws IOException { + return new FileOutputStream(file, true) { + @Override + public void write(byte[] bytes) throws IOException { + if (writes.incrementAndGet() <= whole) { + super.write(bytes); + return; + } + super.write(bytes, 0, bytes.length - 3); + throw new IOException("No space left on device"); + } + }; + } + }; + } + + @Test + public void eventsBeingWrittenStillCountAgainstCapacity() throws Exception { + // A commit takes the staged events out of the buffer and writes them outside the buffer's lock. + // They used to be counted nowhere until the write returned, and capacity admitted that many more. + CountDownLatch writing = new CountDownLatch(1); + CountDownLatch letWriteFinish = new CountDownLatch(1); + EventStore store = new EventStore(sharedDirectory(), 2, true, logger) { + @Override + FileOutputStream openForAppending(File file) throws IOException { + writing.countDown(); + try { + letWriteFinish.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return super.openForAppending(file); + } + }; + assertTrue(store.stage(event("first"), false)); + assertTrue(store.stage(event("second"), false)); + + Thread committer = new Thread(store::commit); + committer.start(); + try { + assertTrue("the commit never reached the disk", writing.await(5, TimeUnit.SECONDS)); + + assertEquals(2, store.getPendingEventCount()); + assertFalse("admitted past capacity while the write was under way", + store.stage(event("third"), false)); + } finally { + letWriteFinish.countDown(); + committer.join(5_000); + } + + assertEquals(2, store.getPendingEventCount()); + assertFalse(store.stage(event("third"), false)); + assertEquals(Arrays.asList("first", "second"), visibleKeys(store)); + } + + @Test + public void removingABatchTwiceStopsCountingItOnce() { + // Deliveries can overlap, and two of them can both get to the same batch. + for (boolean persist : new boolean[] {true, false}) { + EventStore store = new EventStore(new File(sharedDirectory(), "persist-" + persist), + CAPACITY, persist, logger); + store.stage(event("first")); + EventStore.Batch first = closeOne(store); + store.stage(event("second")); + store.closeBatch(); + store.pendingBatches(); + + store.remove(first); + store.remove(first); + + assertEquals("persisting: " + persist, 1, store.getPendingEventCount()); + } + } + + @Test + public void aDeliveredBatchWhoseFileWillNotDeleteIsNotListedAgain() { + EventStore store = storeFor(); + store.stage(event("first")); + EventStore.Batch first = closeOne(store); + store.stage(event("second")); + EventStore.Batch second = closeOne(store); + assertNotNull(first); + assertNotNull(second); + + File directory = sharedDirectory(); + // A directory with no write permission is one its entries cannot be unlinked from, which is what the + // filesystem refusing a delete looks like from here. + assertTrue("could not make the directory read-only", directory.setWritable(false)); + try { + store.remove(first); + + // Delivered, so it neither counts against capacity nor comes back for another delivery, even + // though its file is still sitting there. + assertEquals(Arrays.asList(second), store.pendingBatches()); + assertEquals(1, store.getPendingEventCount()); + } finally { + assertTrue("could not make the directory writable again", directory.setWritable(true)); + } + + // The listing that can delete it does, rather than leaving it for a run that would send it again. + store.pendingBatches(); + assertTrue(filesNamed("ready-" + first.payloadId).isEmpty()); + } + + @Test + public void aLogThatCouldNotBeClosedStaysOpenAndIsClosedWholeByTheNextClose() throws IOException { + EventStore store = storeFor(); + store.stage(event("first")); + store.stage(event("second")); + store.commit(); + + File directory = sharedDirectory(); + // A directory with no write permission is one its entries cannot be renamed in, which is what the + // filesystem refusing to close the log looks like from here. + assertTrue("could not make the directory read-only", directory.setWritable(false)); + try { + assertTrue("closed a log with the directory read-only", store.closeBatch().isEmpty()); + } finally { + assertTrue("could not make the directory writable again", directory.setWritable(true)); + } + assertEquals(2, store.getPendingEventCount()); + + // Appended to the same log, so one batch holds all three and is named for as many. + store.stage(event("third")); + EventStore.Batch closed = closeOne(store); + assertEquals(3, closed.eventCount); + assertEquals(Arrays.asList("first", "second", "third"), keysIn(store.body(closed))); + assertTrue("a log was left behind", filesNamed("pending-").isEmpty()); + + List listed = storeFor().pendingBatches(); + assertEquals(1, listed.size()); + assertEquals(3, listed.get(0).eventCount); + } + + @Test + public void aBatchAnotherProcessDeliveredStopsCountingOnce() { + EventStore main = storeFor(); + EventStore service = storeFor(); + main.stage(event("first")); + EventStore.Batch first = closeOne(main); + main.stage(event("second")); + main.closeBatch(); + + // The other process delivers it, and this one lists again before its own delivery of it finishes. + service.remove(first); + main.pendingBatches(); + main.remove(first); + + assertEquals(1, main.getPendingEventCount()); + } + + @Test + public void aListingDoesNotLoseOrDoubleABatchClosedWhileItRuns() throws Exception { + for (boolean persist : new boolean[] {true, false}) { + EventStore store = new EventStore(new File(sharedDirectory(), "persist-" + persist), + 100_000, persist, logger); + AtomicBoolean done = new AtomicBoolean(); + Thread lister = new Thread(() -> { + while (!done.get()) { + store.pendingBatches(); + // Without it the listings can hold the locks so continuously that the closes starve. + Thread.yield(); + } + }); + lister.start(); + try { + // Checked after every close, because the next listing would correct a miscount before + // the end of the loop could see it. + for (int i = 1; i <= 150; i++) { + store.stage(event("event-" + i)); + store.closeBatch(); + assertEquals("persisting: " + persist, i, store.getPendingEventCount()); + } + } finally { + done.set(true); + lister.join(5_000); + } + } + } + + /** Closes what a test expects to be exactly one batch, as anything short of a failed write leaves. */ + private static EventStore.Batch closeOne(EventStore store) { + List closed = store.closeBatch(); + assertEquals(1, closed.size()); + return closed.get(0); + } + + private List keysIn(byte[] body) { + List keys = new ArrayList<>(); + for (LDValue event : LDValue.parse(new String(body, Charset.forName("UTF-8"))).values()) { + keys.add(event.get("key").stringValue()); + } + return keys; + } + + @Test + public void framesAWriteLandedBeforeItFailedAreSentOnceNotFromTheLogAndMemoryBoth() + throws IOException { + // The file header and the first event land. The next write gets a whole frame and part of + // another onto the disk before the device is full. + EventStore store = storeFailingPartwayAfter(2); + + store.stage(event("written")); + store.commit(); + store.stage(event("landed")); + store.stage(event("torn")); + store.commit(); + assertFalse(store.isPersisting()); + + List closed = store.closeBatch(); + assertEquals("the log's batch and memory's, both reported", 2, closed.size()); + assertEquals(Arrays.asList("written"), keysIn(store.body(closed.get(0)))); + assertEquals(Arrays.asList("landed", "torn"), keysIn(store.body(closed.get(1)))); + + List> bodies = new ArrayList<>(); + for (EventStore.Batch batch : store.pendingBatches()) { + bodies.add(keysIn(store.body(batch))); + } + assertEquals(Arrays.asList(Arrays.asList("written"), Arrays.asList("landed", "torn")), bodies); + } + + @Test + public void summariesAFailedWriteLeftInMemoryStillDoNotCountAsEvents() throws IOException { + // The file header and the first commit land; the second fails partway into its summary. + EventStore store = storeFailingPartwayAfter(2); + store.stage(event("written")); + store.stage(event("summary-written"), true); + store.commit(); + store.stage(event("landed")); + store.stage(event("summary-put-back"), true); + store.commit(); + assertFalse(store.isPersisting()); + assertEquals(2, store.getPendingEventCount()); + + List closed = store.closeBatch(); + assertEquals(2, closed.size()); + assertEquals(Arrays.asList("written", "summary-written"), keysIn(store.body(closed.get(0)))); + assertEquals(Arrays.asList("landed", "summary-put-back"), keysIn(store.body(closed.get(1)))); + assertEquals(2, store.getPendingEventCount()); + store.pendingBatches(); + assertEquals(2, store.getPendingEventCount()); + } + + @Test + public void sendsNoFramesAWriteThatFailedLeftBehindWhenItCouldNotBeCutBack() throws IOException { + EventStore store = storeFor(); + store.stage(event("written")); + EventStore.Batch closed = closeOne(store); + File[] files = sharedDirectory().listFiles((dir, name) -> name.startsWith("ready-")); + assertNotNull(files); + assertEquals(1, files.length); + // Whole frames past the one the batch was closed holding, which memory delivers instead. + ByteArrayOutputStream leftBehind = new ByteArrayOutputStream(); + EventLogFormat.writeFrame(leftBehind, event("from-memory")); + try (FileOutputStream log = new FileOutputStream(files[0], true)) { + log.write(leftBehind.toByteArray()); + } + + assertEquals(Arrays.asList("written"), keysIn(store.body(closed))); + EventStore other = storeFor(); + List listed = other.pendingBatches(); + assertEquals(1, listed.size()); + assertEquals(1, listed.get(0).eventCount); + assertEquals(Arrays.asList("written"), keysIn(other.body(listed.get(0)))); + } + + @Test + public void theNextRunHasNothingToRecoverFromAFirstWriteThatFailedPartway() throws IOException { + // The file header lands, and then a whole frame and part of another before the device is full. + EventStore store = storeFailingPartwayAfter(1); + + store.stage(event("landed")); + store.stage(event("torn")); + store.commit(); + EventStore.Batch fromMemory = closeOne(store); + assertNotNull(fromMemory); + assertEquals(Arrays.asList("landed", "torn"), keysIn(store.body(fromMemory))); + + // Delivered from memory this run, so the next must not find any of it again in the log. + EventStore nextRun = storeFor(); + nextRun.preparePayloadsFromInterruptedLogs(); + assertEquals(0, nextRun.pendingBatches().size()); + } + + // MARK: the two together + + @Test + public void concurrentRecordingFromTwoProcessesLosesAndCorruptsNothing() throws Exception { + int eventsPerProcess = 200; + // Capacity well clear of what the test records, so that a dropped event means a real loss rather + // than the store correctly refusing to exceed its limit. + EventStore main = storeFor(10 * eventsPerProcess); + EventStore service = storeFor(10 * eventsPerProcess); + + CountDownLatch startTogether = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(2); + Thread mainThread = recordingThread(main, "main", eventsPerProcess, startTogether, finished); + Thread serviceThread = recordingThread(service, "service", eventsPerProcess, startTogether, finished); + mainThread.start(); + serviceThread.start(); + startTogether.countDown(); + assertTrue(finished.await(20, TimeUnit.SECONDS)); + + // Every event either process recorded is readable, and every frame parses - a spliced frame would + // show up as a parse failure or a missing event, because a torn frame ends the reader. + Set everything = new HashSet<>(visibleKeys(main)); + everything.addAll(visibleKeys(service)); + assertEquals(2 * eventsPerProcess, everything.size()); + for (int i = 0; i < eventsPerProcess; i++) { + assertTrue("lost main-" + i, everything.contains("main-" + i)); + assertTrue("lost service-" + i, everything.contains("service-" + i)); + } + } + + private Thread recordingThread( + EventStore store, + String prefix, + int count, + CountDownLatch startTogether, + CountDownLatch finished + ) { + return new Thread(() -> { + try { + startTogether.await(); + for (int i = 0; i < count; i++) { + store.stage(event(prefix + "-" + i)); + // Committing on every event maximizes the chance of two writes overlapping, which is + // the interleaving this is looking for. + store.commit(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + finished.countDown(); + } + }); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/OutboundEventBufferSerializationTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/OutboundEventBufferSerializationTest.java new file mode 100644 index 000000000..e0b272ba2 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/OutboundEventBufferSerializationTest.java @@ -0,0 +1,167 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.logging.LDLogAdapter; +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.internal.events.EventOutputFormatter; +import com.launchdarkly.sdk.internal.events.EventSummarizer; +import com.launchdarkly.sdk.internal.events.EventsConfiguration; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.StringWriter; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * The serialization the on-disk event log is built on: one event in, one JSON object out. + *

+ * These are deliberately narrow. Everything the log does depends on a frame holding exactly one + * event object, so the assumption is checked here rather than inferred from a delivered payload. + */ +public class OutboundEventBufferSerializationTest { + private static final LDContext CONTEXT = LDContext.create("user-key"); + private static final String FLAG_KEY = "flag-key"; + + private final LDLogAdapter logAdapter = Logs.none(); + + private OutboundEventBuffer makeBuffer() { + // Unbounded cardinality, so these stay about serialization and nothing else. + return new OutboundEventBuffer(false, Collections.emptyList(), true, Integer.MAX_VALUE, + LDLogger.withAdapter(logAdapter, "")); + } + + private LDValue parse(byte[] serialized) { + return LDValue.parse(new String(serialized, Charset.forName("UTF-8"))); + } + + @Test + public void customEventSerializesToASingleObject() { + byte[] serialized = makeBuffer().serialize( + new Event.Custom(1000, "an-event", CONTEXT, LDValue.of("data"), 2.5)); + + assertNotNull(serialized); + // The bytes are one object, with no array around it, so that a payload can be assembled by + // joining frames with commas. + assertEquals('{', serialized[0]); + assertEquals('}', serialized[serialized.length - 1]); + + LDValue event = parse(serialized); + assertEquals(LDValue.of("custom"), event.get("kind")); + assertEquals(LDValue.of("an-event"), event.get("key")); + assertEquals(LDValue.of(2.5), event.get("metricValue")); + } + + @Test + public void identifyEventSerializesToASingleObject() { + byte[] serialized = makeBuffer().serialize(new Event.Identify(1000, CONTEXT)); + + assertNotNull(serialized); + LDValue event = parse(serialized); + assertEquals(LDValue.of("identify"), event.get("kind")); + assertEquals(LDValue.of(CONTEXT.getKey()), event.get("context").get("key")); + } + + @Test + public void featureEventSerializesToASingleObject() { + Event.FeatureRequest event = new Event.FeatureRequest(1000, FLAG_KEY, CONTEXT, 10, 1, + LDValue.of(true), LDValue.of(false), null, null, true, null, false); + + byte[] serialized = makeBuffer().serialize(event); + + assertNotNull(serialized); + LDValue parsed = parse(serialized); + assertEquals(LDValue.of("feature"), parsed.get("kind")); + assertEquals(LDValue.of(FLAG_KEY), parsed.get("key")); + assertEquals(LDValue.of(1), parsed.get("variation")); + } + + @Test + public void summariesSerializeOneObjectEachAndLeaveTheCountersEmpty() { + OutboundEventBuffer buffer = makeBuffer(); + buffer.summarize(new Event.FeatureRequest(1000, FLAG_KEY, CONTEXT, 10, 1, LDValue.of(true), + LDValue.of(false), null, null, false, null, false)); + buffer.summarize(new Event.FeatureRequest(1001, FLAG_KEY, CONTEXT, 10, 1, LDValue.of(true), + LDValue.of(false), null, null, false, null, false)); + + List summaries = buffer.serializeSummariesAndReset(); + + assertEquals(1, summaries.size()); + LDValue summary = parse(summaries.get(0)); + assertEquals(LDValue.of("summary"), summary.get("kind")); + int counted = 0; + for (LDValue counter : summary.get("features").get(FLAG_KEY).get("counters").values()) { + counted += counter.get("count").intValue(); + } + assertEquals(2, counted); + + // Reset, so the same evaluations are not summarized again by the next call. + assertTrue(buffer.serializeSummariesAndReset().isEmpty()); + } + + @Test + public void theContextLimitHoldsWhateverOrderContextsArriveIn() { + OutboundEventBuffer buffer = new OutboundEventBuffer(false, Collections.emptyList(), true, 1, + LDLogger.withAdapter(logAdapter, "")); + LDContext other = LDContext.create("other-key"); + + assertTrue(summarize(buffer, CONTEXT)); + assertTrue(summarize(buffer, CONTEXT)); + // Equal to the one counted but a different instance, so it misses the reference check and + // has to be found in the set rather than turned away. + assertTrue(summarize(buffer, LDContext.create(CONTEXT.getKey()))); + assertEquals(false, summarize(buffer, other)); + assertTrue(summarize(buffer, CONTEXT)); + assertEquals(false, summarize(buffer, other)); + + buffer.takeSummaries(); + + // The limit lifts with the drain, for the context seen last as much as any other. + assertTrue(summarize(buffer, other)); + assertEquals(false, summarize(buffer, CONTEXT)); + } + + private static boolean summarize(OutboundEventBuffer buffer, LDContext context) { + return buffer.summarize(1000, FLAG_KEY, 10, 1, LDValue.of(true), LDValue.of(false), context); + } + + @Test + public void serializingSummariesWithNothingCountedReturnsNothing() { + assertTrue(makeBuffer().serializeSummariesAndReset().isEmpty()); + } + + @Test + public void framesAssembleIntoTheSamePayloadTheFormatterWritesForTheWholeRun() throws Exception { + Event[] run = { + new Event.Custom(1000, "first", CONTEXT, LDValue.ofNull(), null), + new Event.Custom(1001, "second", CONTEXT, LDValue.ofNull(), null) + }; + + ByteArrayOutputStream frames = new ByteArrayOutputStream(); + for (byte[] serialized : makeBuffer().serializeAll(Arrays.asList(run))) { + EventLogFormat.writeFrame(frames, serialized); + } + byte[] body = EventLogFormat.assembleBody(EventLogFormat.withFileHeader(frames.toByteArray())); + + // Configured as makeBuffer() configures the formatter it wraps, so any difference is the + // framing's and not the configuration's. + EventOutputFormatter formatter = new EventOutputFormatter(new EventsConfiguration(false, 0, + null, 0, null, null, 1, null, 0, false, false, Collections.emptyList(), true)); + StringWriter expected = new StringWriter(); + assertEquals(run.length, formatter.writeOutputEvents(run, + Collections.emptyList(), expected)); + + assertNotNull(body); + assertEquals(LDValue.parse(expected.toString()), parse(body)); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/integrations/LDCrashHandlerTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/integrations/LDCrashHandlerTest.java new file mode 100644 index 000000000..97cb317ed --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/integrations/LDCrashHandlerTest.java @@ -0,0 +1,118 @@ +package com.launchdarkly.sdk.android.integrations; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.android.LDClientInterface; +import com.launchdarkly.sdk.android.LaunchDarklyException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class LDCrashHandlerTest { + private final List calls = new ArrayList<>(); + private final Thread crashed = new Thread("crashed"); + private final Throwable crash = new RuntimeException("boom"); + private final Thread.UncaughtExceptionHandler next = (thread, throwable) -> { + assertSame(crashed, thread); + assertSame(crash, throwable); + calls.add("next"); + }; + + private Thread.UncaughtExceptionHandler originalDefault; + + @Before + public void saveDefaultHandler() { + originalDefault = Thread.getDefaultUncaughtExceptionHandler(); + } + + @After + public void restoreDefaultHandler() { + Thread.setDefaultUncaughtExceptionHandler(originalDefault); + } + + @Test + public void deliversTheEventsBeforePassingTheCrashOn() { + LDClientInterface client = createMock(LDClientInterface.class); + expect(client.flushAndWait(eq(2000L), eq(TimeUnit.MILLISECONDS))).andAnswer(() -> { + calls.add("flush"); + return true; + }); + replay(client); + + new LDCrashHandler(() -> client, 2000, next).uncaughtException(crashed, crash); + + verify(client); + assertEquals(Arrays.asList("flush", "next"), calls); + } + + @Test + public void passesTheCrashOnWhenTheEventsAreNotDelivered() { + LDClientInterface client = createMock(LDClientInterface.class); + expect(client.flushAndWait(eq(2000L), eq(TimeUnit.MILLISECONDS))).andReturn(false); + replay(client); + + new LDCrashHandler(() -> client, 2000, next).uncaughtException(crashed, crash); + + verify(client); + assertEquals(Arrays.asList("next"), calls); + } + + @Test + public void passesTheCrashOnWhenTheClientWasNeverInitialized() { + new LDCrashHandler(() -> { + throw new LaunchDarklyException("LDClient.get() was called before init()!"); + }, 2000, next).uncaughtException(crashed, crash); + + assertEquals(Arrays.asList("next"), calls); + } + + @Test + public void passesTheCrashOnWhenTheFlushThrows() { + LDClientInterface client = createMock(LDClientInterface.class); + expect(client.flushAndWait(eq(2000L), eq(TimeUnit.MILLISECONDS))) + .andThrow(new IllegalStateException("the crash broke the client too")); + replay(client); + + new LDCrashHandler(() -> client, 2000, next).uncaughtException(crashed, crash); + + assertEquals(Arrays.asList("next"), calls); + } + + @Test + public void installsInFrontOfTheCurrentDefaultHandler() { + Thread.setDefaultUncaughtExceptionHandler(next); + + LDCrashHandler.install(2, TimeUnit.SECONDS); + Thread.UncaughtExceptionHandler installed = Thread.getDefaultUncaughtExceptionHandler(); + installed.uncaughtException(crashed, crash); + + assertTrue(installed instanceof LDCrashHandler); + assertEquals(Arrays.asList("next"), calls); + } + + @Test + public void installingTwiceKeepsTheFirstHandler() { + Thread.setDefaultUncaughtExceptionHandler(next); + + LDCrashHandler.install(2, TimeUnit.SECONDS); + Thread.UncaughtExceptionHandler first = Thread.getDefaultUncaughtExceptionHandler(); + LDCrashHandler.install(5, TimeUnit.SECONDS); + + assertSame(first, Thread.getDefaultUncaughtExceptionHandler()); + first.uncaughtException(crashed, crash); + assertEquals(Arrays.asList("next"), calls); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizerTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizerTest.java new file mode 100644 index 000000000..057db864f --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizerTest.java @@ -0,0 +1,99 @@ +package com.launchdarkly.sdk.internal.events; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.ContextKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; + +import org.junit.Test; + +import java.util.List; + +/** + * Checks the summarizer against the one it stands in for. The JVM these run on has + * {@code computeIfAbsent}, so the original can be run alongside it and the two compared directly. + */ +public class SameContextPerContextEventSummarizerTest { + private static final LDContext USER = LDContext.builder("user").name("User").set("plan", "pro").build(); + private static final LDContext DEVICE = LDContext.create(ContextKind.of("device"), "device"); + private static final LDContext MULTI = LDContext.createMulti(USER, DEVICE); + + private static void summarizeTheSame(EventSummarizerInterface a, EventSummarizerInterface b) { + Object[][] evaluations = { + {1000L, "flag-a", 1, 0, LDValue.of(true), LDValue.of(false), USER}, + {1001L, "flag-a", 1, 0, LDValue.of(true), LDValue.of(false), USER}, + {1002L, "flag-a", 2, 1, LDValue.of(false), LDValue.of(false), USER}, + {999L, "flag-b", -1, -1, LDValue.of("x"), LDValue.of("x"), DEVICE}, + {1003L, "flag-a", 1, 0, LDValue.of(true), LDValue.of(false), MULTI}, + {1004L, "flag-c", 3, 2, LDValue.of(2.5), LDValue.ofNull(), LDContext.builder("user") + .name("User").set("plan", "pro").build()}, + }; + for (Object[] e : evaluations) { + for (EventSummarizerInterface s : new EventSummarizerInterface[]{a, b}) { + s.summarizeEvent((Long) e[0], (String) e[1], (Integer) e[2], (Integer) e[3], + (LDValue) e[4], (LDValue) e[5], (LDContext) e[6]); + } + } + } + + @Test + public void summarizesAsTheOriginalDoes() { + PerContextEventSummarizer original = new PerContextEventSummarizer(); + SameContextPerContextEventSummarizer copy = new SameContextPerContextEventSummarizer(); + summarizeTheSame(original, copy); + + assertFalse(copy.isEmpty()); + List expected = original.getSummariesAndReset(); + List actual = copy.getSummariesAndReset(); + // An equal context built separately shares a summary, so three contexts, not four. + assertEquals(3, actual.size()); + assertEquals(expected, actual); + assertTrue(copy.isEmpty()); + assertTrue(copy.getSummariesAndReset().isEmpty()); + } + + @Test + public void restoresAsTheOriginalDoes() { + PerContextEventSummarizer original = new PerContextEventSummarizer(); + SameContextPerContextEventSummarizer copy = new SameContextPerContextEventSummarizer(); + summarizeTheSame(original, copy); + List taken = copy.getSummariesAndReset(); + original.restoreTo(original.getSummariesAndReset()); + copy.restoreTo(taken); + + summarizeTheSame(original, copy); + + assertEquals(original.getSummariesAndReset(), copy.getSummariesAndReset()); + } + + @Test + public void theContextSeenLastIsCountedAfreshAfterEveryReset() { + SameContextPerContextEventSummarizer copy = new SameContextPerContextEventSummarizer(); + PerContextEventSummarizer original = new PerContextEventSummarizer(); + for (Runnable reset : new Runnable[]{ + copy::getSummariesAndReset, + copy::clear, + () -> copy.restoreTo(java.util.Collections.emptyList())}) { + copy.summarizeEvent(1000, "flag", 1, 0, LDValue.of(true), LDValue.of(false), USER); + reset.run(); + copy.summarizeEvent(1001, "flag", 1, 0, LDValue.of(true), LDValue.of(false), USER); + original.summarizeEvent(1001, "flag", 1, 0, LDValue.of(true), LDValue.of(false), USER); + + assertEquals(original.getSummariesAndReset(), copy.getSummariesAndReset()); + } + } + + @Test + public void clearForgetsEverything() { + SameContextPerContextEventSummarizer copy = new SameContextPerContextEventSummarizer(); + copy.summarizeEvent(1000, "flag", 1, 0, LDValue.of(true), LDValue.of(false), USER); + + copy.clear(); + + assertTrue(copy.isEmpty()); + assertTrue(copy.getSummariesAndReset().isEmpty()); + } +} diff --git a/settings.gradle b/settings.gradle index 36c077f5b..fa5361a28 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,5 @@ include(":launchdarkly-android-client-sdk") include(":shared-test-code") include(":example") +include(":test-app") include(":contract-tests") diff --git a/shared-test-code/build.gradle b/shared-test-code/build.gradle index c3fb0e054..9bbaa5785 100644 --- a/shared-test-code/build.gradle +++ b/shared-test-code/build.gradle @@ -12,7 +12,7 @@ ext.versions = [ "androidAnnotation": "1.2.0", "gson": "2.13.2", "junit": "4.13", - "launchdarklyJavaSdkInternal": "1.9.0", + "launchdarklyJavaSdkInternal": "1.12.1", "launchdarklyLogging": "1.1.1", ] diff --git a/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockPlatformState.java b/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockPlatformState.java index 1b08ef95d..8ebf053bc 100644 --- a/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockPlatformState.java +++ b/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockPlatformState.java @@ -16,6 +16,7 @@ public class MockPlatformState implements PlatformState { private volatile boolean foreground = true; private volatile boolean networkAvailable = true; + private volatile File noBackupFilesDir = null; @Override public boolean isNetworkAvailable() { @@ -105,6 +106,15 @@ public File getCacheDir() { return new File(System.getProperty("java.io.tmpdir")); } + @Override + public File getNoBackupFilesDir() { + return noBackupFilesDir != null ? noBackupFilesDir : getCacheDir(); + } + + public void setNoBackupFilesDir(File noBackupFilesDir) { + this.noBackupFilesDir = noBackupFilesDir; + } + @Override public void close() throws IOException {} } diff --git a/test-app/README.md b/test-app/README.md new file mode 100644 index 000000000..e39f8d42f --- /dev/null +++ b/test-app/README.md @@ -0,0 +1,37 @@ +# LaunchDarkly Android SDK test app + +This app is an internal quality harness. For customer integration examples, use `example`. + +Add the following to the repository's root `local.properties`: + +```properties +launchdarkly.mobileKey=mob-... +launchdarkly.environment=production +``` + +Set `launchdarkly.environment=staging` to use LaunchDarkly's staging endpoints. + +## Event-loss scenarios + +Create a boolean flag named `kill-flag`, or enter another flag key in the app. Tap +**Eval+track+kill** to evaluate the flag, track a stand-in error event, request a flush, and +terminate the process five seconds later. This exercises the interval between recording and +delivery without Android lifecycle callbacks masking the result. + +The two immediate controls compare exits that application code can and cannot observe: + +- **Eval+Kill now** records the same pair and sends `SIGKILL` immediately. No handler or SDK code + can run before the process ends. +- **Eval+Crash now** throws an uncaught exception immediately after recording. The installed crash + handler calls `flushAndWait` with a two-second budget before delegating to Android's handler. + +Tier 3 configures `EventPersistence.IMMEDIATE`. With that setting, events recorded by +**Eval+Kill now** should be recovered and delivered after the next launch even though no process +code ran on exit. + +## Over-refresh scenario + +Create a boolean flag named `trackevents-test` with event tracking enabled. Tap +**Over-Refresh Eval** to compare the average evaluation cost while filling the event capacity and +after it is full. The loop intentionally runs on the main thread and is an internal stress test, +not an application integration pattern. diff --git a/test-app/build.gradle b/test-app/build.gradle new file mode 100644 index 000000000..d5b3d28a9 --- /dev/null +++ b/test-app/build.gradle @@ -0,0 +1,50 @@ +import java.util.Properties + +plugins { + id("com.android.application") + // Method counts are an internal quality signal rather than customer setup. + id("com.getkeepsafe.dexcount") +} + +// local.properties is not checked in, so it is where machine-specific settings such as the mobile +// key belong. See test-app/README.md for the keys this app reads. +def localProperties = new Properties() +def localPropertiesFile = rootProject.file("local.properties") +if (localPropertiesFile.exists()) { + localPropertiesFile.withInputStream { localProperties.load(it) } +} + +android { + namespace "com.launchdarkly.sdk.testapp" + compileSdk = 34 + + defaultConfig { + applicationId = "com.launchdarkly.sdk.testapp" + minSdk = 21 + targetSdk = 34 + versionCode = 1 + versionName = "1.0" + + buildConfigField("String", "MOBILE_KEY", + "\"${localProperties.getProperty('launchdarkly.mobileKey', '')}\"") + buildConfigField("String", "LD_ENVIRONMENT", + "\"${localProperties.getProperty('launchdarkly.environment', 'production')}\"") + } + + buildFeatures { + buildConfig = true + } + + buildTypes { + release { + minifyEnabled = true + proguardFiles(getDefaultProguardFile("proguard-android.txt"), "proguard-rules.pro") + } + } +} + +dependencies { + implementation("androidx.appcompat:appcompat:1.2.0") + implementation("com.jakewharton.timber:timber:5.0.1") + implementation(project(":launchdarkly-android-client-sdk")) +} diff --git a/test-app/proguard-rules.pro b/test-app/proguard-rules.pro new file mode 100644 index 000000000..8788708d0 --- /dev/null +++ b/test-app/proguard-rules.pro @@ -0,0 +1,2 @@ +# Allow compile-time-only error-prone annotations to be stripped without breaking shrink. +-dontwarn com.google.errorprone.annotations.** diff --git a/test-app/src/main/AndroidManifest.xml b/test-app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..b9729d0ff --- /dev/null +++ b/test-app/src/main/AndroidManifest.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + diff --git a/test-app/src/main/java/com/launchdarkly/sdk/testapp/ExposureCountingHook.java b/test-app/src/main/java/com/launchdarkly/sdk/testapp/ExposureCountingHook.java new file mode 100644 index 000000000..4ace6bce6 --- /dev/null +++ b/test-app/src/main/java/com/launchdarkly/sdk/testapp/ExposureCountingHook.java @@ -0,0 +1,64 @@ +package com.launchdarkly.sdk.testapp; + +import com.launchdarkly.sdk.EvaluationDetail; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.integrations.EvaluationSeriesContext; +import com.launchdarkly.sdk.android.integrations.Hook; + +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Counts the evaluation series stages it observes, so the example can show what exposure + * deduplication does. Deduplication is configured at registration by wrapping this hook in a + * {@link com.launchdarkly.sdk.android.integrations.DedupingHook}, the same way a customer would + * wrap any other hook. + *

+ * Deduplication skips the whole series, so both counts stay equal and both stop climbing while + * repeated evaluations resolve to the same result. + */ +class ExposureCountingHook extends Hook { + private final String label; + private final Runnable onStage; + private final AtomicInteger befores = new AtomicInteger(); + private final AtomicInteger afters = new AtomicInteger(); + + /** + * @param label a name for this hook, shown in the app's dedupe status + * @param onStage run after each stage so the app can refresh its display + */ + ExposureCountingHook(String label, Runnable onStage) { + super(label); + this.label = label; + this.onStage = onStage; + } + + @Override + public Map beforeEvaluation(EvaluationSeriesContext seriesContext, Map seriesData) { + befores.incrementAndGet(); + onStage.run(); + return seriesData; + } + + @Override + public Map afterEvaluation(EvaluationSeriesContext seriesContext, Map seriesData, + EvaluationDetail evaluationDetail) { + afters.incrementAndGet(); + onStage.run(); + return seriesData; + } + + /** + * @param windowMillis the dedupe window this hook was registered with, for the status line + * @return a line describing this hook's window and how many evaluations have reached it + */ + String status(int windowMillis) { + return String.format(Locale.US, "%s (%d ms): %d (before %d / after %d)", + label, + windowMillis, + afters.get(), + befores.get(), + afters.get()); + } +} diff --git a/test-app/src/main/java/com/launchdarkly/sdk/testapp/MainActivity.java b/test-app/src/main/java/com/launchdarkly/sdk/testapp/MainActivity.java new file mode 100644 index 000000000..d96e498d4 --- /dev/null +++ b/test-app/src/main/java/com/launchdarkly/sdk/testapp/MainActivity.java @@ -0,0 +1,610 @@ +package com.launchdarkly.sdk.testapp; + +import android.content.SharedPreferences; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.view.View; +import android.widget.AdapterView; +import android.widget.ArrayAdapter; +import android.widget.Button; +import android.widget.EditText; +import android.widget.Spinner; +import android.widget.Switch; +import android.widget.TextView; +import android.widget.Toast; + +import androidx.appcompat.app.AppCompatActivity; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.Components; +import com.launchdarkly.sdk.android.ConnectionInformation; +import com.launchdarkly.sdk.android.LDAllFlagsListener; +import com.launchdarkly.sdk.android.LDClient; +import com.launchdarkly.sdk.android.LDConfig; +import com.launchdarkly.sdk.android.LDConfig.Builder.AutoEnvAttributes; +import com.launchdarkly.sdk.android.LDFailure; +import com.launchdarkly.sdk.android.LDStatusListener; +import com.launchdarkly.sdk.android.integrations.DedupingHook; +import com.launchdarkly.sdk.android.integrations.EventPersistence; +import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; +import com.launchdarkly.sdk.android.integrations.LDCrashHandler; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Date; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import timber.log.Timber; + +public class MainActivity extends AppCompatActivity { + + // Two hooks with different windows, to show that each one is deduplicated on its own. + private static final int FAST_DEDUPE_WINDOW_MILLIS = 5_000; + private static final int SLOW_DEDUPE_WINDOW_MILLIS = 10_000; + + // The staging hosts mirror the production ones in StandardEndpoints under the ld-stg domain. + private static final String STAGING_DOMAIN = "ld-stg.launchdarkly.com"; + + private static final String DEFAULT_USER_KEY = "user key"; + + /** How long startup blocks waiting for the first flags to arrive. */ + private static final int INIT_WAIT_SECONDS = 10; + + private static final String PREFERENCES_NAME = "LaunchDarkly-testapp"; + private static final String PERSISTENCE_PREFERENCE = "eventPersistence"; + private static final EventPersistence DEFAULT_PERSISTENCE = EventPersistence.IMMEDIATE; + + /** + * The default of {@code LDConfig.Builder#events} capacity. Past this the event store is full until a + * flush empties it, so every further full event is dropped. + */ + private static final int EVENT_CAPACITY = EventProcessorBuilder.DEFAULT_CAPACITY; + + /** + * Enough evaluations past capacity for the average to settle, and few enough that even a slow device + * finishes the tap well inside the ANR window. + */ + private static final int OVER_CAPACITY_EVALUATIONS = 2000; + + /** + * A batch holds at most 2 MiB, so seven of these fit in one and an eighth does not. Large events + * rather than many small ones, because the 2000-event capacity is reached long before small events + * fill a batch. + */ + private static final int LARGE_EVENT_DATA_CHARS = 256 * 1024; + + /** Seven, seven and six: three batches, the last of them not full. */ + private static final int LARGE_EVENTS = 20; + + /** Null while a persistence change restarts the client, which makes every button a no-op meanwhile. */ + private volatile LDClient ldClient; + private EventPersistence eventPersistence; + private LDStatusListener ldStatusListener; + private LDAllFlagsListener allFlagsListener; + + private final ExposureCountingHook fastHook = + new ExposureCountingHook("fast", this::updateDedupeStatus); + private final ExposureCountingHook slowHook = + new ExposureCountingHook("slow", this::updateDedupeStatus); + private final AtomicInteger evaluationsRequested = new AtomicInteger(); + + private static boolean isStaging() { + return "staging".equalsIgnoreCase(BuildConfig.LD_ENVIRONMENT); + } + + private void updateDedupeStatus() { + if (Looper.myLooper() != MainActivity.this.getMainLooper()) { + new Handler(MainActivity.this.getMainLooper()).post(this::updateDedupeStatus); + return; + } + + String result = String.format(Locale.US, + "Environment: %s\nEvaluations requested: %d\n%s\n%s", + isStaging() ? "staging" : "production", + evaluationsRequested.get(), + fastHook.status(FAST_DEDUPE_WINDOW_MILLIS), + slowHook.status(SLOW_DEDUPE_WINDOW_MILLIS)); + ((TextView) MainActivity.this.findViewById(R.id.dedupe_status)).setText(result); + } + + private void updateStatusString(final ConnectionInformation connectionInformation) { + if (Looper.myLooper() != MainActivity.this.getMainLooper()) { + new Handler(MainActivity.this.getMainLooper()).post(() -> updateStatusString(connectionInformation)); + } else { + TextView connection = MainActivity.this.findViewById(R.id.connection_status); + Long lastSuccess = connectionInformation.getLastSuccessfulConnection(); + Long lastFailure = connectionInformation.getLastFailedConnection(); + + String result = String.format(Locale.US, "Mode: %s\nSuccess at: %s\nFailure at: %s\nFailure type: %s", + connectionInformation.getConnectionMode().toString(), + lastSuccess == null ? "Never" : new Date(lastSuccess).toString(), + lastFailure == null ? "Never" : new Date(lastFailure).toString(), + connectionInformation.getLastFailure() != null ? + connectionInformation.getLastFailure().getFailureType() + : ""); + connection.setText(result); + } + } + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + + setupEval(); + setupFlushButton(); + setupTrackButton(); + setupIdentifyButton(); + setupKillUnsentButton(); + setupKillNowButton(); + setupCrashNowButton(); + setupOverRefreshButton(); + setupFillBatchesButton(); + setupOfflineSwitch(); + setupPersistenceSpinner(); + // Rescues the events for "Eval+Crash now" and cannot run for "Eval+Kill now", which is what + // makes the pair worth pressing. + LDCrashHandler.install(2, TimeUnit.SECONDS); + setupListeners(); + updateDedupeStatus(); + + if (BuildConfig.MOBILE_KEY.isEmpty()) { + String message = "Set launchdarkly.mobileKey in local.properties and rebuild."; + Timber.e(message); + Toast.makeText(this, message, Toast.LENGTH_LONG).show(); + return; + } + + ldClient = startClient(eventPersistence, offlineSwitch().isChecked()); + } + + /** + * Blocks for up to {@link #INIT_WAIT_SECONDS} waiting for flags, and returns the client either way: if + * they have not arrived by then, it is usable with whatever it has cached. + */ + private LDClient startClient(EventPersistence persistence, boolean offline) { + LDConfig.Builder configBuilder = new LDConfig.Builder(AutoEnvAttributes.Enabled) + .mobileKey(BuildConfig.MOBILE_KEY) + .offline(offline) + .http( + Components.httpConfiguration().useReport(false) + // change useReport to `true` if the request is to be REPORT'ed instead of GET'ed + ) + .events( + Components.sendEvents().eventPersistence(persistence) + ) + .hooks( + // Same shape a customer uses for any hook: wrap it at registration. Each + // wrapper has its own window, so neither suppresses the other. + Components.hooks() + .addHook(new DedupingHook(fastHook, FAST_DEDUPE_WINDOW_MILLIS)) + .addHook(new DedupingHook(slowHook, SLOW_DEDUPE_WINDOW_MILLIS)) + ); + + if (isStaging()) { + configBuilder.serviceEndpoints( + Components.serviceEndpoints() + .streaming("https://clientstream." + STAGING_DOMAIN) + .polling("https://clientsdk." + STAGING_DOMAIN) + .events("https://mobile." + STAGING_DOMAIN) + ); + } + + LDConfig ldConfig = configBuilder.build(); + + LDContext context = LDContext.builder(DEFAULT_USER_KEY) + .set("email", "fake@example.com") + .build(); + + LDClient client = LDClient.init(this.getApplication(), ldConfig, context, INIT_WAIT_SECONDS); + Timber.i("client started with event persistence %s", persistence); + updateStatusString(client.getConnectionInformation()); + client.registerStatusListener(ldStatusListener); + client.registerAllFlagsListener(allFlagsListener); + return client; + } + + /** + * The choice is saved rather than held in memory, because the kill and crash buttons are only + * answered on the next launch, and that launch has to read the disk the same way: under + * {@link EventPersistence#DISABLED} the events the dead process left there are not delivered. + */ + private void setupPersistenceSpinner() { + final SharedPreferences preferences = getSharedPreferences(PREFERENCES_NAME, MODE_PRIVATE); + eventPersistence = parsePersistence(preferences.getString(PERSISTENCE_PREFERENCE, null)); + + final Spinner spinner = findViewById(R.id.persistence_spinner); + ArrayAdapter adapter = new ArrayAdapter<>(this, + android.R.layout.simple_spinner_item, EventPersistence.values()); + adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); + spinner.setAdapter(adapter); + spinner.setSelection(eventPersistence.ordinal(), false); + + spinner.setOnItemSelectedListener(new AdapterView.OnItemSelectedListener() { + @Override + public void onItemSelected(AdapterView parent, View view, int position, long id) { + EventPersistence chosen = (EventPersistence) parent.getItemAtPosition(position); + if (chosen == eventPersistence) { + return; + } + eventPersistence = chosen; + preferences.edit().putString(PERSISTENCE_PREFERENCE, chosen.name()).apply(); + restartClient(spinner, chosen); + } + + @Override + public void onNothingSelected(AdapterView parent) { + } + }); + } + + private static EventPersistence parsePersistence(String name) { + if (name != null) { + try { + return EventPersistence.valueOf(name); + } catch (IllegalArgumentException e) { + Timber.w("ignoring saved event persistence %s", name); + } + } + return DEFAULT_PERSISTENCE; + } + + /** + * Off the main thread, because closing waits on the event store and starting waits for flags. The + * spinner stays disabled until it is done, so restarts never overlap. + *

+ * The replacement starts in the offline switch's state rather than being switched after it starts, + * because starting waits for flags, and a client online for that long can deliver the events stored + * before the change. + */ + private void restartClient(final Spinner spinner, final EventPersistence persistence) { + if (BuildConfig.MOBILE_KEY.isEmpty()) { + return; + } + final boolean offline = offlineSwitch().isChecked(); + final LDClient previous = ldClient; + ldClient = null; + spinner.setEnabled(false); + new Thread(() -> { + if (previous != null) { + previous.unregisterStatusListener(ldStatusListener); + previous.unregisterAllFlagsListener(allFlagsListener); + try { + previous.close(); + } catch (IOException e) { + Timber.w(e, "closing the client for a persistence change"); + } + } + LDClient restarted = startClient(persistence, offline); + new Handler(getMainLooper()).post(() -> { + ldClient = restarted; + spinner.setEnabled(true); + // The switch is ignored while there is no client to act on. + boolean offlineNow = offlineSwitch().isChecked(); + if (offlineNow != offline) { + if (offlineNow) { + restarted.setOffline(); + } else { + restarted.setOnline(); + } + } + Toast.makeText(this, "Restarted with " + persistence, Toast.LENGTH_SHORT).show(); + }); + }, "testapp-restart-client").start(); + } + + private void setupListeners() { + ldStatusListener = new LDStatusListener() { + @Override + public void onConnectionModeChanged(final ConnectionInformation connectionInformation) { + updateStatusString(connectionInformation); + } + + @Override + public void onInternalFailure(final LDFailure ldFailure) { + new Handler(MainActivity.this.getMainLooper()).post(() -> { + Toast.makeText(MainActivity.this, ldFailure.toString(), Toast.LENGTH_SHORT).show(); + }); + MainActivity.this.doSafeClientAction(() -> updateStatusString(ldClient.getConnectionInformation())); + } + }; + + allFlagsListener = flagKey -> { + new Handler(MainActivity.this.getMainLooper()).post(() -> { + StringBuilder flags = new StringBuilder("Updated flags: "); + for (String flag : flagKey) { + flags.append(flag).append(" "); + } + Toast.makeText(MainActivity.this, flags.toString(), Toast.LENGTH_SHORT).show(); + }); + MainActivity.this.doSafeClientAction(() -> updateStatusString(ldClient.getConnectionInformation())); + }; + } + + private void setupFlushButton() { + Button flushButton = findViewById(R.id.flush_button); + flushButton.setOnClickListener(v -> { + Timber.i("flush onClick"); + MainActivity.this.doSafeClientAction(() -> ldClient.flush()); + }); + } + + private interface LDClientAction { + void call(); + } + + private void doSafeClientAction(LDClientAction function) { + if (ldClient != null) { + function.call(); + } + } + + private interface LDClientGet { + V get(); + } + + private V doSafeClientGet(LDClientGet function) { + return ldClient != null ? function.get() : null; + } + + private void setupTrackButton() { + Button trackButton = findViewById(R.id.track_button); + trackButton.setOnClickListener(v -> { + Timber.i("track onClick"); + MainActivity.this.doSafeClientAction(() -> ldClient.track("Android event name")); + }); + } + + /** + * The flag the kill and crash buttons evaluate: whatever is typed in the feature key field, or a + * default, so the buttons work without anything being typed first. + */ + private String flagKeyToKillOver() { + String typedKey = ((EditText) findViewById(R.id.feature_flag_key)).getText().toString().trim(); + return typedKey.isEmpty() ? "kill-flag" : typedKey; + } + + /** + * Records the pair whose survival is in question: an evaluation, which is the exposure, and a + * track, standing in for the error an application reports just before it dies. + * + *

Returns false when there is no client, in which case nothing was recorded and ending the + * process would demonstrate nothing. + */ + private boolean recordExposureAndError(String flagKey) { + if (ldClient == null) { + return false; + } + ldClient.boolVariation(flagKey, false); + ldClient.track("$ld:telemetry:error"); + return true; + } + + /** + * Reproduces in-memory event loss: evaluate (exposure) and track (stand-in for an error), + * wait 5s so both calls are queued, then kill the process before the 30s flush. + * {@code finish()} or backgrounding would run the SDK's background flush, so this uses + * {@link android.os.Process#killProcess}. + */ + private void setupKillUnsentButton() { + Button killUnsentButton = findViewById(R.id.kill_unsent_button); + killUnsentButton.setOnClickListener(v -> { + final String flagKey = flagKeyToKillOver(); + Timber.w("eval+track+kill flag=%s", flagKey); + if (!recordExposureAndError(flagKey)) { + return; + } + ldClient.flush(); + new Handler(Looper.getMainLooper()).postDelayed( + () -> android.os.Process.killProcess(android.os.Process.myPid()), + 5_000); + }); + } + + /** + * The same sequence with nothing at all between the track and the process dying: no flush to + * deliver the events, no delay for a timer to fire in, and SIGKILL to itself, which cannot be + * caught, so no part of the SDK gets to run on the way out. + * + *

Whether the exposure and the track are reported therefore says exactly one thing: whether + * recording them had already put them somewhere that outlives the process. They should arrive on + * the next launch of the app, not this one. + */ + private void setupKillNowButton() { + Button killNowButton = findViewById(R.id.kill_now_button); + killNowButton.setOnClickListener(v -> { + final String flagKey = flagKeyToKillOver(); + Timber.w("eval+track+kill now flag=%s", flagKey); + if (!recordExposureAndError(flagKey)) { + return; + } + android.os.Process.killProcess(android.os.Process.myPid()); + }); + } + + /** + * The same again, ending in an uncaught exception instead of a signal the process never sees. + * + *

This is the shape a customer report takes: app code fails immediately after reporting the + * failure. Unlike SIGKILL, an uncaught exception runs the default handler before the process + * goes, so this is the one variant an application can rescue on its own, which + * {@link LDCrashHandler} does by calling {@link LDClient#flushAndWait} from there. So these + * events should arrive and the ones from the button next to it should not. + */ + private void setupCrashNowButton() { + Button crashNowButton = findViewById(R.id.crash_now_button); + crashNowButton.setOnClickListener(v -> { + final String flagKey = flagKeyToKillOver(); + Timber.w("eval+track+crash now flag=%s", flagKey); + if (!recordExposureAndError(flagKey)) { + return; + } + throw new RuntimeException( + "Eval+Crash: deliberate uncaught exception immediately after track, to test event persistence"); + }); + } + + /** + * Reproduces the over-refresh pathology: a UI that re-evaluates one flag far more often than anything + * about it changed -- a recomposition or layout loop -- against a flag whose events LaunchDarkly is + * tracking. Runs on the main thread, because that is where the loop it stands in for runs. + * + *

The comparison between the two phases is the point rather than either number on its own. The first + * {@link #EVENT_CAPACITY} evaluations have room in the event store; every one after that is dropped, + * because nothing empties it until a flush. So the two phases costing the same per evaluation means the + * SDK is serializing events it then discards, which is the whole of the waste. A cost of a couple of + * microseconds instead means the flag has event tracking off, and this measured the summary counters + * rather than the thing in question. + */ + private void setupOverRefreshButton() { + Button overRefreshButton = findViewById(R.id.over_refresh_button); + overRefreshButton.setOnClickListener(v -> { + if (ldClient == null) { + return; + } + final String flagKey = "trackevents-test"; + Timber.w("over-refresh eval flag=%s", flagKey); + + double filling = averageMicroseconds(flagKey, EVENT_CAPACITY); + double overCapacity = averageMicroseconds(flagKey, OVER_CAPACITY_EVALUATIONS); + + String result = String.format(Locale.US, + "Over-refresh %s\n%d filling: %.0f \u00b5s/eval\n%d over capacity: %.0f \u00b5s/eval", + flagKey, + EVENT_CAPACITY, filling, + OVER_CAPACITY_EVALUATIONS, overCapacity); + Timber.w(result); + ((TextView) findViewById(R.id.result_textView)).setText(result); + }); + } + + /** + * Records more events than one batch holds, so a flush sends several requests, each with a payload + * ID of its own. Offline, or followed by a kill, it leaves several {@code ready-} files on disk for + * the next launch to deliver. Each event's index is in its data, so the dashboard shows whether all + * of them arrived and none twice. + */ + private void setupFillBatchesButton() { + Button fillBatchesButton = findViewById(R.id.fill_batches_button); + fillBatchesButton.setOnClickListener(v -> { + if (ldClient == null) { + return; + } + char[] filler = new char[LARGE_EVENT_DATA_CHARS]; + Arrays.fill(filler, 'x'); + String padding = new String(filler); + for (int i = 1; i <= LARGE_EVENTS; i++) { + ldClient.trackData("fill-batches", LDValue.buildObject() + .put("index", i) + .put("of", LARGE_EVENTS) + .put("padding", padding) + .build()); + } + String result = String.format(Locale.US, + "Recorded %d events of %d KiB: three batches of at most 2 MiB", + LARGE_EVENTS, LARGE_EVENT_DATA_CHARS / 1024); + Timber.w(result); + ((TextView) findViewById(R.id.result_textView)).setText(result); + }); + } + + /** + * @return the mean wall time, in microseconds, of {@code count} evaluations of the flag + */ + private double averageMicroseconds(String flagKey, int count) { + long start = System.nanoTime(); + for (int i = 0; i < count; i++) { + ldClient.boolVariation(flagKey, false); + } + return (System.nanoTime() - start) / 1_000.0 / count; + } + + private void setupIdentifyButton() { + Button identify = findViewById(R.id.identify_button); + identify.setOnClickListener(v -> { + Timber.i("identify onClick"); + String typedKey = ((EditText) MainActivity.this.findViewById(R.id.userKey_editText)) + .getText().toString().trim(); + // An empty key builds an invalid context, which identify rejects before it resets the + // hooks' dedupe caches. Fall back to the key the client started with, so identifying to + // an unchanged context still demonstrates that the reset happens either way. + final String userKey = typedKey.isEmpty() ? DEFAULT_USER_KEY : typedKey; + final LDContext updatedContext = LDContext.create(userKey); + MainActivity.this.doSafeClientAction(() -> { + ldClient.identify(updatedContext); + }); + MainActivity.this.updateDedupeStatus(); + }); + } + + private Switch offlineSwitch() { + return findViewById(R.id.offlineSwitch); + } + + private void setupOfflineSwitch() { + offlineSwitch().setOnCheckedChangeListener((compoundButton, isChecked) -> + MainActivity.this.doSafeClientAction(isChecked ? () -> ldClient.setOffline() : () -> ldClient.setOnline()) + ); + } + + private void setupEval() { + final Spinner spinner = findViewById(R.id.type_spinner); + ArrayAdapter adapter = ArrayAdapter.createFromResource(this, + R.array.types_array, android.R.layout.simple_spinner_item); + adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); + spinner.setAdapter(adapter); + + Button evalButton = findViewById(R.id.eval_button); + evalButton.setOnClickListener(v -> { + Timber.i("eval onClick"); + final String flagKey = ((EditText) MainActivity.this.findViewById(R.id.feature_flag_key)).getText().toString(); + evaluationsRequested.incrementAndGet(); + + String type = spinner.getSelectedItem().toString(); + final String result; + String logResult; + switch (type) { + case "String": + result = MainActivity.this.doSafeClientGet(() -> ldClient.stringVariation(flagKey, "default")); + logResult = result == null ? "no result" : result; + Timber.i(logResult); + ((TextView) MainActivity.this.findViewById(R.id.result_textView)).setText(result); + MainActivity.this.doSafeClientAction(() -> { + ldClient.registerFeatureFlagListener(flagKey, flagKey1 -> { + evaluationsRequested.incrementAndGet(); + ((TextView) MainActivity.this.findViewById(R.id.result_textView)) + .setText(ldClient.stringVariation(flagKey1, "default")); + MainActivity.this.updateDedupeStatus(); + }); + }); + MainActivity.this.updateDedupeStatus(); + return; + case "Boolean": + result = MainActivity.this.doSafeClientGet(() -> String.valueOf(ldClient.boolVariation(flagKey, false))); + break; + case "Integer": + result = MainActivity.this.doSafeClientGet(() -> String.valueOf(ldClient.intVariation(flagKey, 0))); + break; + case "Float": + result = MainActivity.this.doSafeClientGet(() -> String.valueOf(ldClient.doubleVariation(flagKey, 0.0))); + break; + case "Value": + result = MainActivity.this.doSafeClientGet(() -> String.valueOf(ldClient.jsonValueVariation(flagKey, null))); + break; + default: + result = null; + break; + } + + logResult = result == null ? "no result" : result; + Timber.i(logResult); + ((TextView) MainActivity.this.findViewById(R.id.result_textView)).setText(result); + MainActivity.this.updateDedupeStatus(); + }); + } + +} diff --git a/test-app/src/main/res/layout/activity_main.xml b/test-app/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000..189913b28 --- /dev/null +++ b/test-app/src/main/res/layout/activity_main.xml @@ -0,0 +1,260 @@ + + + +