diff --git a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs index 336d4fec..75a7b213 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs @@ -110,6 +110,22 @@ public void AnEnvelopeTheBackendWouldRefuseIsRefusedHere(string body, string err Assert.Equal(error, Rejected(CurationDeskWrites.Ingest, body)); } + [Theory] + [InlineData("post")] + [InlineData("vote")] + [InlineData("curator_vote")] + [InlineData("flag")] + [InlineData("wave_nsfw")] + public void EveryEventTypeTheBackendStoresIsForwarded(string type) + { + // The list here must match the backend's: a type missing from it is a 400 at + // the gateway, which the sender treats as permanent and dead-letters. + var payload = Ok(CurationDeskWrites.Ingest, + "{\"v\":1,\"type\":\"" + type + "\",\"id\":\"" + type + ":bob/p\",\"payload\":{\"author\":\"bob\",\"permlink\":\"p\",\"over\":true}}"); + Assert.Equal(type, payload["type"]!.GetValue()); + Assert.True(payload["payload"]!["over"]!.GetValue()); + } + [Fact] public void TheEventIdLengthBoundIsTheBackendsColumn() { diff --git a/dotnet/EcencyApi.Tests/WaveNsfwTests.cs b/dotnet/EcencyApi.Tests/WaveNsfwTests.cs new file mode 100644 index 00000000..f3e0cfc3 --- /dev/null +++ b/dotnet/EcencyApi.Tests/WaveNsfwTests.cs @@ -0,0 +1,209 @@ +using System.Text.Json.Nodes; +using EcencyApi.Infrastructure; +using Xunit; + +namespace EcencyApi.Tests; + +/// +/// The image-check filter on promoted entries. The verdicts come from the waves +/// indexer; what can silently go wrong here is the rest: dropping the wrong +/// entry, treating an unusable answer as "nothing is flagged", or taking the +/// promoted list down when the indexer cannot be reached. +/// +[Collection("wave-nsfw")] +public class WaveNsfwTests : IDisposable +{ + private readonly Func, Task> _saved = WaveNsfw.Lookup; + + public void Dispose() => WaveNsfw.Lookup = _saved; + + // MemCache is process-wide and has no reset, so every test names its own waves. + private static string Tag() => Guid.NewGuid().ToString("N")[..10]; + + private static JsonObject Post(string? author, string? permlink) + { + var o = new JsonObject(); + if (author != null) o["author"] = author; + if (permlink != null) o["permlink"] = permlink; + return o; + } + + private static UpstreamResponse Answer(int status, JsonNode? json) => new() + { + Status = status, + Json = json, + Headers = new HttpResponseHeaders2(new HttpResponseMessage()), + }; + + private static UpstreamResponse Flagged(params string[] keys) => + Answer(200, new JsonObject { ["flagged"] = new JsonArray(keys.Select(k => (JsonNode?)k).ToArray()) }); + + private static string?[] PermlinksOf(JsonArray arr) => + arr.Select(e => e is JsonObject o && o.TryGetPropertyValue("permlink", out var p) && p is JsonValue + ? p.ToString() + : null).ToArray(); + + [Fact] + public void KeysAreAuthorSlashPermlinkLowercasedAuthorAndDeduplicated() + { + var entries = new JsonArray(Post("Alice", "Wave-1"), Post("alice", "Wave-1"), Post("bob", "p"), + Post(null, "p"), Post("carol", null), Post("", "p"), new JsonObject { ["author"] = 7, ["permlink"] = "p" }); + + Assert.Equal(new[] { "alice/Wave-1", "bob/p" }, WaveNsfw.Keys(entries)); + } + + [Fact] + public void TheLookupEndpointRepeatsPostAndEncodesEachKey() + { + Assert.Equal("waves/nsfw?post=alice%2Fwave-1&post=bob%2Fa%20b%26c", + WaveNsfw.Endpoint(new[] { "alice/wave-1", "bob/a b&c" })); + } + + [Fact] + public void OnlyTheFlaggedEntryIsDropped() + { + var entries = new JsonArray(Post("alice", "clean"), Post("bob", "explicit"), Post("bob", "other")); + + var result = WaveNsfw.FilterFlagged(entries, WaveNsfw.ToSet(new[] { "bob/explicit" })); + + Assert.Equal(new[] { "clean", "other" }, PermlinksOf(result)); + } + + [Fact] + public void FilteringNeverReachesTheCachedPromotedList() + { + // The filter detaches the entries it keeps from the array it is given. That + // is only safe because the promoted list never hands out its cached array: + // MemCache clones on write and again on every read. If either clone went + // away, one filtered request would empty the cache for everyone after it. + var key = "wave-nsfw-test-" + Tag(); + var original = new JsonArray(Post("alice", "clean"), Post("bob", "explicit"), Post("carol", "other")); + MemCache.Set(key, original, 60); + + // the instance a cache-miss request keeps using after it stored the list + var missCopy = WaveNsfw.FilterFlagged(original, WaveNsfw.ToSet(new[] { "bob/explicit" })); + // the instance a cache-hit request is handed + var hitCopy = WaveNsfw.FilterFlagged(MemCache.Get(key)!, WaveNsfw.ToSet(new[] { "bob/explicit" })); + + Assert.Equal(new[] { "clean", "other" }, PermlinksOf(missCopy)); + Assert.Equal(new[] { "clean", "other" }, PermlinksOf(hitCopy)); + Assert.Equal(new[] { "clean", "explicit", "other" }, PermlinksOf(MemCache.Get(key)!)); + MemCache.Del(key); + } + + [Fact] + public void NothingFlaggedLeavesTheListUntouched() + { + var entries = new JsonArray(Post("alice", "clean")); + Assert.Same(entries, WaveNsfw.FilterFlagged(entries, WaveNsfw.ToSet(Array.Empty()))); + } + + [Fact] + public void AnEntryThatCannotBeKeyedIsKeptNotDropped() + { + // An unreadable shape is not a verdict, and dropping it would silently + // shrink the promoted list. + var entries = new JsonArray(Post(null, "explicit"), new JsonObject { ["author"] = new JsonObject() }, + Post("bob", "explicit")); + + var result = WaveNsfw.FilterFlagged(entries, WaveNsfw.ToSet(new[] { "bob/explicit" })); + + Assert.Equal(2, result.Count); + } + + [Theory] + [InlineData(500, "{\"flagged\":[]}")] + [InlineData(200, "{\"error\":\"x\"}")] + [InlineData(200, "{\"flagged\":\"all\"}")] + [InlineData(200, "[]")] + public void AnUnusableAnswerIsAFailureNotAnEmptyVerdict(int status, string json) + { + // Cached as "nothing is flagged", a 500 would switch the filter off. + Assert.Throws(() => WaveNsfw.ReadFlagged(Answer(status, JsonNode.Parse(json)))); + } + + [Fact] + public void RowsThatAreNotKeysAreSkipped() + { + var answer = Answer(200, JsonNode.Parse("{\"flagged\":[\"bob/p\",7,null,\"\",{\"a\":1}]}")); + Assert.Equal(new[] { "bob/p" }, WaveNsfw.ReadFlagged(answer)); + } + + [Fact] + public async Task TheFlaggedSetIsAskedOnceAndThenServedFromTheCache() + { + var t = Tag(); + var entries = new JsonArray(Post("alice", "clean-" + t), Post("bob", "explicit-" + t)); + var asked = new List(); + WaveNsfw.Lookup = keys => + { + asked.Add(keys.ToArray()); + return Task.FromResult(Flagged("bob/explicit-" + t)); + }; + + var first = await WaveNsfw.GetFlagged(entries); + var second = await WaveNsfw.GetFlagged(entries); + + Assert.Equal(new[] { "bob/explicit-" + t }, first); + Assert.Equal(first, second); + Assert.Single(asked); + Assert.Equal(new[] { "alice/clean-" + t, "bob/explicit-" + t }, asked[0]); + } + + [Fact] + public async Task AnEmptyListAsksNothing() + { + WaveNsfw.Lookup = _ => throw new Xunit.Sdk.XunitException("no lookup for no keys"); + Assert.Empty(await WaveNsfw.GetFlagged(new JsonArray(Post(null, null)))); + } + + [Fact] + public async Task AFailedLookupFallsBackToTheLastGoodAnswerAndNeverThrows() + { + var t = Tag(); + var entries = new JsonArray(Post("alice", "clean-" + t), Post("bob", "explicit-" + t)); + WaveNsfw.Lookup = _ => Task.FromResult(Flagged("bob/explicit-" + t)); + await WaveNsfw.GetFlagged(entries); + + // The promoted list changed (a new cache key) and the indexer is down. + var later = new JsonArray(Post("bob", "explicit-" + t), Post("carol", "new-" + t)); + var calls = 0; + WaveNsfw.Lookup = _ => + { + calls++; + throw new HttpRequestException("indexer unreachable"); + }; + + var flagged = await WaveNsfw.GetFlagged(later); + var again = await WaveNsfw.GetFlagged(later); + + Assert.Contains("bob/explicit-" + t, flagged); + Assert.Equal(new[] { "new-" + t }, PermlinksOf(WaveNsfw.FilterFlagged(later, flagged))); + // The failure is held, so a dead indexer is not asked on every request. + Assert.Equal(1, calls); + Assert.Equal(flagged, again); + } + + [Fact] + public async Task ALongListIsCheckedInFullAcrossSeveralLookups() + { + var t = Tag(); + var entries = new JsonArray(); + for (var i = 0; i < WaveNsfw.MaxPosts + 5; i++) + { + entries.Add(Post("a" + i, "p-" + t)); + } + var last = "a" + (WaveNsfw.MaxPosts + 4) + "/p-" + t; + var sizes = new List(); + WaveNsfw.Lookup = keys => + { + sizes.Add(keys.Count); + return Task.FromResult(keys.Contains(last) ? Flagged(last) : Flagged()); + }; + + var flagged = await WaveNsfw.GetFlagged(entries); + + Assert.Equal(new[] { WaveNsfw.MaxPosts, 5 }, sizes); + Assert.Equal(new[] { last }, flagged); + } +} diff --git a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs index 20e76baa..849457bc 100644 --- a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs +++ b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs @@ -799,8 +799,9 @@ public sealed record Route(string UpstreamPath, string[] Keys, bool ForwardClien public const int MaxVoteWeight = 10000; /// The backend keeps a curator note in a varchar(200). public const int MaxCuratorNoteLength = 200; - /// The four event types erobot pushes (spec 7.2). - public static readonly IReadOnlySet IngestTypes = new HashSet { "post", "vote", "curator_vote", "flag" }; + /// The event types erobot pushes (spec 7.2). `wave_nsfw` carries the image + /// check's verdict for a wave; the backend range-checks its payload like the others. + public static readonly IReadOnlySet IngestTypes = new HashSet { "post", "vote", "curator_vote", "flag", "wave_nsfw" }; /// The backend keeps the event id in a varchar(200). public const int MaxIngestIdLength = 200; diff --git a/dotnet/EcencyApi/Handlers/PrivateApi.Feeds.cs b/dotnet/EcencyApi/Handlers/PrivateApi.Feeds.cs index 2ca30205..13ad6424 100644 --- a/dotnet/EcencyApi/Handlers/PrivateApi.Feeds.cs +++ b/dotnet/EcencyApi/Handlers/PrivateApi.Feeds.cs @@ -77,6 +77,12 @@ public static async Task PromotedEntries(HttpContext ctx) // refresh instead of waiting out the promoted cache. posts = ModerationMutes.FilterMutedAuthors(posts, await ModerationMutes.Get()); + // Same reasoning for the image check: the waves indexer leaves a wave with + // a detected explicit picture out of every feed it serves, and a promoted + // slot must not be the way back in. Clients cannot do this themselves, a + // promoted entry comes from the chain and carries no verdict. + posts = WaveNsfw.FilterFlagged(posts, await WaveNsfw.GetFlagged(posts)); + await ctx.SendJson(200, posts); } @@ -163,6 +169,28 @@ public static async Task WavesFeed(HttpContext ctx) await Upstream.Pipe(ApiClient.ApiRequest(u, HttpMethod.Get), ctx); } + /// + /// GET /private-api/waves/nsfw?author=&permlink= : did the image check find + /// this wave explicit? For the wave page opened by a link, which loads the + /// wave from the chain and so has no feed row to read the verdict from. + /// Answers the indexer's `{"flagged": [...]}`, empty when it is not flagged. + /// + public static async Task WavesNsfw(HttpContext ctx) + { + var q = ctx.Request.Query; + var author = q["author"]; + var permlink = q["permlink"]; + if (author.Count != 1 || permlink.Count != 1 + || string.IsNullOrEmpty(author[0]) || string.IsNullOrEmpty(permlink[0])) + { + await ctx.SendJson(400, new JsonObject { ["error"] = "author and permlink required" }); + return; + } + + var u = WaveNsfw.Endpoint(new[] { author[0]!.TrimStart('@') + "/" + permlink[0] }); + await Upstream.Pipe(ApiClient.ApiRequest(u, HttpMethod.Get), ctx); + } + public static async Task WavesShorts(HttpContext ctx) { var q = ctx.Request.Query; diff --git a/dotnet/EcencyApi/Handlers/Routes.cs b/dotnet/EcencyApi/Handlers/Routes.cs index e954644b..57da6ff3 100644 --- a/dotnet/EcencyApi/Handlers/Routes.cs +++ b/dotnet/EcencyApi/Handlers/Routes.cs @@ -66,6 +66,7 @@ public static void Map(WebApplication app) app.MapGet("/private-api/pub-notifications/{username}", PrivateApi.PublicUnreadNotifications); app.MapGet("/private-api/waves/feed", PrivateApi.WavesFeed); app.MapGet("/private-api/waves/shorts", PrivateApi.WavesShorts); + app.MapGet("/private-api/waves/nsfw", PrivateApi.WavesNsfw); app.MapGet("/private-api/waves/tags", PrivateApi.WavesTags); app.MapGet("/private-api/waves/account", PrivateApi.WavesAccount); app.MapGet("/private-api/waves/following", PrivateApi.WavesFollowing); diff --git a/dotnet/EcencyApi/Infrastructure/WaveNsfw.cs b/dotnet/EcencyApi/Infrastructure/WaveNsfw.cs new file mode 100644 index 00000000..01676ce5 --- /dev/null +++ b/dotnet/EcencyApi/Infrastructure/WaveNsfw.cs @@ -0,0 +1,186 @@ +using System.Text.Json.Nodes; + +namespace EcencyApi.Infrastructure; + +/// +/// The image check's verdicts for waves, as kept by the waves indexer. +/// +/// The indexer leaves a wave with a detected explicit picture out of every +/// shared feed it serves. Promoted entries never go through it, so without this +/// a flagged wave could still be bought into the most prominent slot of the +/// feed, the same hole the moderation mute list had before ModerationMutes. +/// +/// Asked from the indexer rather than decided here: the threshold and the +/// stored verdicts live in one place. Fails open, and to the last answer it +/// had: a filter that cannot load must not take the promoted list down. +/// +public static class WaveNsfw +{ + private const string CachePrefix = "wave-nsfw-flagged:"; + + /// Survives a failed lookup. Never expires on purpose. + private const string LastGoodCacheKey = "wave-nsfw-flagged-last-good"; + + /// A verdict lands seconds after a wave is posted; this bounds how long a + /// just-flagged promoted wave stays up. + private const double TtlSeconds = 60; + + /// Holds a failure so a dead indexer costs one short call per interval, + /// not one per promoted-entries request. + private const double FailureTtlSeconds = 30; + + /// This sits on the promoted-entries request path, so it is short. + private const int TimeoutMs = 2000; + + /// The indexer answers at most this many posts per call. + public const int MaxPosts = 100; + + /// Replaceable for tests. Returns the indexer's answer for these keys. + internal static Func, Task> Lookup = DefaultLookup; + + private static Task DefaultLookup(IReadOnlyList keys) => + ApiClient.ApiRequest(Endpoint(keys), HttpMethod.Get, timeoutMs: TimeoutMs); + + /// `waves/nsfw?post=a/p&post=b/q`, each key form-encoded. + internal static string Endpoint(IEnumerable keys) => + "waves/nsfw?" + string.Join("&", keys.Select(k => "post=" + Uri.EscapeDataString(k))); + + /// + /// `author/permlink` for every entry that has both, deduplicated. Entries + /// with an unreadable author or permlink are skipped here and therefore kept + /// by the filter. + /// + internal static List Keys(JsonArray entries) + { + var keys = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var entry in entries) + { + var key = Key(entry); + if (key != null && seen.Add(key)) + { + keys.Add(key); + } + } + return keys; + } + + internal static string? Key(JsonNode? entry) + { + var author = ReadString(entry, "author"); + var permlink = ReadString(entry, "permlink"); + return string.IsNullOrEmpty(author) || string.IsNullOrEmpty(permlink) + ? null + : author.ToLowerInvariant() + "/" + permlink; + } + + /// Lenient on purpose, see ModerationMutes.ReadString. + private static string? ReadString(JsonNode? owner, string property) => + owner is JsonObject o + && o.TryGetPropertyValue(property, out var node) + && node is JsonValue value + && JsVal.TryGetStringLenient(value, out var s) + ? s + : null; + + /// + /// The flagged subset of these entries. Empty when there is nothing to ask, + /// and the last good answer when the indexer cannot be reached. + /// + public static async Task> GetFlagged(JsonArray entries) + { + var keys = Keys(entries); + if (keys.Count == 0) + { + return new HashSet(StringComparer.Ordinal); + } + + // The promoted list is the same for every caller between its own + // refreshes, so one entry per distinct list is one entry in practice. + var cacheKey = CachePrefix + string.Join(",", keys.OrderBy(k => k, StringComparer.Ordinal)); + var cached = MemCache.Get(cacheKey); + if (cached != null) + { + return ToSet(cached); + } + + string[] flagged; + try + { + // One call per MaxPosts keys, so a long promoted list is checked in + // full instead of being filtered only as far as the first chunk. + var found = new List(); + foreach (var chunk in keys.Chunk(MaxPosts)) + { + found.AddRange(ReadFlagged(await Lookup(chunk))); + } + flagged = found.ToArray(); + MemCache.Set(cacheKey, flagged, TtlSeconds); + MemCache.Set(LastGoodCacheKey, flagged); + } + catch (Exception) + { + // Deliberately silent (CLAUDE.md, "No hot-path logging"). The last + // good answer may name waves that are no longer promoted; that is + // harmless, the filter only drops what is in both sets. + flagged = MemCache.Get(LastGoodCacheKey) ?? Array.Empty(); + MemCache.Set(cacheKey, flagged, FailureTtlSeconds); + } + + return ToSet(flagged); + } + + /// + /// `{"flagged": ["author/permlink", ...]}`. Anything else throws, so an + /// indexer that answers 200 with an unusable body is a failure (fall back), + /// not an empty verdict cached as "nothing is flagged". + /// + internal static string[] ReadFlagged(UpstreamResponse response) + { + if (response.Status != 200 || response.Json is not JsonObject body + || !body.TryGetPropertyValue("flagged", out var node) || node is not JsonArray rows) + { + throw new InvalidOperationException("unusable wave nsfw lookup answer"); + } + + var flagged = new List(); + foreach (var row in rows) + { + if (row is JsonValue value && JsVal.TryGetStringLenient(value, out var key) && !string.IsNullOrEmpty(key)) + { + flagged.Add(key); + } + } + return flagged.ToArray(); + } + + internal static HashSet ToSet(IEnumerable keys) => new(keys, StringComparer.Ordinal); + + /// + /// Drop the flagged entries. Returns a new array; an entry whose author or + /// permlink cannot be read is kept, an unreadable shape is not a verdict. + /// + public static JsonArray FilterFlagged(JsonArray entries, ISet flagged) + { + if (flagged.Count == 0) + { + return entries; + } + + var kept = new JsonArray(); + foreach (var entry in entries.ToArray()) + { + var key = Key(entry); + if (key != null && flagged.Contains(key)) + { + continue; + } + + // A node can only live in one parent, so detach before re-parenting. + entry?.Parent?.AsArray().Remove(entry); + kept.Add(entry); + } + + return kept; + } +}