fix: output limits, BaseException safety, include confinement, usings/statics/inner exceptions (0.3.0)

README.md

@@ -38,8 +38,8 @@ Every host already has its own Roslyn constraints (Unity pins 4.8 because of `Sy
 ## C# in two lines
 
 ```xml
-<PackageReference Include="com.IvanMurzak.CodeExec" Version="0.2.1" />
-<PackageReference Include="com.IvanMurzak.CodeExec.Roslyn" Version="0.2.1" />
+<PackageReference Include="com.IvanMurzak.CodeExec" Version="0.3.0" />
+<PackageReference Include="com.IvanMurzak.CodeExec.Roslyn" Version="0.3.0" />
 <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0" />  <!-- your choice, 3.8+ -->
 ```
 
@@ -57,6 +57,9 @@ ExecuteResult result = await runner.ExecuteAsync(new ExecuteRequest { Code = "Ap
 - Unknown simple names are imported automatically when exactly one namespace defines them (`autoImported`);
   ambiguous names are reported, never guessed. CoreLib's `Internal.*` namespaces are never imported.
 - `ValueConverter` turns the result into something that outlives the snippet (Unity: ReflectorNet JSON).
+- Output is limited before it reaches the agent ([§6](spec/contract.md#6-output-limits)): a text value, the logs (first
+  and last lines kept, 2k per line) and the error, 20k characters each by default (`MaxValueChars`, `MaxLogChars`,
+  `MaxLogLineChars`, `MaxErrorChars` on the request; 0 = no limit). Includes stay inside `IncludeRoot` unless `AllowIncludeOutsideRoot`.
 - Hosts that load assemblies from memory (Godot's game assembly) pass a `locationResolver` to
   `LoadedAssembliesReferenceProvider`, otherwise snippets cannot see those types. On CoreCLR a snippet binds to the
   copy the process already runs (any load context) before the default context is asked, so a host's
@@ -68,6 +71,7 @@ ExecuteResult result = await runner.ExecuteAsync(new ExecuteRequest { Code = "Ap
 import code_exec
 code_exec.run("x = 21\nx * 2", include=["helpers"], include_root=".ai-game-dev/snippets", imports=["unreal"])
 code_exec.run_b64(base64_json_request)   # JSON in, JSON out: one string literal from C++
+code_exec.run("print('x')", max_log_chars=0)   # the same limits as C# (JSON: maxLogChars, ...); 0 = no limit
 ```
 
 ## Development

dotnet/CodeExec.Roslyn/src/RoslynCSharpCompiler.cs

@@ -26,6 +26,7 @@ internal sealed class RoslynCSharpCompiler : ICSharpCompiler
     {
         internal const string SnippetFile = "snippet.cs";
         internal const string GeneratedFile = "<generated>";
+        internal const string UsingsFile = "<usings>";
 
         static readonly CSharpParseOptions ParseOptions = new CSharpParseOptions(LanguageVersion.Latest, DocumentationMode.None, SourceCodeKind.Regular);
         static readonly ConcurrentDictionary<string, (DateTime Stamp, MetadataReference Reference)> ReferenceCache =
@@ -88,7 +89,7 @@ public CompileResult Compile(CompileRequest request)
             }
 
             // Auto-import can only add usings to the generated snippet: full-source code and includes are compiled verbatim.
-            var result = Emit(compilation, snippet != null ? trees[0] : null);
+            var result = Emit(compilation, snippet != null ? trees[0] : null, request.Usings);
             result.EntryType = entryType;
             result.EntryMethod = entryMethod;
             result.ResultType = resultType;
@@ -110,7 +111,7 @@ public CompileResult Compile(CompileRequest request)
             return expression == null ? null : compilation.GetSemanticModel(tree).GetTypeInfo(expression).Type;
         }
 
-        static CompileResult Emit(CSharpCompilation compilation, SyntaxTree? importable)
+        static CompileResult Emit(CSharpCompilation compilation, SyntaxTree? importable, IReadOnlyList<string> usings)
         {
             using (var image = new MemoryStream())
             using (var symbols = new MemoryStream())
@@ -122,9 +123,10 @@ static CompileResult Emit(CSharpCompilation compilation, SyntaxTree? importable)
                     Success = emit.Success,
                     Image = emit.Success ? image.ToArray() : null,
                     Symbols = emit.Success ? symbols.ToArray() : null,
-                    Diagnostics = errors.Select(ToDiagnostic).ToList(),
+                    Diagnostics = errors.Select(d => ToDiagnostic(d, usings)).ToList(),
                     UnresolvedNames = errors
-                        .Where(d => (d.Id == "CS0246" || d.Id == "CS0103") && importable != null && d.Location.SourceTree == importable)
+                        .Where(d => (d.Id == "CS0246" || d.Id == "CS0103") && importable != null && d.Location.SourceTree == importable
+                            && d.Location.GetMappedLineSpan().Path != UsingsFile) // a bad using is no type to import
                         .Select(d => importable!.GetRoot().FindNode(d.Location.SourceSpan, getInnermostNodeForTie: true))
                         .Select(n => n as SimpleNameSyntax ?? n.DescendantNodesAndSelf().OfType<SimpleNameSyntax>().FirstOrDefault())
                         .Where(n => n != null)
@@ -135,18 +137,28 @@ static CompileResult Emit(CSharpCompilation compilation, SyntaxTree? importable)
             }
         }
 
-        static CodeDiagnostic ToDiagnostic(Diagnostic d)
+        static CodeDiagnostic ToDiagnostic(Diagnostic d, IReadOnlyList<string> usings)
         {
             var span = d.Location.GetMappedLineSpan();
             var file = span.IsValid ? span.Path ?? "" : "";
+            var line = span.IsValid ? span.StartLinePosition.Line + 1 : 0;
+            var column = span.IsValid ? span.StartLinePosition.Character + 1 : 0;
+            var message = d.GetMessage();
+            if (file == UsingsFile && line >= 1 && line <= usings.Count)
+            {
+                // Not the agent's code: name the using (`usings` = the host's defaults plus the request's own).
+                message = $"using '{usings[line - 1]}': {message}";
+                file = "usings";
+                line = column = 0;
+            }
             return new CodeDiagnostic
             {
                 Id = d.Id,
                 Severity = d.Severity.ToString().ToLowerInvariant(),
                 File = file == GeneratedFile ? "" : file,
-                Line = span.IsValid ? span.StartLinePosition.Line + 1 : 0,
-                Column = span.IsValid ? span.StartLinePosition.Character + 1 : 0,
-                Message = d.GetMessage(),
+                Line = line,
+                Column = column,
+                Message = message,
             };
         }
 

dotnet/CodeExec.Roslyn/src/SnippetWriter.cs

@@ -68,8 +68,9 @@ public static Output Write(string code, IReadOnlyList<string> usings, IReadOnlyL
             var sb = new StringBuilder();
             foreach (var e in root.Externs) // extern aliases must precede every using
                 Emit(sb, e.GetLocation(), e.ToString());
-            foreach (var u in usings)
-                sb.Append("using ").Append(u).Append(";\n");
+            // The runtime's usings (host defaults + request) map to their index, so a bad one is reported as itself.
+            for (var i = 0; i < usings.Count; i++)
+                sb.Append("#line ").Append(i + 1).Append(" \"").Append(RoslynCSharpCompiler.UsingsFile).Append("\"\nusing ").Append(usings[i]).Append(";\n");
             foreach (var u in root.Usings)
                 Emit(sb, u.GetLocation(), u.ToString());
             foreach (var t in topLevel)

dotnet/CodeExec.Tests/GoldenTests.cs

@@ -30,6 +30,10 @@ public async Task Golden(string name)
                 Usings = Strings(c, "usings"),
                 Include = Strings(c, "include"),
             };
+            if (c.TryGetProperty("maxValueChars", out var maxValue)) request.MaxValueChars = maxValue.GetInt32();
+            if (c.TryGetProperty("maxLogChars", out var maxLogs)) request.MaxLogChars = maxLogs.GetInt32();
+            if (c.TryGetProperty("maxLogLineChars", out var maxLine)) request.MaxLogLineChars = maxLine.GetInt32();
+            if (c.TryGetProperty("maxErrorChars", out var maxError)) request.MaxErrorChars = maxError.GetInt32();
             if (c.TryGetProperty("entry", out var entry))
             {
                 request.EntryType = entry.GetProperty("type").GetString();
@@ -39,7 +43,8 @@ public async Task Golden(string name)
             var runner = Fixtures.Runner(Fixtures.Path(Fixture.GetProperty("includeRoot").GetString()!));
             var result = await runner.ExecuteAsync(request);
             var expect = c.GetProperty("expect");
-            var describe = $"{name}: success={result.Success} kind={result.ErrorKind} value={result.Value} error={result.Error} diagnostics={string.Join(" | ", result.Diagnostics)}";
+            var shownValue = result.Value == null ? null : OutputLimits.Clip(result.Value.ToString()!, 300, "test output");
+            var describe = $"{name}: success={result.Success} kind={result.ErrorKind} value={shownValue} error={result.Error} diagnostics={string.Join(" | ", result.Diagnostics)}";
 
             Assert.True(expect.GetProperty("success").GetBoolean() == result.Success, describe);
             if (expect.TryGetProperty("value", out var value))
@@ -68,6 +73,14 @@ public async Task Golden(string name)
                 foreach (var text in logs.EnumerateArray())
                     Assert.True(result.Logs.Any(l => l.Message.Contains(text.GetString()!)), describe + " logs=" + string.Join(" | ", result.Logs));
             }
+            if (expect.TryGetProperty("logs", out var exactLogs))
+                Assert.Equal(exactLogs.EnumerateArray().Select(e => e.GetString()!).ToArray(), result.Logs.Select(l => l.Message).ToArray());
+            if (expect.TryGetProperty("truncated", out var truncated))
+                Assert.Equal(truncated.EnumerateArray().Select(e => e.GetString()!).ToArray(), result.Truncated.ToArray());
+            if (expect.TryGetProperty("valueEndsWith", out var endsWith))
+                Assert.True(result.Value is string v && v.EndsWith(endsWith.GetString()!, StringComparison.Ordinal), describe);
+            if (expect.TryGetProperty("valueLength", out var length))
+                Assert.True(result.Value is string v && v.Length == length.GetInt32(), describe);
         }
 
         static string[] Strings(JsonElement c, string property) =>

dotnet/CodeExec.Tests/RunnerTests.cs

@@ -510,5 +510,185 @@ public void RuntimeStackTracePointsAtSnippetLines(string code)
         public void FormatsTypeNames(Type type, string expected) => Assert.Equal(expected, TypeNames.ToCSharp(type));
 
         public static class Outer { public sealed class Inner { } }
+
+        // A host that cannot unload reuses a snippet's assembly only when the snippet has no static state of its own:
+        // every run of a stateful one starts from fresh statics, as on a host that unloads.
+        [Theory]
+        [InlineData("public static class Ctr { public static int N; }\n++Ctr.N", "1", true)]
+        [InlineData("static int counter = 3;\n++counter", "4", true)]
+        [InlineData("public static class K { public static readonly List<int> L = new List<int>(); }\nK.L.Add(1);\nK.L.Count", "1", true)]
+        [InlineData("public static class P { public static int N { get; set; } }\n++P.N", "1", true)]
+        [InlineData("public static class Cache { public static Func<int> F; public static int Calls; }\nCache.F ??= () => ++Cache.Calls;\nCache.F()", "1", true)]
+        [InlineData("Func<int, int> twice = x => x * 2;\nnew[] { 1, 2 }.Select(twice).Sum()", "6", false)] // lambda caches only
+        [InlineData("var seen = new List<int>();\nFunc<int, int> twice = x => x * 2;\nseen.Add(twice(1));\nnew[] { 1, 2 }.Sum() + seen.Count", "4", false)]
+        [InlineData("const int K = 2;\nnew[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.Length * K", "20", false)] // array data only
+        public void StatefulSnippetsNeverSeeAnotherRunsStatics(string code, string expected, bool stateful)
+        {
+            var runner = Fixtures.Runner(configure: o => o.Host = new DefaultAssemblyHost());
+            for (var i = 0; i < 3; i++)
+            {
+                var result = runner.Execute(new ExecuteRequest { Code = code });
+                Assert.True(result.Success, result.Error);
+                Assert.Equal(expected, result.Value);
+            }
+            // A stateful snippet loads on every run. A stateless one is reused (the first run may load twice: assemblies
+            // the process loads during it change the reference set, hence the cache key).
+            if (stateful)
+                Assert.Equal(3, runner.Stats.LoadedAssemblies);
+            else
+                Assert.True(runner.Stats.LoadedAssemblies < 3, "loads: " + runner.Stats.LoadedAssemblies);
+        }
+
+        [Fact]
+        public void FailingStaticInitializerOnReuseIsARuntimeError()
+        {
+            var runner = Fixtures.Runner(configure: o => o.Host = new DefaultAssemblyHost());
+            const string code = "public static class Boom { public static int Runs; static Boom() { if (System.Environment.GetEnvironmentVariable(\"CODEEXEC_TEST_BOOM\") == \"1\") throw new InvalidOperationException(\"init\"); } }\n++Boom.Runs";
+            Assert.Equal("1", runner.Execute(new ExecuteRequest { Code = code }).Value);
+            Environment.SetEnvironmentVariable("CODEEXEC_TEST_BOOM", "1");
+            try
+            {
+                var result = runner.Execute(new ExecuteRequest { Code = code });
+                Assert.Equal(ErrorKind.Runtime, result.ErrorKind);
+                Assert.Contains("init", result.Error);
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("CODEEXEC_TEST_BOOM", null);
+            }
+        }
+
+        [Fact]
+        public void BadUsingIsReportedAsTheUsing()
+        {
+            var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "1", Usings = new[] { "No.Such.Namespace" } });
+            Assert.Equal(ErrorKind.Compile, result.ErrorKind);
+            var diagnostic = Assert.Single(result.Diagnostics);
+            Assert.Equal("usings", diagnostic.File);
+            Assert.Equal(0, diagnostic.Line);
+            Assert.StartsWith("using 'No.Such.Namespace': ", diagnostic.Message);
+        }
+
+        [Fact]
+        public void SnippetErrorsKeepTheirCoordinatesNextToUsings()
+        {
+            var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "var a = 1;\nundefinedThing", Usings = new[] { "System.Text" } });
+            var diagnostic = Assert.Single(result.Diagnostics);
+            Assert.Equal("snippet.cs", diagnostic.File);
+            Assert.Equal(2, diagnostic.Line);
+        }
+
+        [Fact]
+        public void DiagnosticsAreCapped()
+        {
+            var code = string.Join("\n", Enumerable.Range(0, 30).Select(i => $"int x{i} = \"s\";"));
+            var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = code });
+            Assert.Equal(ErrorKind.Compile, result.ErrorKind);
+            Assert.Equal(OutputLimits.MaxDiagnostics + 1, result.Diagnostics.Count);
+            Assert.Equal("CE0002", result.Diagnostics.Last().Id);
+            Assert.StartsWith("10 more diagnostic(s) left out", result.Diagnostics.Last().Message);
+        }
+
+        [Fact]
+        public void IncludeOutsideTheRootNeedsTheHostsConsent()
+        {
+            var root = Fixtures.Path("fixtures/include/csharp/geo");
+            var outside = Fixtures.Path("fixtures/include/csharp/Broken.cs");
+            foreach (var entry in new[] { "../Broken.cs", outside })
+            {
+                var refused = Fixtures.Runner(root).Execute(new ExecuteRequest { Code = "1", Include = new[] { entry } });
+                Assert.Equal(ErrorKind.InvalidRequest, refused.ErrorKind);
+                Assert.Contains("outside the include root", refused.Error);
+            }
+            var allowed = Fixtures.Runner(root, o => o.AllowIncludeOutsideRoot = true).Execute(new ExecuteRequest { Code = "1", Include = new[] { "../Broken.cs" } });
+            Assert.Equal(ErrorKind.Compile, allowed.ErrorKind); // Broken.cs is read and compiled now
+            // An absolute path inside the root is fine, and is reported relative to it.
+            var inside = Fixtures.Runner(root).Execute(new ExecuteRequest { Code = "1", Include = new[] { root } });
+            Assert.True(inside.Success, inside.Error);
+        }
+
+        [Fact]
+        public void InnerExceptionsAreReportedAndBounded()
+        {
+            var result = Fixtures.Runner().Execute(new ExecuteRequest
+            {
+                Code = "Exception e = new ArgumentException(\"cause 0\");\nfor (var i = 1; i < 9; i++) e = new InvalidOperationException(\"cause \" + i, e);\nthrow e;",
+            });
+            Assert.Equal(ErrorKind.Runtime, result.ErrorKind);
+            Assert.StartsWith("System.InvalidOperationException: cause 8", result.Error);
+            Assert.Contains(" ---> System.InvalidOperationException: cause 3", result.Error);
+            Assert.DoesNotContain("cause 2", result.Error);
+            Assert.Contains(" ---> ... more inner exception(s) left out", result.Error);
+        }
+
+        [Fact]
+        public void RunawayConsoleOutputKeepsFirstAndLastLines()
+        {
+            var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "for (var i = 0; i < 20000; i++) Console.WriteLine(\"line \" + i);" });
+            Assert.True(result.Success, result.Error);
+            Assert.Equal("line 0", result.Logs.First().Message);
+            Assert.Equal("line 19999", result.Logs.Last().Message);
+            Assert.Single(result.Logs, l => l.Level == "warning" && l.Message.Contains("log line(s) left out"));
+            Assert.True(result.Logs.Where(l => l.Level != "warning").Sum(l => l.Message.Length + OutputLimits.LogEntryChars) <= OutputLimits.DefaultLogChars);
+            Assert.Equal(new[] { "logs" }, result.Truncated);
+        }
+
+        [Fact]
+        public void NonTextValueIsLeftToTheHost()
+        {
+            var big = Enumerable.Range(0, 200000).ToArray();
+            var result = Fixtures.Runner(configure: o => o.ValueConverter = v => v is int[] a ? a.Length : v).Execute(new ExecuteRequest { Code = "Enumerable.Range(0, 200000).ToArray()" });
+            Assert.Equal(big.Length, result.Value);
+            Assert.Empty(result.Truncated);
+        }
+
+        [Fact]
+        public void ClipNeverSplitsASurrogatePair()
+        {
+            var clipped = OutputLimits.Clip("ab\U0001F600cd", 3, "hint");
+            Assert.Equal("ab... [truncated: 2 of 6 characters shown; hint]", clipped);
+            Assert.Equal("abc", OutputLimits.Clip("abc", 3, "hint"));
+            Assert.Equal("abcdef", OutputLimits.Clip("abcdef", 0, "hint"));
+        }
+
+        [Fact]
+        public void EmptyIncludeRootIsFineWithoutIncludes()
+        {
+            var result = Fixtures.Runner(configure: o => o.IncludeRoot = "").Execute(new ExecuteRequest { Code = "1" });
+            Assert.True(result.Success, result.Error);
+        }
+
+        sealed class NullMessageCapture : ILogCapture
+        {
+            public IDisposable Begin(Action<LogEntry> sink)
+            {
+                sink(new LogEntry("info", null!));
+                return new System.IO.StringWriter(); // anything disposable
+            }
+        }
+
+        [Fact]
+        public void HostLogEntryWithoutAMessageIsKept()
+        {
+            var result = Fixtures.Runner(configure: o => o.LogCaptures.Add(new NullMessageCapture())).Execute(new ExecuteRequest { Code = "1" });
+            Assert.True(result.Success, result.Error);
+            Assert.Contains(result.Logs, l => l.Message == "");
+        }
+
+        [Fact]
+        public void EarlyErrorsAreClippedToo()
+        {
+            var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "1", Include = new[] { new string('d', 300) + ".cs" }, MaxErrorChars = 30 });
+            Assert.Equal(ErrorKind.InvalidRequest, result.ErrorKind);
+            Assert.EndsWith("characters shown; raise a shorter message]", result.Error);
+            Assert.Equal(new[] { "error" }, result.Truncated);
+        }
+
+        [Fact]
+        public void NegativeLimitIsAnInvalidRequest()
+        {
+            var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "1", MaxErrorChars = -5 });
+            Assert.Equal(ErrorKind.InvalidRequest, result.ErrorKind);
+        }
     }
 }