diff --git a/README.md b/README.md
index bc8e01d..404547e 100644
--- a/README.md
+++ b/README.md
@@ -38,8 +38,8 @@ Every host already has its own Roslyn constraints (Unity pins 4.8 because of `Sy
## C# in two lines
```xml
-
-
+
+
```
@@ -57,6 +57,9 @@ ExecuteResult result = await runner.ExecuteAsync(new ExecuteRequest { Code = "Ap
- Unknown simple names are imported automatically when exactly one namespace defines them (`autoImported`);
ambiguous names are reported, never guessed. CoreLib's `Internal.*` namespaces are never imported.
- `ValueConverter` turns the result into something that outlives the snippet (Unity: ReflectorNet JSON).
+- Output is limited before it reaches the agent ([§6](spec/contract.md#6-output-limits)): a text value, the logs (first
+ and last lines kept, 2k per line) and the error, 20k characters each by default (`MaxValueChars`, `MaxLogChars`,
+ `MaxLogLineChars`, `MaxErrorChars` on the request; 0 = no limit). Includes stay inside `IncludeRoot` unless `AllowIncludeOutsideRoot`.
- Hosts that load assemblies from memory (Godot's game assembly) pass a `locationResolver` to
`LoadedAssembliesReferenceProvider`, otherwise snippets cannot see those types. On CoreCLR a snippet binds to the
copy the process already runs (any load context) before the default context is asked, so a host's
@@ -68,6 +71,7 @@ ExecuteResult result = await runner.ExecuteAsync(new ExecuteRequest { Code = "Ap
import code_exec
code_exec.run("x = 21\nx * 2", include=["helpers"], include_root=".ai-game-dev/snippets", imports=["unreal"])
code_exec.run_b64(base64_json_request) # JSON in, JSON out: one string literal from C++
+code_exec.run("print('x')", max_log_chars=0) # the same limits as C# (JSON: maxLogChars, ...); 0 = no limit
```
## Development
diff --git a/dotnet/CodeExec.Roslyn/src/RoslynCSharpCompiler.cs b/dotnet/CodeExec.Roslyn/src/RoslynCSharpCompiler.cs
index 9045469..7f4669c 100644
--- a/dotnet/CodeExec.Roslyn/src/RoslynCSharpCompiler.cs
+++ b/dotnet/CodeExec.Roslyn/src/RoslynCSharpCompiler.cs
@@ -26,6 +26,7 @@ internal sealed class RoslynCSharpCompiler : ICSharpCompiler
{
internal const string SnippetFile = "snippet.cs";
internal const string GeneratedFile = "";
+ internal const string UsingsFile = "";
static readonly CSharpParseOptions ParseOptions = new CSharpParseOptions(LanguageVersion.Latest, DocumentationMode.None, SourceCodeKind.Regular);
static readonly ConcurrentDictionary ReferenceCache =
@@ -88,7 +89,7 @@ public CompileResult Compile(CompileRequest request)
}
// Auto-import can only add usings to the generated snippet: full-source code and includes are compiled verbatim.
- var result = Emit(compilation, snippet != null ? trees[0] : null);
+ var result = Emit(compilation, snippet != null ? trees[0] : null, request.Usings);
result.EntryType = entryType;
result.EntryMethod = entryMethod;
result.ResultType = resultType;
@@ -110,7 +111,7 @@ public CompileResult Compile(CompileRequest request)
return expression == null ? null : compilation.GetSemanticModel(tree).GetTypeInfo(expression).Type;
}
- static CompileResult Emit(CSharpCompilation compilation, SyntaxTree? importable)
+ static CompileResult Emit(CSharpCompilation compilation, SyntaxTree? importable, IReadOnlyList usings)
{
using (var image = new MemoryStream())
using (var symbols = new MemoryStream())
@@ -122,9 +123,10 @@ static CompileResult Emit(CSharpCompilation compilation, SyntaxTree? importable)
Success = emit.Success,
Image = emit.Success ? image.ToArray() : null,
Symbols = emit.Success ? symbols.ToArray() : null,
- Diagnostics = errors.Select(ToDiagnostic).ToList(),
+ Diagnostics = errors.Select(d => ToDiagnostic(d, usings)).ToList(),
UnresolvedNames = errors
- .Where(d => (d.Id == "CS0246" || d.Id == "CS0103") && importable != null && d.Location.SourceTree == importable)
+ .Where(d => (d.Id == "CS0246" || d.Id == "CS0103") && importable != null && d.Location.SourceTree == importable
+ && d.Location.GetMappedLineSpan().Path != UsingsFile) // a bad using is no type to import
.Select(d => importable!.GetRoot().FindNode(d.Location.SourceSpan, getInnermostNodeForTie: true))
.Select(n => n as SimpleNameSyntax ?? n.DescendantNodesAndSelf().OfType().FirstOrDefault())
.Where(n => n != null)
@@ -135,18 +137,28 @@ static CompileResult Emit(CSharpCompilation compilation, SyntaxTree? importable)
}
}
- static CodeDiagnostic ToDiagnostic(Diagnostic d)
+ static CodeDiagnostic ToDiagnostic(Diagnostic d, IReadOnlyList usings)
{
var span = d.Location.GetMappedLineSpan();
var file = span.IsValid ? span.Path ?? "" : "";
+ var line = span.IsValid ? span.StartLinePosition.Line + 1 : 0;
+ var column = span.IsValid ? span.StartLinePosition.Character + 1 : 0;
+ var message = d.GetMessage();
+ if (file == UsingsFile && line >= 1 && line <= usings.Count)
+ {
+ // Not the agent's code: name the using (`usings` = the host's defaults plus the request's own).
+ message = $"using '{usings[line - 1]}': {message}";
+ file = "usings";
+ line = column = 0;
+ }
return new CodeDiagnostic
{
Id = d.Id,
Severity = d.Severity.ToString().ToLowerInvariant(),
File = file == GeneratedFile ? "" : file,
- Line = span.IsValid ? span.StartLinePosition.Line + 1 : 0,
- Column = span.IsValid ? span.StartLinePosition.Character + 1 : 0,
- Message = d.GetMessage(),
+ Line = line,
+ Column = column,
+ Message = message,
};
}
diff --git a/dotnet/CodeExec.Roslyn/src/SnippetWriter.cs b/dotnet/CodeExec.Roslyn/src/SnippetWriter.cs
index 1306ef0..a9c1f6f 100644
--- a/dotnet/CodeExec.Roslyn/src/SnippetWriter.cs
+++ b/dotnet/CodeExec.Roslyn/src/SnippetWriter.cs
@@ -68,8 +68,9 @@ public static Output Write(string code, IReadOnlyList usings, IReadOnlyL
var sb = new StringBuilder();
foreach (var e in root.Externs) // extern aliases must precede every using
Emit(sb, e.GetLocation(), e.ToString());
- foreach (var u in usings)
- sb.Append("using ").Append(u).Append(";\n");
+ // The runtime's usings (host defaults + request) map to their index, so a bad one is reported as itself.
+ for (var i = 0; i < usings.Count; i++)
+ sb.Append("#line ").Append(i + 1).Append(" \"").Append(RoslynCSharpCompiler.UsingsFile).Append("\"\nusing ").Append(usings[i]).Append(";\n");
foreach (var u in root.Usings)
Emit(sb, u.GetLocation(), u.ToString());
foreach (var t in topLevel)
diff --git a/dotnet/CodeExec.Tests/GoldenTests.cs b/dotnet/CodeExec.Tests/GoldenTests.cs
index 9b482e6..3f8bad1 100644
--- a/dotnet/CodeExec.Tests/GoldenTests.cs
+++ b/dotnet/CodeExec.Tests/GoldenTests.cs
@@ -30,6 +30,10 @@ public async Task Golden(string name)
Usings = Strings(c, "usings"),
Include = Strings(c, "include"),
};
+ if (c.TryGetProperty("maxValueChars", out var maxValue)) request.MaxValueChars = maxValue.GetInt32();
+ if (c.TryGetProperty("maxLogChars", out var maxLogs)) request.MaxLogChars = maxLogs.GetInt32();
+ if (c.TryGetProperty("maxLogLineChars", out var maxLine)) request.MaxLogLineChars = maxLine.GetInt32();
+ if (c.TryGetProperty("maxErrorChars", out var maxError)) request.MaxErrorChars = maxError.GetInt32();
if (c.TryGetProperty("entry", out var entry))
{
request.EntryType = entry.GetProperty("type").GetString();
@@ -39,7 +43,8 @@ public async Task Golden(string name)
var runner = Fixtures.Runner(Fixtures.Path(Fixture.GetProperty("includeRoot").GetString()!));
var result = await runner.ExecuteAsync(request);
var expect = c.GetProperty("expect");
- var describe = $"{name}: success={result.Success} kind={result.ErrorKind} value={result.Value} error={result.Error} diagnostics={string.Join(" | ", result.Diagnostics)}";
+ var shownValue = result.Value == null ? null : OutputLimits.Clip(result.Value.ToString()!, 300, "test output");
+ var describe = $"{name}: success={result.Success} kind={result.ErrorKind} value={shownValue} error={result.Error} diagnostics={string.Join(" | ", result.Diagnostics)}";
Assert.True(expect.GetProperty("success").GetBoolean() == result.Success, describe);
if (expect.TryGetProperty("value", out var value))
@@ -68,6 +73,14 @@ public async Task Golden(string name)
foreach (var text in logs.EnumerateArray())
Assert.True(result.Logs.Any(l => l.Message.Contains(text.GetString()!)), describe + " logs=" + string.Join(" | ", result.Logs));
}
+ if (expect.TryGetProperty("logs", out var exactLogs))
+ Assert.Equal(exactLogs.EnumerateArray().Select(e => e.GetString()!).ToArray(), result.Logs.Select(l => l.Message).ToArray());
+ if (expect.TryGetProperty("truncated", out var truncated))
+ Assert.Equal(truncated.EnumerateArray().Select(e => e.GetString()!).ToArray(), result.Truncated.ToArray());
+ if (expect.TryGetProperty("valueEndsWith", out var endsWith))
+ Assert.True(result.Value is string v && v.EndsWith(endsWith.GetString()!, StringComparison.Ordinal), describe);
+ if (expect.TryGetProperty("valueLength", out var length))
+ Assert.True(result.Value is string v && v.Length == length.GetInt32(), describe);
}
static string[] Strings(JsonElement c, string property) =>
diff --git a/dotnet/CodeExec.Tests/RunnerTests.cs b/dotnet/CodeExec.Tests/RunnerTests.cs
index d46aa9a..3a92da0 100644
--- a/dotnet/CodeExec.Tests/RunnerTests.cs
+++ b/dotnet/CodeExec.Tests/RunnerTests.cs
@@ -510,5 +510,185 @@ public void RuntimeStackTracePointsAtSnippetLines(string code)
public void FormatsTypeNames(Type type, string expected) => Assert.Equal(expected, TypeNames.ToCSharp(type));
public static class Outer { public sealed class Inner { } }
+
+ // A host that cannot unload reuses a snippet's assembly only when the snippet has no static state of its own:
+ // every run of a stateful one starts from fresh statics, as on a host that unloads.
+ [Theory]
+ [InlineData("public static class Ctr { public static int N; }\n++Ctr.N", "1", true)]
+ [InlineData("static int counter = 3;\n++counter", "4", true)]
+ [InlineData("public static class K { public static readonly List L = new List(); }\nK.L.Add(1);\nK.L.Count", "1", true)]
+ [InlineData("public static class P { public static int N { get; set; } }\n++P.N", "1", true)]
+ [InlineData("public static class Cache { public static Func F; public static int Calls; }\nCache.F ??= () => ++Cache.Calls;\nCache.F()", "1", true)]
+ [InlineData("Func twice = x => x * 2;\nnew[] { 1, 2 }.Select(twice).Sum()", "6", false)] // lambda caches only
+ [InlineData("var seen = new List();\nFunc twice = x => x * 2;\nseen.Add(twice(1));\nnew[] { 1, 2 }.Sum() + seen.Count", "4", false)]
+ [InlineData("const int K = 2;\nnew[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.Length * K", "20", false)] // array data only
+ public void StatefulSnippetsNeverSeeAnotherRunsStatics(string code, string expected, bool stateful)
+ {
+ var runner = Fixtures.Runner(configure: o => o.Host = new DefaultAssemblyHost());
+ for (var i = 0; i < 3; i++)
+ {
+ var result = runner.Execute(new ExecuteRequest { Code = code });
+ Assert.True(result.Success, result.Error);
+ Assert.Equal(expected, result.Value);
+ }
+ // A stateful snippet loads on every run. A stateless one is reused (the first run may load twice: assemblies
+ // the process loads during it change the reference set, hence the cache key).
+ if (stateful)
+ Assert.Equal(3, runner.Stats.LoadedAssemblies);
+ else
+ Assert.True(runner.Stats.LoadedAssemblies < 3, "loads: " + runner.Stats.LoadedAssemblies);
+ }
+
+ [Fact]
+ public void FailingStaticInitializerOnReuseIsARuntimeError()
+ {
+ var runner = Fixtures.Runner(configure: o => o.Host = new DefaultAssemblyHost());
+ const string code = "public static class Boom { public static int Runs; static Boom() { if (System.Environment.GetEnvironmentVariable(\"CODEEXEC_TEST_BOOM\") == \"1\") throw new InvalidOperationException(\"init\"); } }\n++Boom.Runs";
+ Assert.Equal("1", runner.Execute(new ExecuteRequest { Code = code }).Value);
+ Environment.SetEnvironmentVariable("CODEEXEC_TEST_BOOM", "1");
+ try
+ {
+ var result = runner.Execute(new ExecuteRequest { Code = code });
+ Assert.Equal(ErrorKind.Runtime, result.ErrorKind);
+ Assert.Contains("init", result.Error);
+ }
+ finally
+ {
+ Environment.SetEnvironmentVariable("CODEEXEC_TEST_BOOM", null);
+ }
+ }
+
+ [Fact]
+ public void BadUsingIsReportedAsTheUsing()
+ {
+ var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "1", Usings = new[] { "No.Such.Namespace" } });
+ Assert.Equal(ErrorKind.Compile, result.ErrorKind);
+ var diagnostic = Assert.Single(result.Diagnostics);
+ Assert.Equal("usings", diagnostic.File);
+ Assert.Equal(0, diagnostic.Line);
+ Assert.StartsWith("using 'No.Such.Namespace': ", diagnostic.Message);
+ }
+
+ [Fact]
+ public void SnippetErrorsKeepTheirCoordinatesNextToUsings()
+ {
+ var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "var a = 1;\nundefinedThing", Usings = new[] { "System.Text" } });
+ var diagnostic = Assert.Single(result.Diagnostics);
+ Assert.Equal("snippet.cs", diagnostic.File);
+ Assert.Equal(2, diagnostic.Line);
+ }
+
+ [Fact]
+ public void DiagnosticsAreCapped()
+ {
+ var code = string.Join("\n", Enumerable.Range(0, 30).Select(i => $"int x{i} = \"s\";"));
+ var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = code });
+ Assert.Equal(ErrorKind.Compile, result.ErrorKind);
+ Assert.Equal(OutputLimits.MaxDiagnostics + 1, result.Diagnostics.Count);
+ Assert.Equal("CE0002", result.Diagnostics.Last().Id);
+ Assert.StartsWith("10 more diagnostic(s) left out", result.Diagnostics.Last().Message);
+ }
+
+ [Fact]
+ public void IncludeOutsideTheRootNeedsTheHostsConsent()
+ {
+ var root = Fixtures.Path("fixtures/include/csharp/geo");
+ var outside = Fixtures.Path("fixtures/include/csharp/Broken.cs");
+ foreach (var entry in new[] { "../Broken.cs", outside })
+ {
+ var refused = Fixtures.Runner(root).Execute(new ExecuteRequest { Code = "1", Include = new[] { entry } });
+ Assert.Equal(ErrorKind.InvalidRequest, refused.ErrorKind);
+ Assert.Contains("outside the include root", refused.Error);
+ }
+ var allowed = Fixtures.Runner(root, o => o.AllowIncludeOutsideRoot = true).Execute(new ExecuteRequest { Code = "1", Include = new[] { "../Broken.cs" } });
+ Assert.Equal(ErrorKind.Compile, allowed.ErrorKind); // Broken.cs is read and compiled now
+ // An absolute path inside the root is fine, and is reported relative to it.
+ var inside = Fixtures.Runner(root).Execute(new ExecuteRequest { Code = "1", Include = new[] { root } });
+ Assert.True(inside.Success, inside.Error);
+ }
+
+ [Fact]
+ public void InnerExceptionsAreReportedAndBounded()
+ {
+ var result = Fixtures.Runner().Execute(new ExecuteRequest
+ {
+ Code = "Exception e = new ArgumentException(\"cause 0\");\nfor (var i = 1; i < 9; i++) e = new InvalidOperationException(\"cause \" + i, e);\nthrow e;",
+ });
+ Assert.Equal(ErrorKind.Runtime, result.ErrorKind);
+ Assert.StartsWith("System.InvalidOperationException: cause 8", result.Error);
+ Assert.Contains(" ---> System.InvalidOperationException: cause 3", result.Error);
+ Assert.DoesNotContain("cause 2", result.Error);
+ Assert.Contains(" ---> ... more inner exception(s) left out", result.Error);
+ }
+
+ [Fact]
+ public void RunawayConsoleOutputKeepsFirstAndLastLines()
+ {
+ var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "for (var i = 0; i < 20000; i++) Console.WriteLine(\"line \" + i);" });
+ Assert.True(result.Success, result.Error);
+ Assert.Equal("line 0", result.Logs.First().Message);
+ Assert.Equal("line 19999", result.Logs.Last().Message);
+ Assert.Single(result.Logs, l => l.Level == "warning" && l.Message.Contains("log line(s) left out"));
+ Assert.True(result.Logs.Where(l => l.Level != "warning").Sum(l => l.Message.Length + OutputLimits.LogEntryChars) <= OutputLimits.DefaultLogChars);
+ Assert.Equal(new[] { "logs" }, result.Truncated);
+ }
+
+ [Fact]
+ public void NonTextValueIsLeftToTheHost()
+ {
+ var big = Enumerable.Range(0, 200000).ToArray();
+ var result = Fixtures.Runner(configure: o => o.ValueConverter = v => v is int[] a ? a.Length : v).Execute(new ExecuteRequest { Code = "Enumerable.Range(0, 200000).ToArray()" });
+ Assert.Equal(big.Length, result.Value);
+ Assert.Empty(result.Truncated);
+ }
+
+ [Fact]
+ public void ClipNeverSplitsASurrogatePair()
+ {
+ var clipped = OutputLimits.Clip("ab\U0001F600cd", 3, "hint");
+ Assert.Equal("ab... [truncated: 2 of 6 characters shown; hint]", clipped);
+ Assert.Equal("abc", OutputLimits.Clip("abc", 3, "hint"));
+ Assert.Equal("abcdef", OutputLimits.Clip("abcdef", 0, "hint"));
+ }
+
+ [Fact]
+ public void EmptyIncludeRootIsFineWithoutIncludes()
+ {
+ var result = Fixtures.Runner(configure: o => o.IncludeRoot = "").Execute(new ExecuteRequest { Code = "1" });
+ Assert.True(result.Success, result.Error);
+ }
+
+ sealed class NullMessageCapture : ILogCapture
+ {
+ public IDisposable Begin(Action sink)
+ {
+ sink(new LogEntry("info", null!));
+ return new System.IO.StringWriter(); // anything disposable
+ }
+ }
+
+ [Fact]
+ public void HostLogEntryWithoutAMessageIsKept()
+ {
+ var result = Fixtures.Runner(configure: o => o.LogCaptures.Add(new NullMessageCapture())).Execute(new ExecuteRequest { Code = "1" });
+ Assert.True(result.Success, result.Error);
+ Assert.Contains(result.Logs, l => l.Message == "");
+ }
+
+ [Fact]
+ public void EarlyErrorsAreClippedToo()
+ {
+ var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "1", Include = new[] { new string('d', 300) + ".cs" }, MaxErrorChars = 30 });
+ Assert.Equal(ErrorKind.InvalidRequest, result.ErrorKind);
+ Assert.EndsWith("characters shown; raise a shorter message]", result.Error);
+ Assert.Equal(new[] { "error" }, result.Truncated);
+ }
+
+ [Fact]
+ public void NegativeLimitIsAnInvalidRequest()
+ {
+ var result = Fixtures.Runner().Execute(new ExecuteRequest { Code = "1", MaxErrorChars = -5 });
+ Assert.Equal(ErrorKind.InvalidRequest, result.ErrorKind);
+ }
}
}
diff --git a/dotnet/CodeExec/CodeRunner.cs b/dotnet/CodeExec/CodeRunner.cs
index 507dfcc..7c0b9d1 100644
--- a/dotnet/CodeExec/CodeRunner.cs
+++ b/dotnet/CodeExec/CodeRunner.cs
@@ -20,6 +20,9 @@ public sealed class CodeRunnerOptions
/// Folder that relative include paths resolve against.
public string IncludeRoot { get; set; } = Directory.GetCurrentDirectory();
+ /// Accept includes outside (absolute paths, ..). Off by default (spec §1).
+ public bool AllowIncludeOutsideRoot { get; set; }
+
/// Defaults to .
public IAssemblyHost? Host { get; set; }
@@ -150,22 +153,31 @@ public PreparedSnippet Prepare(ExecuteRequest request)
if (request == null)
throw new ArgumentNullException(nameof(request));
var stopwatch = Stopwatch.StartNew();
- PreparedSnippet Fail(ErrorKind kind, string error) =>
- new PreparedSnippet(request, ExecuteResult.Fail(kind, error), null, false, stopwatch.ElapsedMilliseconds);
+ PreparedSnippet Fail(ErrorKind kind, string error)
+ {
+ // An early error can quote the request (an include path): the error limit applies to it too.
+ var failure = ExecuteResult.Fail(kind, OutputLimits.Clip(error, Math.Max(request.MaxErrorChars, 0), OutputLimits.ErrorHint));
+ if (!ReferenceEquals(failure.Error, error))
+ failure.Truncated = new[] { "error" };
+ return new PreparedSnippet(request, failure, null, false, stopwatch.ElapsedMilliseconds);
+ }
if (string.IsNullOrWhiteSpace(request.Code))
return Fail(ErrorKind.InvalidRequest, "code is empty");
if (string.IsNullOrWhiteSpace(request.EntryType) && !string.IsNullOrWhiteSpace(request.EntryMethod))
return Fail(ErrorKind.InvalidRequest, "entry method given without entry type");
+ if (request.MaxValueChars < 0 || request.MaxLogChars < 0 || request.MaxLogLineChars < 0 || request.MaxErrorChars < 0)
+ return Fail(ErrorKind.InvalidRequest, "output limits must be >= 0 (0 = no limit)");
List parameters;
List includes;
try
{
parameters = request.Parameters.Select(p => new ParameterDeclaration(p.Name, TypeNames.ToCSharp(p.Type))).ToList();
- includes = IncludeResolver.Read(request.Include, _options.IncludeRoot);
+ includes = IncludeResolver.Read(request.Include, _options.IncludeRoot, _options.AllowIncludeOutsideRoot);
}
- catch (Exception e) when (e is IOException || e is ArgumentException || e is UnauthorizedAccessException)
+ catch (Exception e) when (e is IOException || e is ArgumentException || e is UnauthorizedAccessException
+ || e is NotSupportedException || e is System.Security.SecurityException) // bad include paths (`a.cs:zz` on .NET Framework)
{
return Fail(ErrorKind.InvalidRequest, e.Message);
}
@@ -228,7 +240,7 @@ PreparedSnippet Fail(ErrorKind kind, string error) =>
if (compiled.Image == null)
{
var failure = ExecuteResult.Fail(ErrorKind.Compile, "Compilation failed");
- failure.Diagnostics = compiled.Diagnostics.Concat(ambiguous).ToList();
+ failure.Diagnostics = OutputLimits.Diagnostics(ambiguous.Concat(compiled.Diagnostics)); // the actionable CE0001 first
failure.AutoImported = autoImported;
failure.CompileMs = stopwatch.ElapsedMilliseconds;
return new PreparedSnippet(request, failure, null, false, failure.CompileMs);
@@ -311,7 +323,8 @@ Execution Start(PreparedSnippet prepared)
///
/// Loads the entry's assembly. A host that cannot unload (Mono) keeps ONE loaded entry per cache key: it loads
/// outside the cache lock, publishes under it, and reuses whatever was published first. The image bytes are dropped
- /// and the key leaves the LRU, which only holds images still to load.
+ /// and the key leaves the LRU, which only holds images still to load. A snippet with static state of its own is
+ /// never published: each run loads a fresh copy, so no run sees another run's statics (spec §3.5).
///
ILoadedAssembly Load(CacheEntry entry)
{
@@ -329,6 +342,9 @@ ILoadedAssembly Load(CacheEntry entry)
var loaded = _host.Load(image, symbols, entry.AssemblyName);
Interlocked.Increment(ref Stats.loadedAssemblies); // permanent, even when a concurrent run wins below
+ entry.HasStaticState ??= StaticState.Has(loaded.Assembly); // the same for every copy: reflect once
+ if (entry.HasStaticState == true)
+ return loaded; // stays an image in the LRU: the next identical request loads it again
lock (_cacheLock)
{
if (_loaded.TryGetValue(entry.Key, out var published))
@@ -408,25 +424,54 @@ public string YieldsName
}
}
- /// Type: message plus only the stack frames of the snippet's own assembly (snippet and includes).
+ ///
+ /// Type: message plus only the stack frames of the snippet's own assembly (snippet and includes): the first and
+ /// the last of them; then each inner exception the same way after
+ /// ---> , at most . Never throws, whatever the exception's members do.
+ ///
internal static string FormatException(Exception exception, Assembly? snippet)
{
while ((exception is TargetInvocationException || exception is AggregateException) && exception.InnerException != null)
exception = exception.InnerException;
- var sb = new StringBuilder(exception.GetType().FullName).Append(": ").Append(exception.Message);
+ var sb = new StringBuilder();
+ AppendException(sb, exception, snippet);
+ var inner = exception.InnerException;
+ for (var depth = 0; inner != null && depth < OutputLimits.MaxCauses; depth++, inner = inner.InnerException)
+ AppendException(sb.AppendLine().Append(" ---> "), inner, snippet);
+ if (inner != null)
+ sb.AppendLine().Append(" ---> ... more inner exception(s) left out");
+ return sb.ToString();
+ }
+
+ static void AppendException(StringBuilder sb, Exception exception, Assembly? snippet)
+ {
+ string message;
+ try { message = exception.Message; }
+ catch (Exception) { message = ""; }
+ sb.Append(exception.GetType().FullName).Append(": ").Append(message);
if (snippet == null)
- return sb.ToString();
- foreach (var frame in new StackTrace(exception, true).GetFrames() ?? Array.Empty())
+ return;
+ var frames = new List();
+ try
{
- var method = frame.GetMethod();
- if (method?.DeclaringType?.Assembly != snippet)
- continue;
- sb.AppendLine().Append(" at ").Append(method.DeclaringType.FullName).Append('.').Append(method.Name);
- var file = frame.GetFileName();
- if (!string.IsNullOrEmpty(file))
- sb.Append(" in ").Append(file).Append(":line ").Append(frame.GetFileLineNumber());
+ foreach (var frame in new StackTrace(exception, true).GetFrames() ?? Array.Empty())
+ {
+ var method = frame.GetMethod();
+ if (method?.DeclaringType?.Assembly != snippet)
+ continue;
+ var line = " at " + method.DeclaringType.FullName + "." + method.Name;
+ var file = frame.GetFileName();
+ if (!string.IsNullOrEmpty(file))
+ line += " in " + file + ":line " + frame.GetFileLineNumber();
+ frames.Add(line);
+ }
}
- return sb.ToString();
+ catch (Exception) { } // a stack that cannot be read: the message alone
+ const int kept = OutputLimits.FramesKept;
+ if (frames.Count > 2 * kept)
+ frames = frames.Take(kept).Concat(new[] { $" ... {frames.Count - 2 * kept} frame(s) left out" }).Concat(frames.Skip(frames.Count - kept)).ToList();
+ foreach (var line in frames)
+ sb.AppendLine().Append(line);
}
static string NormalizeUsing(string value)
@@ -531,6 +576,9 @@ public CacheEntry(string key, string assemblyName, CompileResult compiled, IRead
public string? ResultType { get; }
public IReadOnlyList AutoImported { get; }
+ /// Hosts that cannot unload: whether the snippet declares static state (then it is never reused).
+ public bool? HasStaticState { get; set; }
+
/// Set only for hosts that cannot unload (Mono): the assembly is loaded once and reused.
public ILoadedAssembly? Loaded { get; set; }
}
@@ -541,7 +589,7 @@ sealed class Execution
readonly CodeRunner _runner;
readonly PreparedSnippet _prepared;
readonly Stopwatch _stopwatch = Stopwatch.StartNew();
- readonly List _logs = new List();
+ readonly LogBudget _logs;
readonly List _captures = new List();
ErrorKind _errorKind;
string? _error;
@@ -553,6 +601,7 @@ public Execution(CodeRunner runner, PreparedSnippet prepared)
{
_runner = runner;
_prepared = prepared;
+ _logs = new LogBudget(prepared.Request.MaxLogChars, prepared.Request.MaxLogLineChars);
}
public ILoadedAssembly? Loaded { get; set; }
@@ -586,7 +635,7 @@ public void Fail(ErrorKind kind, string error)
public void BeginCapture(IList captures)
{
foreach (var capture in captures)
- _captures.Add(capture.Begin(entry => { lock (_logs) _logs.Add(entry); }));
+ _captures.Add(capture.Begin(_logs.Add));
}
public ExecuteResult Finish()
@@ -612,8 +661,17 @@ public ExecuteResult Finish()
Pending = null;
}
- List logs;
- lock (_logs) logs = new List(_logs);
+ // Output limits (spec §6): a text value, the logs and the error, in the order `truncated` lists them.
+ var request = _prepared.Request;
+ var truncated = new List();
+ if (value is string text && !ReferenceEquals(value = OutputLimits.Clip(text, request.MaxValueChars, OutputLimits.ValueHint), text))
+ truncated.Add("value");
+ var logs = _logs.Entries(out var logsTruncated);
+ if (logsTruncated)
+ truncated.Add("logs");
+ var error = _error == null ? null : OutputLimits.Clip(_error, request.MaxErrorChars, OutputLimits.ErrorHint);
+ if (!ReferenceEquals(error, _error))
+ truncated.Add("error");
return new ExecuteResult
{
Success = _errorKind == ErrorKind.None,
@@ -621,7 +679,8 @@ public ExecuteResult Finish()
ReturnType = ReturnType,
Logs = logs,
ErrorKind = _errorKind,
- Error = _error,
+ Error = error,
+ Truncated = truncated,
AutoImported = _prepared.Entry!.AutoImported,
CompileMs = _prepared.CompileMs,
ExecuteMs = _stopwatch.ElapsedMilliseconds,
diff --git a/dotnet/CodeExec/Contracts.cs b/dotnet/CodeExec/Contracts.cs
index b4fd108..6410ae8 100644
--- a/dotnet/CodeExec/Contracts.cs
+++ b/dotnet/CodeExec/Contracts.cs
@@ -54,6 +54,18 @@ public sealed class ExecuteRequest
/// Retry with a namespace when an unknown simple name resolves to exactly one (spec §3.3).
public bool AutoImport { get; set; } = true;
+
+ /// Output limit for a text , in characters (spec §6); 0 = no limit.
+ public int MaxValueChars { get; set; } = OutputLimits.DefaultValueChars;
+
+ /// Output limit for all together (spec §6); 0 = no limit.
+ public int MaxLogChars { get; set; } = OutputLimits.DefaultLogChars;
+
+ /// Output limit for one log message (spec §6); 0 = no limit.
+ public int MaxLogLineChars { get; set; } = OutputLimits.DefaultLogLineChars;
+
+ /// Output limit for (spec §6); 0 = no limit.
+ public int MaxErrorChars { get; set; } = OutputLimits.DefaultErrorChars;
}
public sealed class LogEntry
@@ -74,7 +86,10 @@ public sealed class CodeDiagnostic
{
public string Id { get; set; } = "";
public string Severity { get; set; } = "error";
- /// snippet.cs, an include's relative path, or empty for code the agent did not write.
+ ///
+ /// snippet.cs, an include's relative path, usings for a namespace the runtime adds (host defaults or
+ /// the request's), or empty for other code the agent did not write.
+ ///
public string File { get; set; } = "";
public int Line { get; set; }
public int Column { get; set; }
@@ -106,6 +121,9 @@ public sealed class ExecuteResult
public long ExecuteMs { get; set; }
public bool CacheHit { get; set; }
+ /// The fields an output limit shortened (spec §6), in this order: value, logs, error.
+ public IReadOnlyList Truncated { get; set; } = Array.Empty();
+
internal static ExecuteResult Fail(ErrorKind kind, string error) => new ExecuteResult { ErrorKind = kind, Error = error };
}
}
diff --git a/dotnet/CodeExec/Limits.cs b/dotnet/CodeExec/Limits.cs
new file mode 100644
index 0000000..d270b9c
--- /dev/null
+++ b/dotnet/CodeExec/Limits.cs
@@ -0,0 +1,131 @@
+#nullable enable
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace com.IvanMurzak.CodeExec
+{
+ /// Output limits (spec/contract.md §6): a response goes straight into an agent's context.
+ public static class OutputLimits
+ {
+ public const int DefaultValueChars = 20000;
+ public const int DefaultLogChars = 20000;
+ public const int DefaultLogLineChars = 2000;
+ public const int DefaultErrorChars = 20000;
+
+ /// What one log entry costs on top of its message: its level and line break, as hosts print them.
+ public const int LogEntryChars = 12;
+
+ /// A traceback keeps the first and the last this many of the snippet's frames.
+ public const int FramesKept = 10;
+
+ /// Inner (chained) exceptions reported after the error itself.
+ public const int MaxCauses = 5;
+
+ /// Compile diagnostics returned at most; one more entry says how many were left out.
+ public const int MaxDiagnostics = 20;
+
+ internal const string ValueHint = "return a slice or a summary";
+ internal const string ErrorHint = "raise a shorter message";
+
+ ///
+ /// The first characters plus ... [truncated: N of M characters shown; hint];
+ /// never splits a surrogate pair. 0 = no limit.
+ ///
+ public static string Clip(string text, int limit, string hint)
+ {
+ if (limit <= 0 || text.Length <= limit)
+ return text;
+ var keep = char.IsHighSurrogate(text[limit - 1]) ? limit - 1 : limit;
+ return text.Substring(0, keep) + "... [truncated: " + keep + " of " + text.Length + " characters shown; " + hint + "]";
+ }
+
+ /// The first diagnostics, plus a CE0002 entry counting the rest.
+ internal static IReadOnlyList Diagnostics(IEnumerable diagnostics)
+ {
+ var all = diagnostics.ToList();
+ if (all.Count <= MaxDiagnostics)
+ return all;
+ var kept = all.Take(MaxDiagnostics).ToList();
+ kept.Add(new CodeDiagnostic
+ {
+ Id = "CE0002",
+ Message = $"{all.Count - MaxDiagnostics} more diagnostic(s) left out: fix these first.",
+ });
+ return kept;
+ }
+ }
+
+ ///
+ /// The log budget of spec §6, applied as entries arrive so a runaway print loop cannot grow memory: the first entries up
+ /// to 3/4 of the budget, then the most recent ones that still fit, with one entry saying how many were left out.
+ /// Thread-safe (host log hooks may fire on other threads).
+ ///
+ sealed class LogBudget
+ {
+ readonly int _max, _maxLine;
+ readonly List _head = new List();
+ readonly Queue _tail = new Queue();
+ bool _headOpen = true;
+ long _headUsed, _tailUsed;
+ int _leftOut;
+ bool _clipped;
+
+ public LogBudget(int maxChars, int maxLineChars)
+ {
+ _max = maxChars;
+ _maxLine = maxLineChars;
+ }
+
+ public void Add(LogEntry entry)
+ {
+ if (entry.Message == null) // a host hook's empty message
+ entry = new LogEntry(entry.Level, "");
+ var clipped = OutputLimits.Clip(entry.Message, _maxLine, "print less");
+ lock (_head)
+ {
+ if (!ReferenceEquals(clipped, entry.Message))
+ {
+ _clipped = true;
+ entry = new LogEntry(entry.Level, clipped);
+ }
+ if (_max <= 0)
+ {
+ _head.Add(entry);
+ return;
+ }
+ var size = Cost(entry);
+ if (_headOpen && _headUsed + size <= _max * 3L / 4)
+ {
+ _head.Add(entry);
+ _headUsed += size;
+ return;
+ }
+ _headOpen = false;
+ _tail.Enqueue(entry);
+ _tailUsed += size;
+ while (_tail.Count > 0 && _headUsed + _tailUsed > _max)
+ {
+ _tailUsed -= Cost(_tail.Dequeue());
+ _leftOut++;
+ }
+ }
+ }
+
+ /// The kept entries; when a line was clipped or entries were left out.
+ public List Entries(out bool truncated)
+ {
+ lock (_head)
+ {
+ truncated = _clipped || _leftOut > 0;
+ var entries = new List(_head);
+ if (_leftOut > 0)
+ entries.Add(new LogEntry("warning", $"... {_leftOut} log line(s) left out (output over {_max} characters): print less, or return a summary"));
+ entries.AddRange(_tail);
+ return entries;
+ }
+ }
+
+ static long Cost(LogEntry entry) => entry.Message.Length + OutputLimits.LogEntryChars;
+ }
+}
diff --git a/dotnet/CodeExec/Support.cs b/dotnet/CodeExec/Support.cs
index 4b886ea..c971199 100644
--- a/dotnet/CodeExec/Support.cs
+++ b/dotnet/CodeExec/Support.cs
@@ -63,38 +63,85 @@ static bool IsInternalNamespace(string ns) =>
ns.StartsWith("Internal", StringComparison.Ordinal) && (ns.Length == 8 || ns[8] == '.');
}
+ ///
+ /// Includes (spec §1): inside the include root unless the host allows otherwise; files must be .cs; folders
+ /// contribute their *.cs recursively. Paths are reported relative to the root.
+ ///
static class IncludeResolver
{
- public static List Read(IReadOnlyList include, string root)
+ public static List Read(IReadOnlyList include, string root, bool allowOutsideRoot)
{
var files = new List();
- var seen = new HashSet(StringComparer.OrdinalIgnoreCase);
+ if (include.Count == 0)
+ return files; // the root is not even needed (a host may leave it empty)
+ var seen = new HashSet(PathComparer);
+ var prefix = WithSeparator(Path.GetFullPath(root));
foreach (var entry in include)
{
if (string.IsNullOrWhiteSpace(entry))
throw new ArgumentException("include entry is empty");
var full = Path.GetFullPath(Path.IsPathRooted(entry) ? entry : Path.Combine(root, entry));
+ if (!allowOutsideRoot && !IsUnder(prefix, WithSeparator(full))) // the root itself counts
+ throw new ArgumentException("include is outside the include root: " + entry);
IEnumerable paths;
if (Directory.Exists(full))
paths = Directory.GetFiles(full, "*.cs", SearchOption.AllDirectories).OrderBy(p => p, StringComparer.Ordinal);
else if (File.Exists(full))
+ {
+ if (!full.EndsWith(".cs", StringComparison.OrdinalIgnoreCase))
+ throw new ArgumentException("include is not a .cs file: " + entry);
paths = new[] { full };
+ }
else
throw new FileNotFoundException("include not found: " + entry);
foreach (var path in paths)
{
if (seen.Add(path))
- files.Add(new SourceFile(Relative(root, path), File.ReadAllText(path)));
+ files.Add(new SourceFile(Relative(prefix, path), File.ReadAllText(path)));
}
}
return files;
}
- static string Relative(string root, string path)
+ // Windows paths are case-insensitive; elsewhere a differently cased path may be another folder.
+ static readonly StringComparison PathComparison = Path.DirectorySeparatorChar == '\\' ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
+ static readonly StringComparer PathComparer = Path.DirectorySeparatorChar == '\\' ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal;
+
+ static string WithSeparator(string path) =>
+ path.EndsWith(Path.DirectorySeparatorChar.ToString(), StringComparison.Ordinal) || path.EndsWith(Path.AltDirectorySeparatorChar.ToString(), StringComparison.Ordinal)
+ ? path
+ : path + Path.DirectorySeparatorChar;
+
+ static bool IsUnder(string prefix, string path) => path.StartsWith(prefix, PathComparison);
+
+ static string Relative(string prefix, string path) => (IsUnder(prefix, path) ? path.Substring(prefix.Length) : path).Replace('\\', '/');
+ }
+
+ ///
+ /// Whether a snippet assembly keeps state between runs if it is reused: a static field (an auto-property's or an
+ /// event's backing field included) declared by one of the agent's own types. The compiler's own types (lambda caches,
+ /// array-initializer data) are stateless for this purpose.
+ ///
+ static class StaticState
+ {
+ public static bool Has(Assembly assembly)
{
- var prefix = Path.GetFullPath(root).TrimEnd('\\', '/') + Path.DirectorySeparatorChar;
- var relative = path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) ? path.Substring(prefix.Length) : path;
- return relative.Replace('\\', '/');
+ const BindingFlags flags = BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly;
+ try
+ {
+ foreach (var type in assembly.GetTypes())
+ {
+ if (type.IsDefined(typeof(CompilerGeneratedAttribute), false))
+ continue;
+ if (type.GetFields(flags).Any(f => !f.IsLiteral))
+ return true;
+ }
+ return false;
+ }
+ catch (Exception) // a type or attribute that cannot load: cannot tell, so be safe and load fresh copies
+ {
+ return true;
+ }
}
}
diff --git a/dotnet/Directory.Build.props b/dotnet/Directory.Build.props
index e929b81..5833b19 100644
--- a/dotnet/Directory.Build.props
+++ b/dotnet/Directory.Build.props
@@ -6,7 +6,7 @@
true
$(NoWarn);CS1591
true
- 0.2.1
+ 0.3.0
Ivan Murzak
Copyright (c) 2026 Ivan Murzak
https://github.com/IvanMurzak/Code-Exec
diff --git a/fixtures/golden/csharp.json b/fixtures/golden/csharp.json
index a40347e..73a36c1 100644
--- a/fixtures/golden/csharp.json
+++ b/fixtures/golden/csharp.json
@@ -371,6 +371,171 @@
"success": false,
"errorKind": "EntryPointNotFound"
}
+ },
+ {
+ "name": "value-clipped",
+ "code": "new string('x', 30)",
+ "maxValueChars": 10,
+ "expect": {
+ "success": true,
+ "value": "xxxxxxxxxx... [truncated: 10 of 30 characters shown; return a slice or a summary]",
+ "truncated": [
+ "value"
+ ]
+ }
+ },
+ {
+ "name": "value-default-limit",
+ "code": "new string('x', 30000)",
+ "expect": {
+ "success": true,
+ "valueEndsWith": "xx... [truncated: 20000 of 30000 characters shown; return a slice or a summary]",
+ "truncated": [
+ "value"
+ ]
+ }
+ },
+ {
+ "name": "value-no-limit",
+ "code": "new string('x', 30000)",
+ "maxValueChars": 0,
+ "expect": {
+ "success": true,
+ "valueLength": 30000,
+ "truncated": []
+ }
+ },
+ {
+ "name": "logs-first-and-last",
+ "code": "for (var i = 0; i < 10; i++) Console.WriteLine($\"line {i}\");",
+ "maxLogChars": 60,
+ "expect": {
+ "success": true,
+ "logs": [
+ "line 0",
+ "line 1",
+ "... 7 log line(s) left out (output over 60 characters): print less, or return a summary",
+ "line 9"
+ ],
+ "truncated": [
+ "logs"
+ ]
+ }
+ },
+ {
+ "name": "log-line-clipped",
+ "code": "Console.WriteLine(new string('y', 50));",
+ "maxLogLineChars": 5,
+ "expect": {
+ "success": true,
+ "logs": [
+ "yyyyy... [truncated: 5 of 50 characters shown; print less]"
+ ],
+ "truncated": [
+ "logs"
+ ]
+ }
+ },
+ {
+ "name": "logs-within-budget",
+ "code": "Console.WriteLine(\"a\");\nConsole.WriteLine(\"b\");",
+ "maxLogChars": 26,
+ "expect": {
+ "success": true,
+ "logs": [
+ "a",
+ "b"
+ ],
+ "truncated": []
+ }
+ },
+ {
+ "name": "error-clipped",
+ "code": "throw new InvalidOperationException(new string('z', 100));",
+ "maxErrorChars": 20,
+ "expect": {
+ "success": false,
+ "errorKind": "Runtime",
+ "errorContains": "... [truncated: 20 of ",
+ "truncated": [
+ "error"
+ ]
+ }
+ },
+ {
+ "name": "limit-negative",
+ "code": "1",
+ "maxValueChars": -1,
+ "expect": {
+ "success": false,
+ "errorKind": "InvalidRequest"
+ }
+ },
+ {
+ "name": "deep-traceback",
+ "code": "int R(int n) => n == 0 ? throw new InvalidOperationException(\"deep\") : R(n - 1);\nR(50)",
+ "expect": {
+ "success": false,
+ "errorKind": "Runtime",
+ "errorContains": "32 frame(s) left out"
+ }
+ },
+ {
+ "name": "exception-message-unprintable",
+ "code": "class E : Exception { public override string Message => throw new InvalidOperationException(); }\nthrow new E();",
+ "expect": {
+ "success": false,
+ "errorKind": "Runtime",
+ "errorContains": ""
+ }
+ },
+ {
+ "name": "include-outside-root",
+ "code": "1",
+ "include": [
+ "../outside/helper"
+ ],
+ "expect": {
+ "success": false,
+ "errorKind": "InvalidRequest",
+ "errorContains": "outside the include root"
+ }
+ },
+ {
+ "name": "include-wrong-extension",
+ "code": "1",
+ "include": [
+ "notes.txt"
+ ],
+ "expect": {
+ "success": false,
+ "errorKind": "InvalidRequest",
+ "errorContains": "include is not a ."
+ }
+ },
+ {
+ "name": "inner-exception",
+ "code": "throw new Exception(\"outer\", new ArgumentException(\"inner cause\"));",
+ "expect": {
+ "success": false,
+ "errorKind": "Runtime",
+ "errorContains": " ---> System.ArgumentException: inner cause"
+ }
+ },
+ {
+ "name": "bad-using",
+ "code": "1",
+ "usings": [
+ "No.Such.Namespace"
+ ],
+ "expect": {
+ "success": false,
+ "errorKind": "Compile",
+ "diagnostic": {
+ "file": "usings",
+ "line": 0
+ }
+ }
}
]
}
diff --git a/fixtures/golden/python.json b/fixtures/golden/python.json
index 6a39f97..2fdded8 100644
--- a/fixtures/golden/python.json
+++ b/fixtures/golden/python.json
@@ -18,6 +18,22 @@
{ "name": "include-file", "code": "GREETING + '!'", "include": ["prelude.py"], "expect": { "success": true, "value": "hi!" } },
{ "name": "include-traceback", "code": "explode()", "include": ["broken.py"], "expect": { "success": false, "errorKind": "Runtime", "errorContains": "broken.py" } },
{ "name": "include-missing", "code": "1", "include": ["missing.py"], "expect": { "success": false, "errorKind": "InvalidRequest" } },
- { "name": "empty-code", "code": " ", "expect": { "success": false, "errorKind": "InvalidRequest" } }
+ { "name": "empty-code", "code": " ", "expect": { "success": false, "errorKind": "InvalidRequest" } },
+ {"name": "value-clipped", "code": "'x' * 30", "maxValueChars": 10, "expect": {"success": true, "value": "xxxxxxxxxx... [truncated: 10 of 30 characters shown; return a slice or a summary]", "truncated": ["value"]}},
+ {"name": "value-default-limit", "code": "'x' * 30000", "expect": {"success": true, "valueEndsWith": "xx... [truncated: 20000 of 30000 characters shown; return a slice or a summary]", "truncated": ["value"]}},
+ {"name": "value-no-limit", "code": "'x' * 30000", "maxValueChars": 0, "expect": {"success": true, "valueLength": 30000, "truncated": []}},
+ {"name": "logs-first-and-last", "code": "for i in range(10):\n print('line %d' % i)", "maxLogChars": 60, "expect": {"success": true, "logs": ["line 0", "line 1", "... 7 log line(s) left out (output over 60 characters): print less, or return a summary", "line 9"], "truncated": ["logs"]}},
+ {"name": "log-line-clipped", "code": "print('y' * 50)", "maxLogLineChars": 5, "expect": {"success": true, "logs": ["yyyyy... [truncated: 5 of 50 characters shown; print less]"], "truncated": ["logs"]}},
+ {"name": "logs-within-budget", "code": "print('a')\nprint('b')", "maxLogChars": 26, "expect": {"success": true, "logs": ["a", "b"], "truncated": []}},
+ {"name": "error-clipped", "code": "raise ValueError('z' * 100)", "maxErrorChars": 20, "expect": {"success": false, "errorKind": "Runtime", "errorContains": "... [truncated: 20 of ", "truncated": ["error"]}},
+ {"name": "limit-negative", "code": "1", "maxValueChars": -1, "expect": {"success": false, "errorKind": "InvalidRequest"}},
+ {"name": "deep-traceback", "code": "def r(n):\n if n == 0:\n raise ValueError('deep')\n return r(n - 1)\nr(50)", "expect": {"success": false, "errorKind": "Runtime", "errorContains": "32 frame(s) left out"}},
+ {"name": "exception-message-unprintable", "code": "class E(Exception):\n def __str__(self):\n raise RuntimeError('no')\nraise E()", "expect": {"success": false, "errorKind": "Runtime", "errorContains": ""}},
+ {"name": "include-outside-root", "code": "1", "include": ["../outside/helper"], "expect": {"success": false, "errorKind": "InvalidRequest", "errorContains": "outside the include root"}},
+ {"name": "include-wrong-extension", "code": "1", "include": ["notes.txt"], "expect": {"success": false, "errorKind": "InvalidRequest", "errorContains": "include is not a ."}},
+ {"name": "keyboard-interrupt", "code": "def stop():\n raise KeyboardInterrupt('stop')\nstop()", "expect": {"success": false, "errorKind": "Runtime", "errorContains": "KeyboardInterrupt: stop"}},
+ {"name": "base-exception-subclass", "code": "class Halt(BaseException):\n pass\nraise Halt('x')", "expect": {"success": false, "errorKind": "Runtime", "errorContains": "Halt: x"}},
+ {"name": "generator-exit", "code": "raise GeneratorExit()", "expect": {"success": false, "errorKind": "Runtime", "errorContains": "GeneratorExit"}},
+ {"name": "chained-cause", "code": "try:\n {}['k']\nexcept KeyError as e:\n raise ValueError('outer') from e", "expect": {"success": false, "errorKind": "Runtime", "errorContains": "Caused by: KeyError: 'k'"}}
]
}
diff --git a/fixtures/include/csharp/notes.txt b/fixtures/include/csharp/notes.txt
new file mode 100644
index 0000000..4422441
--- /dev/null
+++ b/fixtures/include/csharp/notes.txt
@@ -0,0 +1 @@
+Not source code: an include must be a .cs file.
diff --git a/fixtures/include/python/notes.txt b/fixtures/include/python/notes.txt
new file mode 100644
index 0000000..65c7f3e
--- /dev/null
+++ b/fixtures/include/python/notes.txt
@@ -0,0 +1 @@
+Not source code: an include must be a .py file.
diff --git a/python/code_exec/__init__.py b/python/code_exec/__init__.py
index 6cf2904..03a0da3 100644
--- a/python/code_exec/__init__.py
+++ b/python/code_exec/__init__.py
@@ -11,4 +11,4 @@
from .runner import run, run_json, run_b64, set_value_converter, SNIPPET_FILE
__all__ = ["run", "run_json", "run_b64", "set_value_converter", "SNIPPET_FILE"]
-__version__ = "0.2.1"
+__version__ = "0.3.0"
diff --git a/python/code_exec/runner.py b/python/code_exec/runner.py
index 046a85a..a977b69 100644
--- a/python/code_exec/runner.py
+++ b/python/code_exec/runner.py
@@ -1,18 +1,38 @@
-"""Snippet runner: spec/contract.md section 4. Standard library only, Python 3.10+."""
+"""Snippet runner: spec/contract.md sections 4 and 6. Standard library only, Python 3.10+."""
import ast
import base64
import builtins
+import collections
import io
import json
+import linecache
import os
import sys
import time
+import tokenize
import traceback
from contextlib import redirect_stderr, redirect_stdout
SNIPPET_FILE = "snippet.py"
+# Output limits (spec section 6), in characters; 0 = no limit.
+MAX_VALUE_CHARS = 20000
+MAX_LOG_CHARS = 20000
+MAX_LOG_LINE_CHARS = 2000
+MAX_ERROR_CHARS = 20000
+_LOG_ENTRY_CHARS = 12 # what one log entry costs on top of its message ("[warning] " and a line break, as hosts print it)
+_FRAMES_KEPT = 10 # a traceback keeps the first and the last _FRAMES_KEPT of the agent's frames
+_MAX_CAUSES = 5 # chained exceptions (__cause__ / __context__) reported after the error itself
+
+# JSON request field -> run() keyword argument.
+_LIMIT_FIELDS = {
+ "maxValueChars": "max_value_chars",
+ "maxLogChars": "max_log_chars",
+ "maxLogLineChars": "max_log_line_chars",
+ "maxErrorChars": "max_error_chars",
+}
+
_value_converter = None
@@ -22,21 +42,48 @@ def set_value_converter(converter):
_value_converter = converter
-def run(code, include=(), include_root=None, imports=()):
- """Execute ``code`` and return the contract's response as a dict (never raises for snippet errors)."""
- started = time.perf_counter()
+def run(code, include=(), include_root=None, imports=(), max_value_chars=MAX_VALUE_CHARS, max_log_chars=MAX_LOG_CHARS,
+ max_log_line_chars=MAX_LOG_LINE_CHARS, max_error_chars=MAX_ERROR_CHARS, include_outside_root=False):
+ """Execute ``code`` and return the contract's response as a dict (never raises for snippet errors).
+
+ ``include_outside_root``: the host accepts includes outside ``include_root`` (absolute paths, ``..``).
+ """
result = _empty()
+ limits = (max_value_chars, max_log_chars, max_log_line_chars, max_error_chars)
+ if any(isinstance(limit, bool) or not isinstance(limit, int) or limit < 0 for limit in limits):
+ return _fail(result, "InvalidRequest", "output limits must be integers >= 0 (0 = no limit)")
+ logs = _Logs(max_log_chars, max_log_line_chars)
+ try:
+ _execute(result, logs, code, include, include_root, imports, include_outside_root)
+ except BaseException as error: # the runner itself failed (a snippet wrecked the interpreter state): still answer
+ result["success"], result["value"] = False, None
+ _fail(result, "Runtime", "%s: %s (while running the snippet)" % (type(error).__name__, _message(error)))
+
+ result["logs"] = logs.entries()
+ value, error = result["value"], result["error"]
+ if isinstance(value, str):
+ result["value"] = _clip(value, max_value_chars, "return a slice or a summary")
+ if isinstance(error, str):
+ result["error"] = _clip(error, max_error_chars, "raise a shorter message")
+ changed = (("value", result["value"] is not value), ("logs", logs.left_out or logs.clipped), ("error", result["error"] is not error))
+ result["truncated"] = [field for field, truncated in changed if truncated]
+ return result
+
+def _execute(result, logs, code, include, include_root, imports, include_outside_root):
+ started = time.perf_counter()
if not isinstance(code, str) or not code.strip():
return _fail(result, "InvalidRequest", "code is empty")
- if isinstance(imports, str):
+ if not isinstance(imports, (list, tuple)) or not all(isinstance(name, str) and name for name in imports):
return _fail(result, "InvalidRequest", "imports must be a list of module names")
+ if include_root is not None and not isinstance(include_root, str):
+ return _fail(result, "InvalidRequest", "includeRoot must be a path")
root = os.path.abspath(include_root or os.getcwd())
try:
- folders, files = _resolve_include(include, root)
+ folders, files = _resolve_include(include, root, include_outside_root)
sources = [(path, _read(full)) for path, full in files]
- except (OSError, ValueError) as error: # UnicodeDecodeError (an include that is not UTF-8) is a ValueError
+ except (OSError, ValueError) as error:
return _fail(result, "InvalidRequest", str(error))
try:
@@ -58,17 +105,16 @@ def run(code, include=(), include_root=None, imports=()):
})
result["compileMs"] = _ms(started)
return _fail(result, "Compile", "Compilation failed")
- except ValueError as error: # null bytes in the source raise ValueError, not SyntaxError, on Python 3.10
+ except (ValueError, RecursionError, MemoryError) as error: # null bytes (3.10), a too deeply nested expression
result["compileMs"] = _ms(started)
- return _fail(result, "Compile", str(error))
+ return _fail(result, "Compile", "%s: %s" % (type(error).__name__, error))
result["compileMs"] = _ms(started)
executing = time.perf_counter()
scope = {"__name__": "__code_exec__", "__builtins__": builtins}
- out, err = _LogStream(result["logs"], "info"), _LogStream(result["logs"], "error")
+ out, err = _LogStream(logs, "info"), _LogStream(logs, "error")
prefixes = [_prefix(folder) for folder in folders]
- saved_path = list(sys.path)
- saved_bytecode = sys.dont_write_bytecode
+ saved = _HostState(code, sources)
try:
sys.path[:0] = folders
if folders:
@@ -77,7 +123,7 @@ def run(code, include=(), include_root=None, imports=()):
sys.dont_write_bytecode = True
for folder in folders:
sys.path_importer_cache.pop(folder, None)
- for name in imports or ():
+ for name in imports:
scope.setdefault(name.split(".")[0], __import__(name)) # like `import a.b`: binds the top package
with redirect_stdout(out), redirect_stderr(err):
for compiled in file_code:
@@ -86,40 +132,55 @@ def run(code, include=(), include_root=None, imports=()):
value = eval(tail, scope) if tail is not None else None
result["value"] = _convert(value)
result["success"] = True
- except (Exception, SystemExit) as error: # a snippet must never take the host down
+ except BaseException as error: # SystemExit, KeyboardInterrupt, ...: a snippet must never take the host down
result["errorKind"] = "Runtime"
- result["error"] = _format_error(error, root, prefixes, {path for path, _ in files})
+ try:
+ result["error"] = _format_error(error, root, prefixes, {path for path, _ in files})
+ except BaseException: # the traceback itself cannot be read
+ result["error"] = "%s: %s" % (type(error).__name__, _message(error))
finally:
out.flush()
err.flush()
- sys.path[:] = saved_path
- sys.dont_write_bytecode = saved_bytecode
- _forget_modules(prefixes)
+ saved.restore(prefixes)
result["executeMs"] = _ms(executing)
return result
+# JSON request field -> the type it must have when present (spec section 1: a wrong type is InvalidRequest).
+_FIELD_TYPES = {"include": list, "imports": list, "includeRoot": str, "includeOutsideRoot": bool}
+
+
def run_json(payload):
"""JSON request (spec section 1 plus Python-only ``imports`` / ``includeRoot``) -> JSON response."""
try:
request = json.loads(payload)
- except ValueError as error:
- return _invalid("request is not JSON: %s" % error)
+ except Exception as error: # ValueError, RecursionError (deep nesting), TypeError (not a string)
+ return _invalid("request is not JSON: %s" % _message(error))
if not isinstance(request, dict):
return _invalid("request must be a JSON object")
- return json.dumps(run(
+ for field, kind in _FIELD_TYPES.items():
+ if request.get(field) is not None and not isinstance(request[field], kind):
+ return _invalid("%s must be a %s" % (field, kind.__name__))
+ limits = {name: request[field] for field, name in _LIMIT_FIELDS.items() if request.get(field) is not None}
+ response = run(
request.get("code", ""),
include=request.get("include") or (),
include_root=request.get("includeRoot"),
imports=request.get("imports") or (),
- ))
+ include_outside_root=request.get("includeOutsideRoot") is True,
+ **limits)
+ try:
+ return json.dumps(response, default=str) # a host converter's odd value must not fail the response
+ except Exception: # non-string keys, a cycle, a str() that raises
+ response["value"] = _message(response["value"])
+ return json.dumps(response, default=str)
def run_b64(payload_b64):
"""Same as :func:`run_json`, with the request base64-encoded: hosts pass it through one string literal."""
try:
payload = base64.b64decode(payload_b64).decode("utf-8")
- except (ValueError, UnicodeDecodeError) as error:
+ except Exception as error: # ValueError, UnicodeDecodeError, TypeError (not a string)
return _invalid("request is not base64 UTF-8: %s" % error)
return run_json(payload)
@@ -127,7 +188,7 @@ def run_b64(payload_b64):
def _empty():
return {
"success": False, "value": None, "logs": [], "errorKind": "None", "error": None, "diagnostics": [],
- "autoImported": [], "compileMs": 0, "executeMs": 0, "cacheHit": False,
+ "autoImported": [], "compileMs": 0, "executeMs": 0, "cacheHit": False, "truncated": [],
}
@@ -146,29 +207,45 @@ def _ms(since):
return int((time.perf_counter() - since) * 1000)
+def _clip(text, limit, hint):
+ """Spec section 6: the first ``limit`` characters plus a note saying how much was shown (``text`` itself if it fits)."""
+ if not limit or len(text) <= limit:
+ return text
+ return "%s... [truncated: %d of %d characters shown; %s]" % (text[:limit], limit, len(text), hint)
+
+
def _read(path):
+ """Reads an include as Python reads a source file: UTF-8, or the encoding its BOM / coding declaration names."""
try:
- with io.open(path, "r", encoding="utf-8") as handle:
+ with tokenize.open(path) as handle:
return handle.read()
- except UnicodeDecodeError as error:
- raise ValueError("include is not UTF-8: %s (%s)" % (path, error)) from None
+ except (SyntaxError, UnicodeDecodeError, LookupError) as error: # bad / unknown / non-text coding declaration
+ raise ValueError("include is not readable as Python source: %s (%s)" % (path, error)) from None
-def _resolve_include(include, root):
- """Folders go on sys.path (import by name); files run in the snippet's namespace first, in order."""
- if isinstance(include, str):
+def _resolve_include(include, root, outside_root):
+ """Folders go on sys.path (import by name); ``.py`` files run in the snippet's namespace first, in order. Both must
+ lie inside ``root`` unless the host allows ``outside_root``."""
+ root_prefix = _prefix(root) # normcased, ends with a separator: _prefix(path) starts with it for the root and below
+ if not isinstance(include, (list, tuple)):
raise ValueError("include must be a list of paths")
folders, files, seen = [], [], set()
- for entry in include or ():
- if not isinstance(entry, str) or not entry.strip():
+ for entry in include:
+ if not isinstance(entry, str):
+ raise ValueError("include entry is not a path: %r" % (entry,))
+ if not entry.strip():
raise ValueError("include entry is empty")
full = os.path.abspath(entry if os.path.isabs(entry) else os.path.join(root, entry))
- if full in seen:
+ if not outside_root and not _prefix(full).startswith(root_prefix):
+ raise ValueError("include is outside the include root: %s" % entry)
+ if os.path.normcase(full) in seen:
continue
- seen.add(full)
+ seen.add(os.path.normcase(full))
if os.path.isdir(full):
folders.append(full)
elif os.path.isfile(full):
+ if not full.lower().endswith(".py"):
+ raise ValueError("include is not a .py file: %s" % entry)
files.append((_relative(full, root), full))
else:
raise FileNotFoundError("include not found: %s" % entry)
@@ -189,14 +266,64 @@ def _relative(path, root):
return relative.replace("\\", "/")
-def _forget_modules(prefixes):
- """Drop modules imported from include folders, so an edited helper is re-imported on the next run."""
- if not prefixes:
- return
- for name, module in list(sys.modules.items()):
- location = getattr(module, "__file__", None)
- if location and _under(location, prefixes):
- del sys.modules[name]
+class _HostState:
+ """What a run may change in the interpreter and restores afterwards, each step on its own: a snippet that broke one
+ of them (``del sys.path``) must not keep the others from being restored, nor make the runner raise."""
+
+ def __init__(self, code, sources):
+ self.path = list(sys.path)
+ self.bytecode = sys.dont_write_bytecode
+ self.cwd = os.getcwd()
+ self.recursion_limit = sys.getrecursionlimit()
+ self.modules = set(sys.modules)
+ # The agent's own sources, so tracebacks show their lines (and never a same-named file elsewhere on sys.path).
+ lines = dict(sources)
+ lines[SNIPPET_FILE] = code
+ self.saved_lines = {path: linecache.cache.get(path) for path in lines}
+ for path, source in lines.items():
+ linecache.cache[path] = (len(source), None, source.splitlines(True), path)
+
+ def restore(self, prefixes):
+ steps = (
+ lambda: sys.setrecursionlimit(self.recursion_limit), # first: the other steps need the stack
+ self._restore_path,
+ lambda: sys.__setattr__("dont_write_bytecode", self.bytecode),
+ lambda: os.chdir(self.cwd),
+ lambda: self._forget_modules(prefixes),
+ self._forget_lines,
+ )
+ for step in steps:
+ try:
+ step()
+ except BaseException: # nothing left to restore it from (the folder is gone, ...): keep restoring the rest
+ pass
+
+ def _restore_path(self):
+ if isinstance(getattr(sys, "path", None), list):
+ sys.path[:] = self.path # same list object: hosts may hold a reference to it
+ else:
+ sys.path = self.path
+
+ def _forget_modules(self, prefixes):
+ """Drop the modules this run imported from include folders, so an edited helper is re-imported next time. Only
+ those: a host module that happens to live under an include folder stays."""
+ if not prefixes:
+ return
+ for name in set(sys.modules) - self.modules:
+ try:
+ location = sys.modules[name].__file__
+ except BaseException:
+ continue
+ if isinstance(location, str) and _under(location, prefixes):
+ del sys.modules[name]
+
+ def _forget_lines(self):
+ for path, previous in self.saved_lines.items():
+ if previous is None:
+ linecache.cache.pop(path, None)
+ else:
+ linecache.cache[path] = previous
+
def _convert(value):
@@ -215,38 +342,136 @@ def _convert(value):
def _format_error(error, root, prefixes, include_files):
- """``Type: message`` plus only the frames the agent wrote (snippet.py, include files, include folders)."""
+ """``Type: message`` plus only the frames the agent wrote (snippet.py, include files, include folders), then the
+ exceptions it was raised from (``__cause__``) or while handling (``__context__``), at most _MAX_CAUSES of them."""
+ lines = _format_one(error, root, prefixes, include_files)
+ seen = {id(error)}
+ while True:
+ cause, label = _cause(error)
+ if cause is None or id(cause) in seen:
+ break
+ if len(seen) > _MAX_CAUSES:
+ lines.append("... more chained exception(s) left out")
+ break
+ seen.add(id(cause))
+ first, *rest = _format_one(cause, root, prefixes, include_files)
+ lines.append("%s: %s" % (label, first))
+ lines.extend(rest)
+ error = cause
+ return "\n".join(lines)
+
+
+def _cause(error):
+ if error.__cause__ is not None:
+ return error.__cause__, "Caused by"
+ if error.__context__ is not None and not error.__suppress_context__:
+ return error.__context__, "Raised while handling"
+ return None, None
+
+
+def _format_one(error, root, prefixes, include_files):
frames = []
- for frame in traceback.extract_tb(error.__traceback__):
+ # An explicit limit: a snippet's sys.tracebacklimit must not hide its own frames.
+ walked = traceback.walk_tb(error.__traceback__)
+ for frame in traceback.StackSummary.extract(walked, limit=sys.maxsize, lookup_lines=False):
if frame.filename == SNIPPET_FILE or frame.filename in include_files:
frames.append(frame)
elif _under(frame.filename, prefixes):
frame.filename = _relative(os.path.abspath(frame.filename), root)
frames.append(frame)
- lines = ["%s: %s" % (type(error).__name__, error)]
- lines.extend(line.rstrip("\n") for line in traceback.format_list(frames))
- return "\n".join(lines)
+ lines = ["%s: %s" % (type(error).__name__, _message(error))]
+ if len(frames) > 2 * _FRAMES_KEPT:
+ left_out = len(frames) - 2 * _FRAMES_KEPT
+ lines.extend(_format_frames(frames[:_FRAMES_KEPT]))
+ lines.append(" ... %d frame(s) left out" % left_out)
+ lines.extend(_format_frames(frames[-_FRAMES_KEPT:]))
+ else:
+ lines.extend(_format_frames(frames))
+ return lines
+
+
+def _format_frames(frames):
+ # One frame at a time: format_list would fold repeated frames, and the kept frames are exactly the first and last 10.
+ return [traceback.format_list([frame])[0].rstrip("\n") for frame in frames]
+
+
+def _message(value):
+ """``str(value)``, or a placeholder when its own ``__str__`` raises."""
+ try:
+ return str(value)
+ except BaseException:
+ return ""
+
+
+class _Logs:
+ """The log budget of spec section 6, applied as lines arrive, so a runaway print loop cannot grow memory: the
+ first lines up to 3/4 of the budget, then the most recent ones that still fit."""
+
+ def __init__(self, max_chars, max_line_chars):
+ self._max = max_chars
+ self._max_line = max_line_chars
+ self._head, self._tail = [], collections.deque()
+ self._head_open = True
+ self._head_used = self._tail_used = 0
+ self.left_out = 0
+ self.clipped = False
+
+ def add(self, level, message):
+ clipped = _clip(message, self._max_line, "print less")
+ self.clipped = self.clipped or clipped is not message
+ entry = {"level": level, "message": clipped}
+ if not self._max:
+ self._head.append(entry)
+ return
+ size = _cost(entry)
+ if self._head_open and self._head_used + size <= self._max * 3 // 4:
+ self._head.append(entry)
+ self._head_used += size
+ return
+ self._head_open = False
+ self._tail.append(entry)
+ self._tail_used += size
+ while self._tail and self._head_used + self._tail_used > self._max:
+ self._tail_used -= _cost(self._tail.popleft())
+ self.left_out += 1
+
+ def entries(self):
+ if not self.left_out:
+ return self._head + list(self._tail)
+ marker = {"level": "warning", "message": "... %d log line(s) left out (output over %d characters): print less, "
+ "or return a summary" % (self.left_out, self._max)}
+ return self._head + [marker] + list(self._tail)
+
+
+def _cost(entry):
+ return len(entry["message"]) + _LOG_ENTRY_CHARS
class _LogStream(io.TextIOBase):
"""Collects written text as one log entry per line."""
- def __init__(self, sink, level):
+ def __init__(self, logs, level):
super().__init__()
- self._sink = sink
+ self._logs = logs
self._level = level
- self._buffer = ""
+ self._parts = []
def writable(self):
return True
def write(self, text):
- *lines, self._buffer = (self._buffer + text).split("\n")
- for line in lines:
- self._sink.append({"level": self._level, "message": line.rstrip("\r")})
+ if not isinstance(text, str):
+ raise TypeError("write() argument must be str, not %s" % type(text).__name__)
+ *ended, rest = text.split("\n")
+ for piece in ended:
+ self._parts.append(piece)
+ self._logs.add(self._level, "".join(self._parts).rstrip("\r"))
+ self._parts = []
+ if rest:
+ self._parts.append(rest)
return len(text)
def flush(self):
- if self._buffer:
- self._sink.append({"level": self._level, "message": self._buffer})
- self._buffer = ""
+ if self._parts:
+ self._logs.add(self._level, "".join(self._parts))
+ self._parts = []
diff --git a/python/tests/test_code_exec.py b/python/tests/test_code_exec.py
index 8fef355..cbe7f4f 100644
--- a/python/tests/test_code_exec.py
+++ b/python/tests/test_code_exec.py
@@ -23,7 +23,10 @@ class GoldenTests(unittest.TestCase):
def test_golden(self):
for case in GOLDEN["cases"]:
with self.subTest(case["name"]):
- result = code_exec.run(case["code"], include=case.get("include", ()), include_root=INCLUDE_ROOT)
+ # Through the JSON entry point, as hosts call it: every request field of the case is passed as is.
+ request = {key: value for key, value in case.items() if key not in ("name", "expect")}
+ request["includeRoot"] = INCLUDE_ROOT
+ result = json.loads(code_exec.run_json(json.dumps(request)))
expect = case["expect"]
describe = "%s -> %s" % (case["name"], json.dumps(result))
self.assertEqual(expect["success"], result["success"], describe)
@@ -38,6 +41,14 @@ def test_golden(self):
self.assertTrue(any(d["file"] == wanted["file"] and d["line"] == wanted["line"] for d in result["diagnostics"]), describe)
for text in expect.get("logsContain", ()):
self.assertTrue(any(text in entry["message"] for entry in result["logs"]), describe)
+ if "logs" in expect:
+ self.assertEqual(expect["logs"], [entry["message"] for entry in result["logs"]], describe)
+ if "truncated" in expect:
+ self.assertEqual(expect["truncated"], result["truncated"], describe)
+ if "valueEndsWith" in expect:
+ self.assertTrue((result["value"] or "").endswith(expect["valueEndsWith"]), describe[:500])
+ if "valueLength" in expect:
+ self.assertEqual(expect["valueLength"], len(result["value"] or ""), describe[:500])
class BehaviourTests(unittest.TestCase):
@@ -67,9 +78,10 @@ def test_non_utf8_include_is_an_invalid_request(self):
path = os.path.join(folder, "latin1.py")
with open(path, "wb") as handle:
handle.write(b"\xff\xfe = 1\n")
- result = code_exec.run("1", include=[path])
+ result = code_exec.run("1", include=["latin1.py"], include_root=folder)
self.assertEqual("InvalidRequest", result["errorKind"])
self.assertIn("latin1.py", result["error"])
+ self.assertIn("not readable as Python source", result["error"])
finally:
shutil.rmtree(folder)
@@ -95,10 +107,10 @@ def test_edited_include_module_is_reloaded(self):
path = os.path.join(folder, "helper_mod_reload.py")
with open(path, "w") as handle:
handle.write("VALUE = 1\n")
- first = code_exec.run("import helper_mod_reload\nhelper_mod_reload.VALUE", include=[folder])
+ first = code_exec.run("import helper_mod_reload\nhelper_mod_reload.VALUE", include=[folder], include_outside_root=True)
with open(path, "w") as handle:
handle.write("VALUE = 2\n")
- second = code_exec.run("import helper_mod_reload\nhelper_mod_reload.VALUE", include=[folder])
+ second = code_exec.run("import helper_mod_reload\nhelper_mod_reload.VALUE", include=[folder], include_outside_root=True)
self.assertEqual(("1", "2"), (first["value"], second["value"]))
finally:
shutil.rmtree(folder)
@@ -117,5 +129,226 @@ def test_stderr_goes_to_error_logs(self):
self.assertIn({"level": "info", "message": "partial"}, result["logs"])
+class _Unprintable:
+ def __str__(self):
+ raise RuntimeError("no str")
+
+ __repr__ = __str__
+
+
+class RobustnessTests(unittest.TestCase):
+ """A snippet must never take the host down or flood it (spec sections 4 and 6)."""
+
+ def setUp(self):
+ self.folder = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.folder)
+
+ def write(self, name, data):
+ path = os.path.join(self.folder, name)
+ with open(path, "wb") as handle:
+ handle.write(data)
+ return path
+
+ def test_every_base_exception_is_a_runtime_error(self):
+ for code in ("raise KeyboardInterrupt", "raise GeneratorExit", "class B(BaseException): pass\nraise B()",
+ "import sys\nsys.exit(0)", "exit()"):
+ with self.subTest(code):
+ result = code_exec.run(code)
+ self.assertEqual("Runtime", result["errorKind"], result)
+
+ def test_keyboard_interrupt_keeps_the_snippet_frames(self):
+ error = code_exec.run("def stop():\n raise KeyboardInterrupt('now')\nstop()")["error"]
+ self.assertTrue(error.startswith("KeyboardInterrupt: now"), error)
+ self.assertIn('snippet.py", line 2, in stop', error)
+ self.assertNotIn("runner.py", error)
+
+ def test_unprintable_exception_in_a_base_exception_subclass(self):
+ result = code_exec.run("class E(KeyboardInterrupt):\n def __str__(self):\n raise KeyboardInterrupt\nraise E()")
+ self.assertEqual("Runtime", result["errorKind"])
+ self.assertIn("E: ", result["error"])
+
+ def test_value_whose_str_and_repr_raise(self):
+ result = code_exec.run("class X:\n def __repr__(self):\n raise RuntimeError('no repr')\nX()")
+ self.assertEqual("Runtime", result["errorKind"])
+ self.assertIn("RuntimeError: no repr", result["error"])
+
+ def test_implicit_context_is_reported(self):
+ error = code_exec.run("try:\n 1 / 0\nexcept ZeroDivisionError:\n raise ValueError('then')")["error"]
+ self.assertTrue(error.startswith("ValueError: then"), error)
+ self.assertIn("Raised while handling: ZeroDivisionError: division by zero", error)
+ suppressed = code_exec.run("try:\n 1 / 0\nexcept ZeroDivisionError:\n raise ValueError('then') from None")["error"]
+ self.assertNotIn("ZeroDivisionError", suppressed)
+
+ def test_mutual_recursion_traceback_is_bounded(self):
+ result = code_exec.run("def a(n):\n return b(n)\ndef b(n):\n return a(n)\na(0)")
+ self.assertEqual("Runtime", result["errorKind"])
+ self.assertIn("frame(s) left out", result["error"])
+ self.assertLess(len(result["error"]), 3000)
+
+ def test_runaway_print_loop_keeps_first_and_last_lines(self):
+ result = code_exec.run("for i in range(20000):\n print('line', i)")
+ messages = [entry["message"] for entry in result["logs"]]
+ self.assertEqual("line 0", messages[0])
+ self.assertEqual("line 19999", messages[-1])
+ self.assertTrue(any("log line(s) left out" in m for m in messages))
+ self.assertLessEqual(sum(len(m) + runner._LOG_ENTRY_CHARS for m in messages if "left out" not in m), runner.MAX_LOG_CHARS)
+ self.assertEqual(["logs"], result["truncated"])
+
+ def test_log_budget_matches_the_batch_rule(self):
+ # Spec section 6 states the rule over the whole list: the streaming budget must give the same entries.
+ import random
+ rng = random.Random(7)
+ for _ in range(300):
+ sizes = [rng.randint(0, 40) for _ in range(rng.randint(0, 30))]
+ budget = rng.randint(1, 300)
+ logs = runner._Logs(budget, 0)
+ for i, size in enumerate(sizes):
+ logs.add("info", "%d" % i + "x" * size)
+ messages = [e["message"] for e in logs.entries() if e["level"] == "info"]
+ costs = [len(m) + runner._LOG_ENTRY_CHARS for m in ("%d" % i + "x" * size for i, size in enumerate(sizes))]
+ expected = list(range(len(sizes)))
+ if sum(costs) > budget:
+ head, used = 0, 0
+ while head < len(costs) and used + costs[head] <= budget * 3 // 4:
+ used += costs[head]
+ head += 1
+ tail = len(costs)
+ while tail > head and used + costs[tail - 1] <= budget:
+ used += costs[tail - 1]
+ tail -= 1
+ expected = list(range(head)) + list(range(tail, len(costs)))
+ self.assertEqual(expected, [int(m.rstrip("x")) for m in messages], (sizes, budget))
+
+ def test_huge_value_and_error_are_clipped_by_default(self):
+ self.assertLess(len(code_exec.run("'x' * 5000000")["value"]), 20200)
+ self.assertLess(len(code_exec.run("raise ValueError('y' * 1000000)")["error"]), 20200)
+
+ def test_limits_come_through_the_json_request(self):
+ response = json.loads(code_exec.run_json(json.dumps({"code": "'abcdef'", "maxValueChars": 3})))
+ self.assertEqual("abc... [truncated: 3 of 6 characters shown; return a slice or a summary]", response["value"])
+ for bad in (-1, "10", 1.5, True):
+ with self.subTest(bad):
+ response = json.loads(code_exec.run_json(json.dumps({"code": "1", "maxLogChars": bad})))
+ self.assertEqual("InvalidRequest", response["errorKind"])
+
+ def test_malformed_json_fields_are_invalid_requests(self):
+ for request in ({"code": "1", "includeRoot": 5}, {"code": "1", "imports": [5]}, {"code": "1", "imports": 5},
+ {"code": "1", "include": [1]}, {"code": "1", "include": 5}):
+ with self.subTest(request):
+ self.assertEqual("InvalidRequest", json.loads(code_exec.run_json(json.dumps(request)))["errorKind"])
+
+ def test_odd_converter_value_still_serializes(self):
+ runner.set_value_converter(lambda v: object())
+ try:
+ response = json.loads(code_exec.run_json(json.dumps({"code": "1"})))
+ self.assertTrue(response["success"], response)
+ self.assertIn("object", response["value"])
+ finally:
+ runner.set_value_converter(None)
+
+ def test_working_directory_is_restored(self):
+ before = os.getcwd()
+ code_exec.run("import os\nos.chdir(%r)" % self.folder)
+ self.assertEqual(before, os.getcwd())
+
+ def test_include_with_bom_or_coding_declaration(self):
+ self.write("bom.py", b"\xef\xbb\xbfA = 1\n")
+ self.write("latin.py", b"# -*- coding: latin-1 -*-\nB = '\xe9'\n")
+ result = code_exec.run("A, B", include=["bom.py", "latin.py"], include_root=self.folder)
+ self.assertTrue(result["success"], result)
+ self.assertEqual('[1, "\\u00e9"]', result["value"])
+
+ def test_include_must_stay_inside_the_root(self):
+ inside = os.path.join(self.folder, "inside")
+ os.mkdir(inside)
+ self.write("outside.py", b"X = 1\n")
+ for entry in ("../outside.py", os.path.join(self.folder, "outside.py")):
+ with self.subTest(entry):
+ result = code_exec.run("X", include=[entry], include_root=inside)
+ self.assertEqual("InvalidRequest", result["errorKind"])
+ self.assertIn("outside the include root", result["error"])
+ allowed = json.loads(code_exec.run_json(json.dumps(
+ {"code": "X", "include": ["../outside.py"], "includeRoot": inside, "includeOutsideRoot": True})))
+ self.assertEqual("1", allowed["value"], allowed)
+ # An absolute path inside the root and the root itself are fine.
+ self.write(os.path.join("inside", "h.py"), b"Y = 2\n")
+ self.assertEqual("2", code_exec.run("Y", include=[os.path.join(inside, "h.py")], include_root=inside)["value"])
+ self.assertTrue(code_exec.run("1", include=["."], include_root=inside)["success"])
+
+ def test_broken_interpreter_state_is_restored_and_reported(self):
+ before_path, before_limit = list(sys.path), sys.getrecursionlimit()
+ for code in ("import sys\ndel sys.path\n1", "import sys\nsys.path = None\n1",
+ "import sys, inspect\nsys.setrecursionlimit(len(inspect.stack()) + 8)\nraise ValueError('low')"):
+ with self.subTest(code):
+ result = code_exec.run(code) # must answer, not raise
+ self.assertIn(result["errorKind"], ("None", "Runtime"), result)
+ self.assertEqual(before_path, sys.path)
+ self.assertEqual(before_limit, sys.getrecursionlimit())
+
+ def test_non_text_coding_declaration_is_an_invalid_request(self):
+ self.write("rot.py", b"# coding: rot13\nK = 1\n")
+ self.assertEqual("InvalidRequest", code_exec.run("K", include=["rot.py"], include_root=self.folder)["errorKind"])
+
+ def test_json_entry_points_never_raise(self):
+ self.assertEqual("InvalidRequest", json.loads(code_exec.run_json("[" * 100000))["errorKind"])
+ self.assertEqual("InvalidRequest", json.loads(code_exec.run_b64(None))["errorKind"])
+ for converted in ({(1, 2): 3}, None):
+ runner.set_value_converter(lambda v, converted=converted: converted if converted is not None else _Unprintable())
+ try:
+ response = json.loads(code_exec.run_json(json.dumps({"code": "1"})))
+ self.assertTrue(response["success"], response)
+ finally:
+ runner.set_value_converter(None)
+
+ def test_falsy_wrong_json_types_are_invalid_requests(self):
+ for request in ({"include": ""}, {"include": {}}, {"include": False}, {"imports": {}}, {"imports": 0},
+ {"includeOutsideRoot": "true"}, {"includeRoot": 0}):
+ request["code"] = "1"
+ with self.subTest(request):
+ self.assertEqual("InvalidRequest", json.loads(code_exec.run_json(json.dumps(request)))["errorKind"])
+
+ def test_host_modules_under_an_include_folder_stay_loaded(self):
+ self.write("hostmod_ce.py", b"STATE = []\n")
+ sys.path.insert(0, self.folder)
+ try:
+ import hostmod_ce
+ code_exec.run("1", include=["."], include_root=self.folder)
+ self.assertIs(hostmod_ce, sys.modules.get("hostmod_ce"))
+ finally:
+ sys.path.remove(self.folder)
+ sys.modules.pop("hostmod_ce", None)
+
+ def test_traceback_shows_the_agents_own_lines(self):
+ cwd = os.getcwd()
+ self.write("snippet.py", b"UNRELATED_LINE_1\nUNRELATED_LINE_2\n")
+ os.chdir(self.folder) # a stray snippet.py where linecache would look
+ try:
+ error = code_exec.run("import sys\nsys.tracebacklimit = 0\nraise ValueError('mine')")["error"]
+ finally:
+ os.chdir(cwd)
+ self.assertIn('snippet.py", line 3', error)
+ self.assertIn("raise ValueError('mine')", error)
+ self.assertNotIn("UNRELATED", error)
+
+ @unittest.skipUnless(os.name == "nt", "case-insensitive paths")
+ def test_same_include_in_another_case_runs_once(self):
+ self.write("count.py", b"import builtins\nbuiltins._ce_count = getattr(builtins, '_ce_count', 0) + 1\n")
+ try:
+ code_exec.run("1", include=["count.py", "COUNT.py"], include_root=self.folder)
+ import builtins
+ self.assertEqual(1, builtins._ce_count)
+ finally:
+ import builtins
+ builtins.__dict__.pop("_ce_count", None)
+
+ def test_include_file_must_be_python(self):
+ self.write("data.json", b"{}")
+ result = code_exec.run("1", include=["data.json"], include_root=self.folder)
+ self.assertEqual("InvalidRequest", result["errorKind"])
+ self.assertIn("not a .py file", result["error"])
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/spec/contract.md b/spec/contract.md
index 130605e..683ba86 100644
--- a/spec/contract.md
+++ b/spec/contract.md
@@ -1,4 +1,4 @@
-# Code-Exec contract (v0.2)
+# Code-Exec contract (v0.3)
One request/response shape for every runtime (C# and Python) and every host (Unity, Godot, BRUTAL, Unreal, plain
.NET). **This file is normative: a runtime that disagrees with it is the bug.** Behaviour changes start here, then
@@ -10,12 +10,30 @@ One request/response shape for every runtime (C# and Python) and every host (Uni
|---|---|---|
| `code` | string, required | The snippet. See §3 (C#) / §4 (Python). |
| `usings` | string[] | C# only. Extra namespaces, added to the host's defaults. Aliases (`X = A.B`) and `static A.B` are allowed. |
-| `include` | string[] | Files or folders to compile/load with the snippet. Folders are recursive. Relative paths resolve against the host's include root. |
+| `include` | string[] | Files or folders to compile/load with the snippet (§1.1). |
| `entry` | `{ type, method }` | C# only. Full-source mode: the code is compiled verbatim and `type.method` is invoked. |
| `parameters` | `{ name, type, value }[]` | C# only. Arguments passed to the snippet; `type` is a host type, `value` an already-deserialized object. |
| `timeoutMs` | int | C# only. Upper bound for awaiting an asynchronous result. It does **not** abort code already running. |
| `imports` | string[] | Python only. Modules the host binds into the snippet's namespace before it runs (`unreal`). |
| `includeRoot` | string | Python JSON entry point only (`run_json` / `run_b64`): folder relative `include` paths resolve against. |
+| `includeOutsideRoot` | bool | Python JSON entry point only: the host accepts includes outside the include root (§1.1). Default `false`. |
+| `maxValueChars` | int | Output limit for a text `value` (§6). Default `20000`. |
+| `maxLogChars` | int | Output limit for all `logs` together (§6). Default `20000`. |
+| `maxLogLineChars` | int | Output limit for one log message (§6). Default `2000`. |
+| `maxErrorChars` | int | Output limit for `error` (§6). Default `20000`. |
+
+Every field but `code` is optional. For the four limits, `0` means no limit; a negative (or non-integer) limit is
+`InvalidRequest`. A request without them gets the defaults. A field of the wrong JSON type is `InvalidRequest`.
+
+### 1.1. Includes
+- Relative paths resolve against the host's include root. An include must lie **inside** that root (the root itself
+ included): a path that leaves it (`../x`, an absolute path elsewhere) is `InvalidRequest`, unless the host allows it
+ (C# `CodeRunnerOptions.AllowIncludeOutsideRoot`, Python `include_outside_root=True` / `includeOutsideRoot`). Includes
+ are the agent's helper code, not a way to compile arbitrary project files.
+- A file must be source of the runtime's language: `.cs` (C#), `.py` (Python); anything else is `InvalidRequest`.
+ Folders are recursive and contribute only such files (C#: `*.cs`; Python: the folder goes on `sys.path`).
+- Diagnostics and stack frames name an include by its path relative to the root (`/` separators), or by its absolute
+ path when the host allowed one outside the root.
## 2. Response
@@ -27,10 +45,11 @@ One request/response shape for every runtime (C# and Python) and every host (Uni
| `logs` | `{ level, message }[]` | Output written while the snippet ran (`Console`, `print`, engine log hooks). |
| `errorKind` | enum | `None`, `InvalidRequest`, `Compile`, `EntryPointNotFound`, `Runtime`, `Timeout`, `NotAvailable`. |
| `error` | string | Human-readable error; for `Runtime`: `ExceptionType: message` plus the snippet's stack frames. |
-| `diagnostics` | `{ file, line, column, id, severity, message }[]` | Compile errors in the **agent's** coordinates: `snippet.cs` / `snippet.py` or the include's relative path. |
+| `diagnostics` | `{ file, line, column, id, severity, message }[]` | Compile errors in the **agent's** coordinates: `snippet.cs` / `snippet.py` or the include's relative path (§3.4). At most 20, then one `CE0002` entry counting the rest. |
| `autoImported` | string[] | C# only. Namespaces added automatically (§3.3). |
| `compileMs`, `executeMs` | int | Timings. |
| `cacheHit` | bool | The compiled snippet was reused. |
+| `truncated` | string[] | The fields an output limit shortened (§6), in this order: `value`, `logs`, `error`. Empty when nothing was. |
## 3. C# snippet semantics
@@ -65,7 +84,16 @@ types of the loaded assemblies, except the runtime's own `Internal` / `Internal.
reported in `autoImported`. Several namespaces → no guess: an extra diagnostic `CE0001` lists the candidates.
### 3.4. Errors
-Compile errors are reported with `file` = `snippet.cs` (the agent's own line numbers) or the include's path.
+Compile errors are reported with `file` = `snippet.cs` (the agent's own line numbers) or the include's path. An error
+in a namespace the runtime adds (the host's defaults and the request's `usings`) has `file` = `usings`, line and column
+`0`, and a message that names it: `using 'No.Such.Namespace': The type or namespace name 'No' could not be found ...`.
+Any exception the snippet throws (or its result's conversion throws) is a `Runtime` error: `ExceptionType: message`,
+the snippet's frames, then each inner exception the same way after ` ---> ` (at most 5). An exception whose `Message`
+itself throws is reported as `ExceptionType: `. What .NET cannot catch at all (a stack overflow)
+still ends the process: a snippet that may recurse deeply must bound its own depth.
+
+Host defaults that collide (`System` and `UnityEngine` both define `Object` and `Random`: `CS0104`) are the host's to
+settle with alias defaults, e.g. `Object = UnityEngine.Object`, `Random = UnityEngine.Random`.
### 3.5. Lifetime
Each execution is stateless. On CoreCLR the snippet's assembly is loaded into its own collectible
@@ -73,7 +101,9 @@ Each execution is stateless. On CoreCLR the snippet's assembly is loaded into it
already runs (any load context still alive; same identity first, else the same name at a version no lower than
referenced; ties: the default context, then the higher version, then the later loaded) before the default context is
asked, so the snippet sees the host's live statics, never a second copy. On Mono (Unity) assemblies cannot be unloaded;
-identical requests reuse the already-loaded assembly (cache).
+identical requests reuse the already-loaded assembly (cache) — unless the snippet (or an include) declares static state
+of its own (a static field, auto-property or field-like event in one of its types): then every run loads a fresh copy,
+so no run ever sees another run's statics. Such a snippet costs one assembly per run until the host's domain reloads.
## 4. Python snippet semantics
@@ -81,18 +111,54 @@ identical requests reuse the already-loaded assembly (cache).
- `include` folders are put on `sys.path` for the run; modules loaded from them are dropped from `sys.modules`
before each run, so edited helpers are picked up. `include` files are executed into the snippet's namespace first.
- `print` / `sys.stdout` / `sys.stderr` go to `logs`.
-- Each run gets a fresh namespace; `sys.path` is restored afterwards. A snippet raising `SystemExit` is a `Runtime`
- error — it never stops the host.
-- `SyntaxError` → `Compile` with the snippet's line/column. An exception → `Runtime`; the traceback keeps only
- frames from `snippet.py` and the includes.
+- Each run gets a fresh namespace; `sys.path`, the working directory and the recursion limit are restored afterwards,
+ and the modules the run imported from include folders are dropped (host modules imported before stay).
+- `include` files are read as Python reads source files: UTF-8, or the encoding named by a UTF-8 BOM or a PEP 263
+ coding declaration. A file that does not decode (or names an unknown or non-text encoding) is `InvalidRequest`.
+- `SyntaxError` → `Compile` with the snippet's line/column. Any exception → `Runtime`, **including every
+ `BaseException`** (`SystemExit`, `KeyboardInterrupt`, `GeneratorExit`, custom subclasses): a snippet never stops or
+ interrupts the host. The traceback keeps only frames from `snippet.py` and the includes, then follows the chain (at
+ most 5): `Caused by: ...` for `raise ... from cause`, `Raised while handling: ...` for an implicit `__context__`. An
+ exception whose `str()` raises is reported as `ExceptionType: `.
- `value` conversion: `None` → null; `str`, `int`, `float`, `bool` → their `str()`; `dict` / `list` / `tuple` →
JSON when serializable; anything else → `str()` (hosts may plug a converter, e.g. Unreal objects → path).
## 5. Value text in golden tests
Golden fixtures compare `value` as text: C# `ToString()` (`True`, `42`), Python `str()` (`True`, `42`).
+## 6. Output limits
+A response goes straight into an agent's context, so a runaway `print` loop or a huge value must not flood it. The
+runtime applies the request's limits (§1) before it answers; hosts may limit further (their own logs, other fields).
+Lengths are in characters: UTF-16 code units in C#, code points in Python. A limit of `0` disables that limit.
+
+- **Clip.** A text longer than its limit `N` keeps its first `N` characters followed by
+ `... [truncated: N of M characters shown; HINT]` (`M` = the full length). C# never splits a surrogate pair: it then
+ keeps `N - 1` characters and says so.
+- **`value`** — clipped to `maxValueChars` when the converted value is a string; hint `return a slice or a summary`.
+ A host converter that returns something else (an object it serializes later) limits that itself.
+- **`error`** — clipped to `maxErrorChars`; hint `raise a shorter message`. Before that, a `Runtime` traceback with more
+ than 20 of the agent's frames keeps the first 10 and the last 10, with one line `... K frame(s) left out` between.
+- **`logs`** — every message is clipped to `maxLogLineChars` (hint `print less`). An entry then costs its message length
+ plus 12 (its level and line break as hosts print them). When the entries cost more than `maxLogChars` in total, the
+ runtime keeps the longest run of first entries that costs at most `floor(maxLogChars * 3 / 4)`, then the longest run
+ of last entries that keeps the total within `maxLogChars`, and puts between them one entry
+ `{ "level": "warning", "message": "... K log line(s) left out (output over maxLogChars characters): print less, or
+ return a summary" }` (with the numbers filled in; this entry costs nothing). Runtimes apply this as lines arrive, so
+ a snippet that prints line after line without end does not grow the host's memory either (one unfinished line is
+ held until it ends).
+- `truncated` lists `value`, `logs` and `error` when their limit changed them (a clipped log line counts as `logs`).
+
## Changelog
+- **v0.3** — output limits (§6): request fields `maxValueChars`, `maxLogChars`, `maxLogLineChars`, `maxErrorChars`
+ (defaults 20000 / 20000 / 2000 / 20000, the caps Unreal-MCP and Godot-MCP applied on their side) and response field
+ `truncated`; long tracebacks keep their first and last 10 frames; at most 20 diagnostics. Includes stay inside the
+ include root unless the host allows otherwise, and must be `.cs` / `.py` files (§1.1). Errors report inner / chained
+ exceptions. C#: a bad `using` is reported as itself (`file` = `usings`); a snippet with static state of its own never
+ shares it between runs on Mono. Python: every `BaseException` (`KeyboardInterrupt`, `GeneratorExit`, ...) is a
+ `Runtime` error, the working directory is restored, includes honour a BOM / coding declaration, a malformed JSON
+ request field is `InvalidRequest`. Both runtimes: an exception whose message cannot be printed is still reported.
+
- **v0.2.1** — auto-import skips CoreLib's `Internal.*` namespaces; CoreCLR snippets bind to the already-loaded host
assemblies (any load context) and always reference System.Runtime / System.Console / System.Collections /
System.Linq, loaded or not, when the runtime directory is on disk.