fix: defer Newtonsoft resolver detection in the OpenAPI filters

src/Thinktecture.Runtime.Extensions.Swashbuckle/Swashbuckle/Internal/ThinktectureParameterFilter.cs

@@ -14,7 +14,7 @@ namespace Thinktecture.Swashbuckle.Internal;
 public class ThinktectureParameterFilter : IParameterFilter
 {
    private readonly bool _createExtraSchemasForParameters;
-   private readonly bool _documentNewtonsoftJsonSerialization;
+   private readonly Lazy<bool> _documentNewtonsoftJsonSerialization;
 
    /// <summary>
    /// This is an internal API that supports the Thinktecture.Runtime.Extensions infrastructure and not subject to
@@ -27,7 +27,7 @@ public ThinktectureParameterFilter(
       IOptions<ThinktectureSchemaFilterOptions> options)
    {
       _createExtraSchemasForParameters = options.Value.CreateExtraSchemasForParameters;
-      _documentNewtonsoftJsonSerialization = ThinktectureSchemaFilter.IsNewtonsoftDataContractResolverRegistered(serviceProvider);
+      _documentNewtonsoftJsonSerialization = ThinktectureSchemaFilter.CreateNewtonsoftDataContractResolverDetection(serviceProvider);
    }
 
    /// <inheritdoc />
@@ -44,7 +44,7 @@ public void Apply(
          return;
 
       modelBindingType = context.ParameterInfo.ParameterType.NormalizeStructType(modelBindingType);
-      var serializationType = ThinktectureSchemaFilter.GetSerializationType(context.ParameterInfo.ParameterType, _documentNewtonsoftJsonSerialization);
+      var serializationType = ThinktectureSchemaFilter.GetSerializationType(context.ParameterInfo.ParameterType, _documentNewtonsoftJsonSerialization.Value);
 
       if (parameter is not OpenApiParameter openApiParameter
           || (parameter.Schema?.AllOf?.Count > 0         // Wrapper made by UseAllOfToExtendReferenceSchemas.
@@ -81,7 +81,7 @@ public void Apply(
       // return the key type, because ThinktectureModelBinderProvider falls back to the key for model binding.
       if (MetadataLookup.Find(parameterType) is Metadata.Keyed keyedMetadata)
       {
-         return ThinktectureSchemaFilter.GetSerializationType(parameterType, _documentNewtonsoftJsonSerialization) == parameterType
+         return ThinktectureSchemaFilter.GetSerializationType(parameterType, _documentNewtonsoftJsonSerialization.Value) == parameterType
                    ? keyedMetadata.Type
                    : keyedMetadata.KeyType;
       }

src/Thinktecture.Runtime.Extensions.Swashbuckle/Swashbuckle/Internal/ThinktectureSchemaFilter.cs

@@ -26,7 +26,7 @@ public class ThinktectureSchemaFilter : ISchemaFilter
    private readonly IAdHocUnionSchemaFilter _adHocUnionSchemaFilter;
    private readonly IRegularUnionSchemaFilter _regularUnionSchemaFilter;
    private readonly SwaggerGenOptions _swaggerGenOptions;
-   private readonly bool _documentNewtonsoftJsonSerialization;
+   private readonly Lazy<bool> _documentNewtonsoftJsonSerialization;
 
    /// <summary>
    /// This is an internal API that supports the Thinktecture.Runtime.Extensions infrastructure and not subject to
@@ -51,7 +51,16 @@ public ThinktectureSchemaFilter(
       _adHocUnionSchemaFilter = adHocUnionSchemaFilter;
       _regularUnionSchemaFilter = regularUnionSchemaFilter;
       _swaggerGenOptions = swaggerGenOptions;
-      _documentNewtonsoftJsonSerialization = IsNewtonsoftDataContractResolverRegistered(serviceProvider);
+      _documentNewtonsoftJsonSerialization = CreateNewtonsoftDataContractResolverDetection(serviceProvider);
+   }
+
+   internal static Lazy<bool> CreateNewtonsoftDataContractResolverDetection(IServiceProvider serviceProvider)
+   {
+      // The filters are created while SwaggerGenOptions are being configured. Resolving ISerializerDataContractResolver
+      // at that point re-enters the SwaggerGenOptions configuration, because Swashbuckle (10.3+) builds the resolver
+      // from SchemaGeneratorOptions, which are configured from SwaggerGenOptions. Deferring the lookup to the first
+      // Apply call breaks that cycle.
+      return new Lazy<bool>(() => IsNewtonsoftDataContractResolverRegistered(serviceProvider));
    }
 
    internal static bool IsNewtonsoftDataContractResolverRegistered(IServiceProvider serviceProvider)
@@ -139,7 +148,7 @@ private bool TryHandleTypeWithObjectFactory(
       OpenApiSchema schema,
       SchemaFilterContext context)
    {
-      var serializationType = GetSerializationType(context.Type, _documentNewtonsoftJsonSerialization);
+      var serializationType = GetSerializationType(context.Type, _documentNewtonsoftJsonSerialization.Value);
 
       // Return to prevent infinite recursion
       if (context.Type == serializationType)

test/Thinktecture.Runtime.Extensions.Swashbuckle.Tests.Sources/Swashbuckle/ThinktectureSchemaFilterTests.ObjectFactories.cs

@@ -98,6 +98,26 @@ public async Task Should_use_Newtonsoft_only_object_factory_when_Newtonsoft_supp
          schema.TryGetProperty("properties", out _).Should().BeFalse();
       }
 
+      [Fact]
+      public async Task Should_generate_document_when_data_contract_resolver_depends_on_SchemaGeneratorOptions()
+      {
+         // Regression: the filters looked up ISerializerDataContractResolver while SwaggerGenOptions were being
+         // configured. With Swashbuckle 10.3+ resolving it re-enters that configuration and recurses endlessly.
+         _buildDataContractResolverFromSchemaGeneratorOptions = true;
+
+         App.MapPost("/test", ([FromBody] ComplexValueObjectWithSystemTextJsonOnlyObjectFactory value) => value);
+
+         var openApi = await GetOpenApiJsonAsync();
+
+         using var document = JsonDocument.Parse(openApi);
+
+         document.RootElement
+                 .GetProperty("components")
+                 .GetProperty("schemas")
+                 .TryGetProperty(nameof(ComplexValueObjectWithSystemTextJsonOnlyObjectFactory), out _)
+                 .Should().BeTrue();
+      }
+
       [Fact]
       public async Task Should_ignore_MessagePack_only_object_factory_when_Newtonsoft_support_is_registered()
       {

test/Thinktecture.Runtime.Extensions.Swashbuckle.Tests.Sources/Swashbuckle/ThinktectureSchemaFilterTests.cs

@@ -7,10 +7,13 @@
 using Microsoft.AspNetCore.Mvc.Formatters;
 using Microsoft.AspNetCore.TestHost;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using Microsoft.OpenApi;
 using NSubstitute;
+using Swashbuckle.AspNetCore.SwaggerGen;
 using Thinktecture.Runtime.Tests.Swashbuckle.Helpers;
 using Thinktecture.Swashbuckle;
 
@@ -31,6 +34,7 @@ public abstract partial class ThinktectureSchemaFilterTests : IAsyncLifetime
    private bool _useStringEnumConverter;
    private bool _keepOrphanedKeyTypeSchemas;
    private bool _addSwaggerGenNewtonsoftSupport;
+   private bool _buildDataContractResolverFromSchemaGeneratorOptions;
    private Type? _controllerType;
 
    protected ThinktectureSchemaFilterTests(ITestOutputHelper testOutputHelper)
@@ -76,6 +80,17 @@ private WebApplication CreateApp()
          appBuilder.Services.AddSwaggerGenNewtonsoftSupport();
       }
 
+      if (_buildDataContractResolverFromSchemaGeneratorOptions)
+      {
+         // Mirrors the registration of Swashbuckle 10.3+, whose ISerializerDataContractResolver depends on
+         // SchemaGeneratorOptions, which in turn are configured from SwaggerGenOptions.
+         appBuilder.Services.Replace(ServiceDescriptor.Singleton<ISerializerDataContractResolver>(serviceProvider =>
+         {
+            _ = serviceProvider.GetRequiredService<IOptions<SchemaGeneratorOptions>>().Value;
+            return new JsonSerializerDataContractResolver(new System.Text.Json.JsonSerializerOptions());
+         }));
+      }
+
       if (_useStringEnumConverter)
       {
          // Swashbuckle's schema generator and Thinktecture's value factory both read the MVC JSON options,