diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs index 3bf843d3fa2c..c4afcc9792bc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Immutable; using System.Collections.Generic; using System.IO; using System.Security.Cryptography.X509Certificates; @@ -14,13 +15,39 @@ public class DependabotProxy : IDependabotProxy /// /// Represents configurations for package registries. /// - /// The type of package registry. - /// The URL of the package registry. - public record class RegistryConfig(string Type, string URL); + public class RegistryConfig + { + /// + /// The type of the package registry. + /// + public string Type { get; init; } = ""; + + /// + /// The URL of the package registry. + /// + public string URL { get; init; } = ""; + + /// + /// A boolean indicating whether this registry replaces the base registry. + /// + [JsonProperty("replaces-base")] + public bool ReplacesBase { get; init; } = false; + }; public string Address { get; } - public HashSet RegistryURLs { get; } = []; + /// + /// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry. + /// + private readonly Dictionary registryMapping = []; + + private ImmutableHashSet? registryURLs; + public ImmutableHashSet RegistryURLs => + registryURLs ??= registryMapping.Keys.ToImmutableHashSet(); + + private ImmutableHashSet? registryBaseURLs; + public ImmutableHashSet RegistryBaseURLs => + registryBaseURLs ??= registryMapping.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToImmutableHashSet(); public string? CertificatePath { get; private set; } @@ -65,7 +92,7 @@ private DependabotProxy(IDependabotProxyConfiguration config, ILogger logger, Te } logger.LogInfo($"Found private registry at '{registry.URL}'"); - RegistryURLs.Add(registry.URL); + registryMapping.AddOrUpdateToLatest(registry.URL, registry.ReplacesBase); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 6c4593f3400c..217a0ec11718 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -17,6 +17,7 @@ internal sealed partial class FeedManager : IDisposable private readonly IFileProvider fileProvider; private readonly DependencyDirectory emptyPackageDirectory; private readonly ImmutableHashSet privateRegistryFeeds; + private readonly ImmutableHashSet defaultFeeds; private readonly IFeedManagerIO feedManagerIo; /// @@ -72,14 +73,24 @@ internal sealed partial class FeedManager : IDisposable /// public ImmutableHashSet ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value; + private readonly Lazy> lazyReachableDefaultFeeds; + + /// + /// Gets the list of reachable default NuGet feeds. + /// + public ImmutableHashSet ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value; + public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo) { this.logger = logger; this.dotnet = dotnet; this.fileProvider = fileProvider; this.feedManagerIo = feedManagerIo; - privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; + privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? []; HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0; + defaultFeeds = dependabotProxy?.RegistryBaseURLs.Any() == true + ? dependabotProxy.RegistryBaseURLs + : [PublicNugetOrgFeed]; emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger); lazyExplicitFeeds = new Lazy>(GetExplicitFeeds); @@ -96,6 +107,7 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(); return reachableFallbackFeeds.ToImmutableHashSet(); }); + lazyReachableDefaultFeeds = new Lazy>(() => CheckSpecifiedFeeds(defaultFeeds)); } public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider) @@ -266,22 +278,6 @@ private ImmutableHashSet CheckSpecifiedFeeds(ImmutableHashSet fe return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet(); } - /// - /// Return true if the default NuGet feed is reachable, false otherwise. - /// If the reachability check is disabled, this method will always return true. - /// - /// True if the default NuGet feed is reachable, false otherwise. - public bool IsDefaultFeedReachable() - { - if (CheckNugetFeedResponsiveness) - { - var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false); - return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount); - } - - return true; - } - /// /// Tests which of the feeds given by are reachable. /// @@ -315,8 +311,8 @@ private List GetReachableFallbackNugetFeeds() var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); if (fallbackFeeds.Count == 0) { - fallbackFeeds.Add(PublicNugetOrgFeed); - logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}"); + fallbackFeeds.UnionWith(defaultFeeds); + logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", defaultFeeds.OrderBy(f => f))}"); var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback); logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}"); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs index 37a11900fddf..aafaf851e356 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs @@ -1,5 +1,5 @@ using System; -using System.Collections.Generic; +using System.Collections.Immutable; using System.Security.Cryptography.X509Certificates; namespace Semmle.Extraction.CSharp.DependencyFetching @@ -14,7 +14,12 @@ public interface IDependabotProxy : IDisposable /// /// The URLs of package registries that are configured for the proxy. /// - HashSet RegistryURLs { get; } + ImmutableHashSet RegistryURLs { get; } + + /// + /// The URLs of package registries that replace the base registry. + /// + ImmutableHashSet RegistryBaseURLs { get; } /// /// The path to the temporary file where the certificate is stored. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs index d4403bb955ef..964a24441387 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs @@ -67,10 +67,6 @@ private class NugetExeWrapper : IPackagesConfigRestore private bool IsWindows => SystemBuildActions.Instance.IsWindows(); - private bool? isDefaultFeedReachable; - private bool IsDefaultFeedReachable => - isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable(); - /// /// Create the package manager for a specified source tree. /// @@ -169,15 +165,15 @@ private bool TryRestoreNugetPackage(string packagesConfig) List sourcesArgument = []; var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList(); - var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable; + var useDefaultFeeds = feedsToUse.Count == 0 && feedManager.ReachableDefaultFeeds.Count > 0; // Explicitly construct the sources to be used for the restore command when checking feed - // responsiveness, using private registries, or falling back to nuget.org. - if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed) + // responsiveness, using private registries, or falling back to default feeds. + if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeeds) { - if (useDefaultFeed) + if (useDefaultFeeds) { - feedsToUse.Add(FeedManager.PublicNugetOrgFeed); + feedsToUse.AddRange(feedManager.ReachableDefaultFeeds); } var restoreFeeds = feedManager.RestoreFeeds(feedsToUse); sourcesArgument = restoreFeeds.SelectMany(feed => ["-Source", feed]).ToList(); diff --git a/csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs b/csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs index 9c8c762f5989..83d899c50912 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs @@ -135,7 +135,8 @@ public void TestDependabotRegistryUrls1() // Verify Assert.NotNull(proxy); - Assert.Equal([], proxy.RegistryURLs); + Assert.Empty(proxy.RegistryURLs); + Assert.Empty(proxy.RegistryBaseURLs); } [Fact] @@ -158,6 +159,7 @@ public void TestDependabotRegistryUrls2() Assert.Equal([ "https://nuget.pkg.github.com/org/index.json" ], proxy.RegistryURLs); + Assert.Empty(proxy.RegistryBaseURLs); } [Fact] @@ -180,6 +182,33 @@ public void TestDependabotRegistryUrls3() Assert.Equal([ "https://example.com/org/index.json" ], proxy.RegistryURLs); + Assert.Empty(proxy.RegistryBaseURLs); + } + + [Fact] + public void TestDependabotReplacesBase1() + { + // Setup + var config = new DependabotConfigurationStub + { + Port = "8080", + Host = "localhost", + RegistryURLs = "[ { \"type\": \"nuget_feed\", \"url\": \"https://example.com/org/index.json\", \"replaces-base\": true }, { \"type\": \"nuget_feed\", \"url\": \"https://example2.com/org/index.json\", \"replaces-base\": false } ]" + }; + + // Execute + using var tempWorkingDirectory = MakeTemporaryDirectory(); + using var proxy = DependabotProxy.Make(config, new LoggerStub(), new DiagnosticsWriterStub(), tempWorkingDirectory); + + // Verify + Assert.NotNull(proxy); + Assert.Equal([ + "https://example.com/org/index.json", + "https://example2.com/org/index.json" + ], proxy.RegistryURLs); + Assert.Equal([ + "https://example.com/org/index.json", + ], proxy.RegistryBaseURLs); } } } diff --git a/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs b/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs index f70efdb4cdcc..0a66b4766c1f 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs @@ -1,8 +1,10 @@ using Xunit; using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Linq; +using System.Security.Cryptography.X509Certificates; using Semmle.Extraction.CSharp.DependencyFetching; namespace Semmle.Extraction.Tests @@ -10,9 +12,21 @@ namespace Semmle.Extraction.Tests public class DependabotProxyStub : IDependabotProxy { public string Address { get; } = ""; - public HashSet RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2"]; + public ImmutableHashSet RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2"]; + public ImmutableHashSet RegistryBaseURLs { get; } = []; public string? CertificatePath { get; } = null; - public System.Security.Cryptography.X509Certificates.X509Certificate2? Certificate { get; } = null; + public X509Certificate2? Certificate { get; } = null; + + public void Dispose() { } + } + + public class DependabotProxyStubWithBaseUrls : IDependabotProxy + { + public string Address { get; } = ""; + public ImmutableHashSet RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2", "https://example.com/base1", "https://example.com/base2"]; + public ImmutableHashSet RegistryBaseURLs { get; } = ["https://example.com/base1", "https://example.com/base2"]; + public string? CertificatePath { get; } = null; + public X509Certificate2? Certificate { get; } = null; public void Dispose() { } } @@ -183,5 +197,45 @@ public void TestFeedsToUse() "https://feed.from/folder1" ], feedsToUse); } + + [Fact] + public void TestDefaultFeeds1() + { + // Setup + var feedManager = MakeFeedManager(); + + // Execute + var reachableDefault = feedManager.ReachableDefaultFeeds; + + // Verify + Assert.Equal([ + "https://api.nuget.org/v3/index.json" + ], reachableDefault); + } + + [Fact] + public void TestDefaultFeeds2() + { + // Setup + var logger = new LoggerStub(); + var dotnet = new DotNetStub([], [], [], []); + var dependabotProxy = new DependabotProxyStubWithBaseUrls(); + var fileProvider = new FileProviderStub(); + var feedManagerIo = new FeedManagerIOStub(["https://example.com/registry2", "https://example.com/base1"]); + var feedManager = new FeedManager(logger, dotnet, dependabotProxy, fileProvider, feedManagerIo); + + // Execute + var reachableDefault = feedManager.ReachableDefaultFeeds; + var reachableFallback = feedManager.ReachableFallbackFeeds; + + // Verify + Assert.Equal([ + "https://example.com/base2" + ], reachableDefault); + Assert.Equal([ + "https://example.com/registry1", + "https://example.com/base2" + ], reachableFallback); + } } } diff --git a/csharp/ql/lib/change-notes/2026-09-03-replaces-base.md b/csharp/ql/lib/change-notes/2026-09-03-replaces-base.md new file mode 100644 index 000000000000..bee1c4aca1e1 --- /dev/null +++ b/csharp/ql/lib/change-notes/2026-09-03-replaces-base.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* In `build-mode: none`, private NuGet registries configured with `replaces-base: true` in the organization-level private registry configuration are now used in place of `nuget.org` as fallback feeds when downloading dependencies.