C#: Support replaces-base via the DependabotProxy.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Immutable;
 using System.Collections.Generic;
 using System.IO;
 using System.Security.Cryptography.X509Certificates;
@@ -14,88 +15,84 @@ public class DependabotProxy : IDependabotProxy
         /// <summary>
         /// Represents configurations for package registries.
         /// </summary>
-        /// <param name="Type">The type of package registry.</param>
-        /// <param name="URL">The URL of the package registry.</param>
-        public record class RegistryConfig(string Type, string URL);
-
-        private readonly string host;
-        private readonly string port;
+        public class RegistryConfig
+        {
+            /// <summary>
+            /// The type of the package registry.
+            /// </summary>
+            public string Type { get; init; } = "";
+
+            /// <summary>
+            /// The URL of the package registry.
+            /// </summary>
+            public string URL { get; init; } = "";
+
+            /// <summary>
+            /// A boolean indicating whether this registry replaces the base registry.
+            /// </summary>
+            [JsonProperty("replaces-base")]
+            public bool ReplacesBase { get; init; } = false;
+        };
 
         public string Address { get; }
 
-        public HashSet<string> RegistryURLs { get; }
+        /// <summary>
+        /// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry.
+        /// </summary>
+        private readonly Dictionary<string, bool> registryMapping = [];
+
+        private ImmutableHashSet<string>? registryURLs;
+        public ImmutableHashSet<string> RegistryURLs =>
+            registryURLs ??= registryMapping.Keys.ToImmutableHashSet();
+
+        private ImmutableHashSet<string>? registryBaseURLs;
+        public ImmutableHashSet<string> RegistryBaseURLs =>
+            registryBaseURLs ??= registryMapping.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToImmutableHashSet();
 
         public string? CertificatePath { get; private set; }
 
         public X509Certificate2? Certificate { get; private set; }
 
-        internal static IDependabotProxy? GetDependabotProxy(
-            ILogger logger, IDiagnosticsWriter diagnosticsWriter, TemporaryDirectory tempWorkingDirectory)
+        private DependabotProxy(IDependabotProxyConfiguration config, ILogger logger, TemporaryDirectory tempWorkingDirectory)
         {
-            // Setting HTTP(S)_PROXY and SSL_CERT_FILE have no effect on Windows or macOS,
-            // but we would still end up using the Dependabot proxy to check for feed reachability.
-            // This would result in us discovering that the feeds are reachable, but `dotnet` would
-            // fail to connect to them. To prevent this from happening, we do not initialise an
-            // instance of `DependabotProxy` on those platforms.
-            if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMacOs()) return null;
-
-            // Obtain and store the address of the Dependabot proxy, if available.
-            var host = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyHost);
-            var port = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyPort);
+            Address = $"http://{config.Host}:{config.Port}";
 
-            if (string.IsNullOrWhiteSpace(host) || string.IsNullOrWhiteSpace(port))
-            {
-                logger.LogInfo("No Dependabot proxy credentials are configured.");
-                return null;
-            }
-
-            var result = new DependabotProxy(host, port);
-            logger.LogInfo($"Dependabot proxy configured at {result.Address}");
-
-            // Obtain and store the proxy's certificate, if available.
-            var cert = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyCertificate);
-
-            if (!string.IsNullOrWhiteSpace(cert))
+            if (!string.IsNullOrWhiteSpace(config.Certificate))
             {
                 var certDirPath = new DirectoryInfo(Path.Join(tempWorkingDirectory.DirInfo.FullName, ".dependabot-proxy"));
                 Directory.CreateDirectory(certDirPath.FullName);
 
-                result.CertificatePath = Path.Join(certDirPath.FullName, "proxy.crt");
-                var certFile = new FileInfo(result.CertificatePath);
+                CertificatePath = Path.Join(certDirPath.FullName, "proxy.crt");
+                var certFile = new FileInfo(CertificatePath);
 
                 using var writer = certFile.CreateText();
-                writer.Write(cert);
+                writer.Write(config.Certificate);
                 writer.Close();
 
-                logger.LogInfo($"Stored Dependabot proxy certificate at {result.CertificatePath}");
+                logger.LogInfo($"Stored Dependabot proxy certificate at {CertificatePath}");
 
-                result.Certificate = X509Certificate2.CreateFromPem(cert);
+                Certificate = X509Certificate2.CreateFromPem(config.Certificate);
             }
 
-            // Try to obtain the list of private registry URLs.
-            var registryURLs = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyURLs);
-
-            if (!string.IsNullOrWhiteSpace(registryURLs))
+            if (!string.IsNullOrWhiteSpace(config.RegistryURLs))
             {
                 try
                 {
-                    // The value of the environment variable should be a JSON array of objects, such as:
-                    // [ { "type": "nuget_feed", "url": "https://nuget.pkg.github.com/org/index.json" } ]
-                    var array = JsonConvert.DeserializeObject<List<RegistryConfig>>(registryURLs);
+                    var array = JsonConvert.DeserializeObject<List<RegistryConfig>>(config.RegistryURLs);
                     if (array is not null)
                     {
-                        foreach (RegistryConfig config in array)
+                        foreach (RegistryConfig registry in array)
                         {
                             // The array contains all configured private registries, not just ones for C#.
                             // We ignore the non-C# ones here.
-                            if (!config.Type.Equals("nuget_feed"))
+                            if (!registry.Type.Equals("nuget_feed"))
                             {
-                                logger.LogDebug($"Ignoring registry at '{config.URL}' since it is not of type 'nuget_feed'.");
+                                logger.LogDebug($"Ignoring registry at '{registry.URL}' since it is not of type 'nuget_feed'.");
                                 continue;
                             }
 
-                            logger.LogInfo($"Found private registry at '{config.URL}'");
-                            result.RegistryURLs.Add(config.URL);
+                            logger.LogInfo($"Found private registry at '{registry.URL}'");
+                            registryMapping.AddOrUpdateToLatest(registry.URL, registry.ReplacesBase);
                         }
                     }
                 }
@@ -104,6 +101,39 @@ public record class RegistryConfig(string Type, string URL);
                     logger.LogError($"Unable to parse '{EnvironmentVariableNames.ProxyURLs}': {ex.Message}");
                 }
             }
+        }
+
+        internal static IDependabotProxy? Make(ILogger logger, IDiagnosticsWriter diagnosticsWriter, TemporaryDirectory tempWorkingDirectory)
+        {
+            // Setting HTTP(S)_PROXY and SSL_CERT_FILE have no effect on Windows or macOS,
+            // but we would still end up using the Dependabot proxy to check for feed reachability.
+            // This would result in us discovering that the feeds are reachable, but `dotnet` would
+            // fail to connect to them. To prevent this from happening, we do not initialise an
+            // instance of `DependabotProxy` on those platforms.
+            if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMacOs())
+            {
+                return null;
+            }
+
+            return Make(new DependabotProxyConfiguration(), logger, diagnosticsWriter, tempWorkingDirectory);
+        }
+
+        /// <summary>
+        /// Creates an instance of the Dependabot proxy using the specified configuration.
+        /// Returns null if the proxy cannot be created.
+        /// This overload is exposed primarily to enable platform-independent unit testing.
+        /// </summary>
+        internal static IDependabotProxy? Make(
+            IDependabotProxyConfiguration proxyConfig, ILogger logger, IDiagnosticsWriter diagnosticsWriter, TemporaryDirectory tempWorkingDirectory)
+        {
+            if (string.IsNullOrWhiteSpace(proxyConfig.Host) || string.IsNullOrWhiteSpace(proxyConfig.Port))
+            {
+                logger.LogDebug("No Dependabot proxy credentials are configured.");
+                return null;
+            }
+
+            var result = new DependabotProxy(proxyConfig, logger, tempWorkingDirectory);
+            logger.LogInfo($"Dependabot proxy configured at {result.Address}");
 
             // Emit a diagnostic for the discovered private registries, so that it is easy
             // for users to see that they were picked up.
@@ -125,17 +155,9 @@ public record class RegistryConfig(string Type, string URL);
             return result;
         }
 
-        private DependabotProxy(string host, string port)
-        {
-            this.host = host;
-            this.port = port;
-            this.Address = $"http://{this.host}:{this.port}";
-            this.RegistryURLs = new HashSet<string>();
-        }
-
         public void Dispose()
         {
-            this.Certificate?.Dispose();
+            Certificate?.Dispose();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxyConfiguration.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public class DependabotProxyConfiguration : IDependabotProxyConfiguration
+    {
+        public string? Host { get; } = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyHost);
+
+        public string? Port { get; } = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyPort);
+
+        public string? Certificate { get; } = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyCertificate);
+
+        public string? RegistryURLs { get; } = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyURLs);
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -106,7 +106,7 @@ void exitCallback(int ret, string msg, bool silent)
                 return BuildScript.Success;
             }).Run(SystemBuildActions.Instance, startCallback, exitCallback);
 
-            dependabotProxy = DependabotProxy.GetDependabotProxy(logger, diagnosticsWriter, tempWorkingDirectory);
+            dependabotProxy = DependabotProxy.Make(logger, diagnosticsWriter, tempWorkingDirectory);
 
             try
             {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs

@@ -17,6 +17,8 @@ internal sealed partial class FeedManager : IDisposable
         private readonly IFileProvider fileProvider;
         private readonly DependencyDirectory emptyPackageDirectory;
         private readonly ImmutableHashSet<string> privateRegistryFeeds;
+        private readonly ImmutableHashSet<string> defaultFeeds;
+
         private readonly IFeedManagerIO feedManagerIo;
 
         /// <summary>
@@ -72,14 +74,24 @@ internal sealed partial class FeedManager : IDisposable
         /// </summary>
         public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;
 
+        private readonly Lazy<ImmutableHashSet<string>> lazyReachableDefaultFeeds;
+
+        /// <summary>
+        /// Gets the list of reachable default NuGet feeds.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value;
+
         public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo)
         {
             this.logger = logger;
             this.dotnet = dotnet;
             this.fileProvider = fileProvider;
             this.feedManagerIo = feedManagerIo;
-            privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
+            privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? [];
             HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
+            defaultFeeds = dependabotProxy?.RegistryBaseURLs.Any() == true
+                ? dependabotProxy.RegistryBaseURLs
+                : [PublicNugetOrgFeed];
             emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
 
             lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
@@ -96,6 +108,7 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP
                 var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
                 return reachableFallbackFeeds.ToImmutableHashSet();
             });
+            lazyReachableDefaultFeeds = new Lazy<ImmutableHashSet<string>>(() => CheckSpecifiedFeeds(defaultFeeds));
         }
 
         public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider)
@@ -266,22 +279,6 @@ private ImmutableHashSet<string> CheckSpecifiedFeeds(ImmutableHashSet<string> fe
             return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();
         }
 
-        /// <summary>
-        /// Return true if the default NuGet feed is reachable, false otherwise.
-        /// If the reachability check is disabled, this method will always return true.
-        /// </summary>
-        /// <returns>True if the default NuGet feed is reachable, false otherwise.</returns>
-        public bool IsDefaultFeedReachable()
-        {
-            if (CheckNugetFeedResponsiveness)
-            {
-                var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
-                return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount);
-            }
-
-            return true;
-        }
-
         /// <summary>
         /// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable.
         /// </summary>
@@ -315,8 +312,8 @@ private List<string> GetReachableFallbackNugetFeeds()
             var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
             {
-                fallbackFeeds.Add(PublicNugetOrgFeed);
-                logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}");
+                fallbackFeeds.UnionWith(defaultFeeds);
+                logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", defaultFeeds.OrderBy(f => f))}");
 
                 var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
                 logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs

@@ -1,5 +1,5 @@
 using System;
-using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Security.Cryptography.X509Certificates;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -14,7 +14,12 @@ public interface IDependabotProxy : IDisposable
         /// <summary>
         /// The URLs of package registries that are configured for the proxy.
         /// </summary>
-        HashSet<string> RegistryURLs { get; }
+        ImmutableHashSet<string> RegistryURLs { get; }
+
+        /// <summary>
+        /// The URLs of package registries that replace the base registry.
+        /// </summary>
+        ImmutableHashSet<string> RegistryBaseURLs { get; }
 
         /// <summary>
         /// The path to the temporary file where the certificate is stored.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxyConfiguration.cs

@@ -0,0 +1,21 @@
+using System;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public interface IDependabotProxyConfiguration
+    {
+        // The host of the Dependabot proxy, if available.
+        string? Host { get; }
+
+        // The port of the Dependabot proxy, if available.
+        string? Port { get; }
+
+        // The certificate of the Dependabot proxy, if available.
+        string? Certificate { get; }
+
+        // The list of package registries that are configured for the proxy, if any.
+        // The value of the environment variable should be a JSON array of objects, such as:
+        // [ { "type": "nuget_feed", "url": "https://nuget.pkg.github.com/org/index.json" } ]
+        string? RegistryURLs { get; }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs

@@ -67,10 +67,6 @@ private class NugetExeWrapper : IPackagesConfigRestore
 
             private bool IsWindows => SystemBuildActions.Instance.IsWindows();
 
-            private bool? isDefaultFeedReachable;
-            private bool IsDefaultFeedReachable =>
-                isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable();
-
             /// <summary>
             /// Create the package manager for a specified source tree.
             /// </summary>
@@ -169,15 +165,15 @@ private bool TryRestoreNugetPackage(string packagesConfig)
 
                 List<string> sourcesArgument = [];
                 var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList();
-                var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable;
+                var useDefaultFeeds = feedsToUse.Count == 0 && feedManager.ReachableDefaultFeeds.Count > 0;
 
                 // Explicitly construct the sources to be used for the restore command when checking feed
-                // responsiveness, using private registries, or falling back to nuget.org.
-                if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed)
+                // responsiveness, using private registries, or falling back to default feeds.
+                if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeeds)
                 {
-                    if (useDefaultFeed)
+                    if (useDefaultFeeds)
                     {
-                        feedsToUse.Add(FeedManager.PublicNugetOrgFeed);
+                        feedsToUse.AddRange(feedManager.ReachableDefaultFeeds);
                     }
                     var restoreFeeds = feedManager.RestoreFeeds(feedsToUse);
                     sourcesArgument = restoreFeeds.SelectMany<string, string>(feed => ["-Source", feed]).ToList();