Bind extension methods to the extended type so they are callable from the instance

CHANGELOG.md

@@ -9,6 +9,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
 
 ### Added
 
+-   Extension methods declared in non-framework assemblies are bound to the type they extend, so they can be called from the instance like in C# (e.g. `order_status.is_open()`)
 -   Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax
 -   Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]).
 -   Add GetPythonThreadID and Interrupt methods in PythonEngine

src/embed_tests/TestExtensionMethods.cs

@@ -0,0 +1,193 @@
+using System.Collections.Generic;
+using System.Linq;
+using NUnit.Framework;
+using Python.Runtime;
+
+namespace Python.EmbeddingTest
+{
+    public enum ExtendedStatus { Open, Closed }
+
+    public interface IExtendedShape
+    {
+        double Side { get; }
+    }
+
+    public class ExtendedModel
+    {
+        public string Name { get; set; } = "model";
+
+        public string Existing() => "member";
+    }
+
+    public class DerivedExtendedModel : ExtendedModel
+    {
+    }
+
+    public class ExtendedSquare : IExtendedShape
+    {
+        public double Side => 2;
+    }
+
+    public static class ExtendedStatusExtensions
+    {
+        public static bool IsOpen(this ExtendedStatus status) => status == ExtendedStatus.Open;
+
+        public static string Describe(this ExtendedStatus status, string prefix, int times = 1)
+            => string.Concat(Enumerable.Repeat(prefix + status, times));
+    }
+
+    public static class ExtendedModelExtensions
+    {
+        public static string Describe(this ExtendedModel model, string prefix = "") => prefix + model.Name;
+
+        // collides with the real instance method, which must win
+        public static string Existing(this ExtendedModel model) => "extension";
+
+        public static double Area(this IExtendedShape shape) => shape.Side * shape.Side;
+
+        public static int Total(this List<int> values) => values.Sum();
+
+        public static T Identity<T>(this T value) => value;
+
+        public static string Twice(this string value) => value + value;
+    }
+
+    public class TestExtensionMethods
+    {
+        private static dynamic module;
+
+        private const string TestModule = @"
+from clr import AddReference
+AddReference(""System"")
+AddReference(""Python.EmbeddingTest"")
+from System.Collections.Generic import List
+from Python.EmbeddingTest import *
+
+def enum_instance():
+    return ExtendedStatus.Open.IsOpen(), ExtendedStatus.Open.is_open(), ExtendedStatus.Closed.is_open()
+
+def enum_with_args():
+    return ExtendedStatus.Open.describe('s-', times=2)
+
+def enum_static_forms():
+    return ExtendedStatusExtensions.is_open(ExtendedStatus.Open), ExtendedStatus.is_open(ExtendedStatus.Closed)
+
+def class_instance():
+    return ExtendedModel().describe(), ExtendedModel().describe(prefix='p-')
+
+def derived_inherits():
+    return DerivedExtendedModel().describe('d-')
+
+def interface_target():
+    return ExtendedSquare().area()
+
+def member_wins():
+    return ExtendedModel().existing()
+
+def closed_generic():
+    values = List[int]()
+    values.Add(1)
+    values.Add(2)
+    return values.total()
+
+def open_generic_not_bound():
+    return hasattr(ExtendedModel(), 'identity')
+
+def python_native_not_bound():
+    return hasattr('ab', 'twice'), ExtendedModelExtensions.twice('ab')
+";
+
+        [OneTimeSetUp]
+        public void OneTimeSetUp()
+        {
+            PythonEngine.Initialize();
+            using var _ = Py.GIL();
+            module = PyModule.FromString("extension_methods", TestModule);
+        }
+
+        [OneTimeTearDown]
+        public void Dispose()
+        {
+            PythonEngine.Shutdown();
+        }
+
+        [Test]
+        public void EnumExtensionIsCallableFromTheInstance()
+        {
+            using var _ = Py.GIL();
+            var result = module.enum_instance();
+            Assert.IsTrue((bool)result[0]);
+            Assert.IsTrue((bool)result[1]);
+            Assert.IsFalse((bool)result[2]);
+        }
+
+        [Test]
+        public void ExtensionArgumentsFollowTheInstance()
+        {
+            using var _ = Py.GIL();
+            Assert.AreEqual("s-Opens-Open", (string)module.enum_with_args());
+        }
+
+        [Test]
+        public void StaticFormsKeepWorking()
+        {
+            using var _ = Py.GIL();
+            var result = module.enum_static_forms();
+            Assert.IsTrue((bool)result[0]);
+            Assert.IsFalse((bool)result[1]);
+        }
+
+        [Test]
+        public void ClassExtensionIsCallableFromTheInstance()
+        {
+            using var _ = Py.GIL();
+            var result = module.class_instance();
+            Assert.AreEqual("model", (string)result[0]);
+            Assert.AreEqual("p-model", (string)result[1]);
+        }
+
+        [Test]
+        public void DerivedClassInheritsBaseClassExtensions()
+        {
+            using var _ = Py.GIL();
+            Assert.AreEqual("d-model", (string)module.derived_inherits());
+        }
+
+        [Test]
+        public void InterfaceExtensionIsBoundToImplementingClass()
+        {
+            using var _ = Py.GIL();
+            Assert.AreEqual(4.0, (double)module.interface_target());
+        }
+
+        [Test]
+        public void RealMemberWinsOverExtension()
+        {
+            using var _ = Py.GIL();
+            Assert.AreEqual("member", (string)module.member_wins());
+        }
+
+        [Test]
+        public void ClosedGenericTargetIsBound()
+        {
+            using var _ = Py.GIL();
+            Assert.AreEqual(3, (int)module.closed_generic());
+        }
+
+        [Test]
+        public void OpenGenericTargetIsNotBound()
+        {
+            using var _ = Py.GIL();
+            Assert.IsFalse((bool)module.open_generic_not_bound());
+        }
+
+        [Test]
+        public void PythonNativeTargetKeepsTheStaticForm()
+        {
+            using var _ = Py.GIL();
+            var result = module.python_native_not_bound();
+            Assert.IsFalse((bool)result[0]);
+            Assert.AreEqual("abab", (string)result[1]);
+        }
+    }
+}

src/perf_tests/Python.PerformanceTests.csproj

@@ -13,7 +13,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.*" />
-    <PackageReference Include="quantconnect.pythonnet" Version="2.0.66" GeneratePathProperty="true">
+    <PackageReference Include="quantconnect.pythonnet" Version="2.0.67" GeneratePathProperty="true">
       <IncludeAssets>compile</IncludeAssets>
     </PackageReference>
   </ItemGroup>
@@ -25,7 +25,7 @@
   </Target>
 
   <Target Name="CopyBaseline" AfterTargets="Build">
-    <Copy SourceFiles="$(NuGetPackageRoot)quantconnect.pythonnet\2.0.66\lib\net10.0\Python.Runtime.dll" DestinationFolder="$(OutDir)baseline" />
+    <Copy SourceFiles="$(NuGetPackageRoot)quantconnect.pythonnet\2.0.67\lib\net10.0\Python.Runtime.dll" DestinationFolder="$(OutDir)baseline" />
   </Target>
 
   <Target Name="CopyNewBuild" AfterTargets="Build">

src/runtime/AssemblyManager.cs

@@ -62,6 +62,7 @@ internal static void Initialize()
             // test/usage after the first init cycle. Clear so the scan runs fresh.
             assembliesNamesCache.Clear();
             assemblies.Clear();
+            ExtensionMethods.Clear();
 
             AppDomain domain = AppDomain.CurrentDomain;
 
@@ -356,7 +357,8 @@ internal static void ScanAssembly(Assembly assembly)
             // gather a list of all of the namespaces contributed to by
             // the assembly.
 
-            foreach (Type t in GetTypes(assembly))
+            var types = GetTypes(assembly);
+            foreach (Type t in types)
             {
                 string ns = t.Namespace ?? "";
                 if (!namespaces.ContainsKey(ns))
@@ -383,6 +385,8 @@ internal static void ScanAssembly(Assembly assembly)
                     GenericUtil.Register(t);
                 }
             }
+
+            ExtensionMethods.Register(assembly, types);
         }
 
         public static AssemblyName[] ListAssemblies()

src/runtime/ClassManager.cs

@@ -416,6 +416,17 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl)
                 }
             }
 
+            // Extension methods are bound as members so they can be called from the instance. A real member with the same name wins, as in C#
+            var extensionMethods = ExtensionMethods.GetMethodsToBind(type).Where(em => !originalMemberNames.Contains(em.Name)).ToArray();
+            if (extensionMethods.Length > 0)
+            {
+                foreach (var em in extensionMethods)
+                {
+                    local.Add(em.Name);
+                }
+                info = info.Concat(extensionMethods).ToArray();
+            }
+
             // Now again to filter w/o losing overloaded member info
             for (i = 0; i < info.Length; i++)
             {

src/runtime/ExtensionMethods.cs

@@ -0,0 +1,114 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+namespace Python.Runtime
+{
+    /// <summary>
+    /// Index of the C# extension methods found in the scanned assemblies, keyed by the type they extend.
+    /// Classes bind them as members so they are callable from an instance, like C# callers do.
+    /// </summary>
+    internal static class ExtensionMethods
+    {
+        private static readonly ConcurrentDictionary<Type, List<MethodInfo>> _methodsByExtendedType = new();
+
+        // The framework ships thousands of extension methods (LINQ above all); binding them onto
+        // every collection would bloat each class, so only user assemblies are indexed.
+        private static readonly string[] _frameworkAssemblyPrefixes = { "System", "Microsoft", "mscorlib", "netstandard", "WindowsBase", "Python.Runtime" };
+
+        /// <summary>
+        /// Drops the index so a re-initialized runtime rebuilds it while re-scanning the assemblies
+        /// </summary>
+        internal static void Clear()
+        {
+            _methodsByExtendedType.Clear();
+        }
+
+        internal static bool IsExtensionMethod(MethodBase method)
+        {
+            return method.IsStatic && method.IsDefined(typeof(ExtensionAttribute), false);
+        }
+
+        /// <summary>
+        /// Indexes the extension methods declared by the given exported types of an assembly
+        /// </summary>
+        internal static void Register(Assembly assembly, IEnumerable<Type> exportedTypes)
+        {
+            if (IsFrameworkAssembly(assembly))
+            {
+                return;
+            }
+
+            foreach (var type in exportedTypes)
+            {
+                // extension methods can only live in non-generic static classes
+                if (!type.IsAbstract || !type.IsSealed || type.IsGenericType || !type.IsDefined(typeof(ExtensionAttribute), false))
+                {
+                    continue;
+                }
+
+                foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Static))
+                {
+                    if (!IsExtensionMethod(method))
+                    {
+                        continue;
+                    }
+
+                    var extendedType = method.GetParameters()[0].ParameterType;
+                    // The binder cannot infer open generic targets (this T, this IEnumerable<T>) from the instance
+                    if (extendedType.ContainsGenericParameters || extendedType.IsByRef)
+                    {
+                        continue;
+                    }
+
+                    var methods = _methodsByExtendedType.GetOrAdd(extendedType, _ => new List<MethodInfo>());
+                    lock (methods)
+                    {
+                        methods.Add(method);
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets the extension methods to bind to the given type: those extending the type itself and the
+        /// interfaces it introduces. Extensions of base classes are inherited through the Python type hierarchy.
+        /// </summary>
+        internal static List<MethodInfo> GetMethodsToBind(Type type)
+        {
+            var result = new List<MethodInfo>();
+            AddMethods(type, result);
+
+            if (!type.IsInterface)
+            {
+                var baseInterfaces = type.BaseType?.GetInterfaces() ?? Type.EmptyTypes;
+                foreach (var @interface in type.GetInterfaces().Except(baseInterfaces))
+                {
+                    AddMethods(@interface, result);
+                }
+            }
+
+            return result;
+        }
+
+        private static void AddMethods(Type extendedType, List<MethodInfo> result)
+        {
+            if (_methodsByExtendedType.TryGetValue(extendedType, out var methods))
+            {
+                lock (methods)
+                {
+                    result.AddRange(methods);
+                }
+            }
+        }
+
+        private static bool IsFrameworkAssembly(Assembly assembly)
+        {
+            var name = assembly.GetName().Name ?? string.Empty;
+            return _frameworkAssemblyPrefixes.Any(prefix => name == prefix || name.StartsWith(prefix + ".", StringComparison.Ordinal));
+        }
+    }
+}