chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
+193
View File
@@ -0,0 +1,193 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using Python.Runtime;
using System.Threading;
using System.Diagnostics;
using QuantConnect.Python;
using QuantConnect.Logging;
using QuantConnect.Configuration;
using System.Collections.Concurrent;
namespace QuantConnect.AlgorithmFactory
{
/// <summary>
/// Helper class used to start a new debugging session
/// </summary>
public static class DebuggerHelper
{
private static readonly ConcurrentQueue<Py.GILState> _threadsState = new();
/// <summary>
/// The different implemented debugging methods
/// </summary>
public enum DebuggingMethod
{
/// <summary>
/// Local debugging through cmdline.
/// <see cref="Language.Python"/> will use built in 'pdb'
/// </summary>
LocalCmdline,
/// <summary>
/// Visual studio local debugging.
/// <see cref="Language.Python"/> will use 'Python Tools for Visual Studio',
/// attach manually selecting `Python` code type.
/// </summary>
VisualStudio,
/// <summary>
/// Python Tool for Visual Studio Debugger for remote python debugging.
/// <see cref="Language.Python"/>. Deprecated, routes to DebugPy which
/// is it's replacement. Used in the same way.
/// </summary>
PTVSD,
/// <summary>
/// DebugPy - a debugger for Python.
/// <see cref="Language.Python"/> can use `Python Extension` in VS Code
/// or attach to Python in Visual Studio
/// </summary>
DebugPy,
/// <summary>
/// PyCharm PyDev Debugger for remote python debugging.
/// <see cref="Language.Python"/> will use 'Python Debug Server' in PyCharm
/// </summary>
PyCharm
}
/// <summary>
/// Will start a new debugging session
/// </summary>
/// <param name="language">The algorithms programming language</param>
/// <param name="workersInitializationCallback">Optionally, the debugging method will set an action which the data stack workers should execute
/// so we can debug code executed by them, this is specially important for python.</param>
public static void Initialize(Language language, out Action workersInitializationCallback)
{
workersInitializationCallback = null;
if (language == Language.Python)
{
DebuggingMethod debuggingType;
Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), true, out debuggingType);
Log.Trace("DebuggerHelper.Initialize(): initializing python...");
PythonInitializer.Initialize();
Log.Trace("DebuggerHelper.Initialize(): python initialization done");
using (Py.GIL())
{
Log.Trace("DebuggerHelper.Initialize(): starting...");
switch (debuggingType)
{
case DebuggingMethod.LocalCmdline:
PythonEngine.RunSimpleString("import pdb; pdb.set_trace()");
break;
case DebuggingMethod.VisualStudio:
Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach...");
PythonEngine.RunSimpleString(@"import sys; import time;
while not sys.gettrace():
time.sleep(0.25)");
break;
case DebuggingMethod.PTVSD:
Log.Trace("DebuggerHelper.Initialize(): waiting for PTVSD debugger to attach at localhost:5678...");
PythonEngine.RunSimpleString("import ptvsd; ptvsd.enable_attach(); ptvsd.wait_for_attach()");
break;
case DebuggingMethod.DebugPy:
PythonEngine.RunSimpleString(@"import debugpy
from AlgorithmImports import *
from QuantConnect.Logging import *
Log.Trace(""DebuggerHelper.Initialize(): debugpy waiting for attach at port 5678..."");
debugpy.listen(('0.0.0.0', 5678))
debugpy.wait_for_client()");
workersInitializationCallback = DebugpyThreadInitialization;
break;
case DebuggingMethod.PyCharm:
Log.Trace("DebuggerHelper.Initialize(): Attempting to connect to Pycharm PyDev debugger server...");
PythonEngine.RunSimpleString(@"import pydevd_pycharm; import time;
count = 1
while count <= 10:
try:
pydevd_pycharm.settrace('localhost', port=6000, stdoutToServer=True, stderrToServer=True, suspend=False)
print('SUCCESS: Connected to local program')
break
except ConnectionRefusedError:
pass
try:
pydevd_pycharm.settrace('host.docker.internal', port=6000, stdoutToServer=True, stderrToServer=True, suspend=False)
print('SUCCESS: Connected to docker container')
break
except ConnectionRefusedError:
pass
print('\n')
print('Failed: Ensure your PyCharm Debugger is actively waiting for a connection at port 6000!')
print('Try ' + count.__str__() + ' out of 10')
print('\n')
count += 1
time.sleep(3)");
break;
}
Log.Trace("DebuggerHelper.Initialize(): started");
}
}
else if(language == Language.CSharp)
{
if (Debugger.IsAttached)
{
Log.Trace("DebuggerHelper.Initialize(): debugger is already attached, triggering initial break.");
}
else
{
Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach...");
while (!Debugger.IsAttached)
{
Thread.Sleep(250);
}
Log.Trace("DebuggerHelper.Initialize(): debugger attached");
}
}
else
{
throw new NotImplementedException($"DebuggerHelper.Initialize(): not implemented for {language}");
}
}
/// <summary>
/// For each thread we need to create it's python state, we do this by taking the GIL and we later release it by calling 'BeginAllowThreads'
/// but we do not dispose of it. If we did, the state of the debugpy calls we do here are lost. So we keep a reference of the GIL we've
/// created so it's not picked up the C# garbage collector and disposed off, which would clear the py thread state.
/// </summary>
private static void DebugpyThreadInitialization()
{
_threadsState.Enqueue(Py.GIL());
PythonEngine.BeginAllowThreads();
Log.Debug($"DebuggerHelper.Initialize({Thread.CurrentThread.Name}): initializing debugpy for thread...");
using (Py.GIL())
{
PythonEngine.RunSimpleString("import debugpy;debugpy.debug_this_thread();debugpy.trace_this_thread(True)");
}
}
}
}
+402
View File
@@ -0,0 +1,402 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Security.Policy;
using Python.Runtime;
using QuantConnect.Interfaces;
using QuantConnect.Logging;
using QuantConnect.AlgorithmFactory.Python.Wrappers;
using QuantConnect.Configuration;
using QuantConnect.Python;
using QuantConnect.Util;
namespace QuantConnect.AlgorithmFactory
{
/// <summary>
/// Loader creates and manages the memory and exception space of the algorithm, ensuring if it explodes the Lean Engine is intact.
/// </summary>
[ClassInterface(ClassInterfaceType.AutoDual)]
public class Loader : MarshalByRefObject
{
// True if we are in a debugging session
private readonly bool _debugging;
// Defines the maximum amount of time we will allow for instantiating an instance of IAlgorithm
private readonly TimeSpan _loaderTimeLimit;
// Language of the loader class.
private readonly Language _language;
// Defines how we resolve a list of type names into a single type name to be instantiated
private readonly Func<List<string>, string> _multipleTypeNameResolverFunction;
// The worker thread instance the loader will use if not null
private readonly WorkerThread _workerThread;
/// <summary>
/// Memory space of the user algorithm
/// </summary>
public AppDomain appDomain { get; set; }
/// <summary>
/// The algorithm's interface type that we'll be trying to load
/// </summary>
private static readonly Type AlgorithmInterfaceType = typeof (IAlgorithm);
/// <summary>
/// The full type name of QCAlgorithm, this is so we don't pick him up when querying for types
/// </summary>
private const string AlgorithmBaseTypeFullName = "QuantConnect.Algorithm.QCAlgorithm";
/// <summary>
/// The full type name of QCAlgorithmFramework, this is so we don't pick him up when querying for types
/// </summary>
private const string FrameworkBaseTypeFullName = "QuantConnect.Algorithm.Framework.QCAlgorithmFramework";
/// <summary>
/// Creates a new loader with a 10 second maximum load time that forces exactly one derived type to be found
/// </summary>
public Loader()
: this(false, Language.CSharp, TimeSpan.FromSeconds(10), names => names.SingleOrDefault())
{
}
/// <summary>
/// Creates a new loader with the specified configuration
/// </summary>
/// <param name="debugging">True if we are debugging</param>
/// <param name="language">Which language are we trying to load</param>
/// <param name="loaderTimeLimit">
/// Used to limit how long it takes to create a new instance
/// </param>
/// <param name="multipleTypeNameResolverFunction">
/// Used to resolve multiple type names found in assembly to a single type name, if null, defaults to names => names.SingleOrDefault()
///
/// When we search an assembly for derived types of IAlgorithm, sometimes the assembly will contain multiple matching types. This is the case
/// for the QuantConnect.Algorithm assembly in this solution. In order to pick the correct type, consumers must specify how to pick the type,
/// that's what this function does, it picks the correct type from the list of types found within the assembly.
/// </param>
/// <param name="workerThread">The worker thread instance the loader should use</param>
public Loader(bool debugging, Language language, TimeSpan loaderTimeLimit, Func<List<string>, string> multipleTypeNameResolverFunction, WorkerThread workerThread = null)
{
_debugging = debugging;
_language = language;
_workerThread = workerThread;
if (multipleTypeNameResolverFunction == null)
{
throw new ArgumentNullException(nameof(multipleTypeNameResolverFunction));
}
_loaderTimeLimit = loaderTimeLimit;
_multipleTypeNameResolverFunction = multipleTypeNameResolverFunction;
}
/// <summary>
/// Creates a new instance of the specified class in the library, safely.
/// </summary>
/// <param name="assemblyPath">Location of the DLL</param>
/// <param name="algorithmInstance">Output algorithm instance</param>
/// <param name="errorMessage">Output error message on failure</param>
/// <returns>Bool true on successfully loading the class.</returns>
public bool TryCreateAlgorithmInstance(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
//Default initialisation of Assembly.
algorithmInstance = null;
errorMessage = "";
//First most basic check:
if (!File.Exists(assemblyPath))
{
return false;
}
switch (_language)
{
case Language.Python:
TryCreatePythonAlgorithm(assemblyPath, out algorithmInstance, out errorMessage);
break;
default:
TryCreateILAlgorithm(assemblyPath, out algorithmInstance, out errorMessage);
break;
}
//Successful load.
return algorithmInstance != null;
}
/// <summary>
/// Create a new instance of a python algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
algorithmInstance = null;
errorMessage = string.Empty;
//File does not exist.
if (!File.Exists(assemblyPath))
{
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}";
return false;
}
var pythonFile = new FileInfo(assemblyPath);
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
try
{
PythonInitializer.Initialize();
algorithmInstance = new AlgorithmPythonWrapper(moduleName);
// we need stdout for debugging
if (!_debugging && Config.GetBool("mute-python-library-logging", true))
{
using (Py.GIL())
{
PythonEngine.Exec(
@"
import logging, os, sys
sys.stdout = open(os.devnull, 'w')
logging.captureWarnings(True)"
);
}
}
}
catch (Exception e)
{
Log.Error(e);
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {e.Message}";
return false;
}
//Successful load.
return true;
}
/// <summary>
/// Create a generic IL algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreateILAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
errorMessage = "";
algorithmInstance = null;
try
{
byte[] debugInformationBytes = null;
// if the assembly is located in the base directory then don't bother loading the pdbs
// manually, they'll be loaded automatically by the .NET runtime.
var directoryName = new FileInfo(assemblyPath).DirectoryName;
if (directoryName != null && directoryName.TrimEnd(Path.DirectorySeparatorChar) != AppDomain.CurrentDomain.BaseDirectory.TrimEnd(Path.DirectorySeparatorChar))
{
// see if the pdb exists
var mdbFilename = assemblyPath + ".mdb";
var pdbFilename = assemblyPath.Substring(0, assemblyPath.Length - 4) + ".pdb";
if (File.Exists(pdbFilename))
{
debugInformationBytes = File.ReadAllBytes(pdbFilename);
}
// see if the mdb exists
if (File.Exists(mdbFilename))
{
debugInformationBytes = File.ReadAllBytes(mdbFilename);
}
}
//Load the assembly:
Assembly assembly;
if (debugInformationBytes == null)
{
Log.Trace("Loader.TryCreateILAlgorithm(): Loading only the algorithm assembly");
assembly = Assembly.LoadFrom(assemblyPath);
}
else
{
Log.Trace("Loader.TryCreateILAlgorithm(): Loading debug information with algorithm");
var assemblyBytes = File.ReadAllBytes(assemblyPath);
assembly = Assembly.Load(assemblyBytes, debugInformationBytes);
}
//Get the list of extention classes in the library:
var types = GetExtendedTypeNames(assembly);
Log.Debug("Loader.TryCreateILAlgorithm(): Assembly types: " + string.Join(",", types));
//No extensions, nothing to load.
if (types.Count == 0)
{
errorMessage = "Algorithm type was not found.";
Log.Error("Loader.TryCreateILAlgorithm(): Types array empty, no algorithm type found.");
return false;
}
if (types.Count > 1)
{
// reshuffle type[0] to the resolved typename
types[0] = _multipleTypeNameResolverFunction.Invoke(types);
if (string.IsNullOrEmpty(types[0]))
{
errorMessage = "Algorithm type name not found, or unable to resolve multiple algorithm types to a single type. Please verify algorithm type name matches the algorithm name in the configuration file and that there is one and only one class derived from QCAlgorithm.";
Log.Error($"Loader.TryCreateILAlgorithm(): {errorMessage}");
return false;
}
}
//Load the assembly into this AppDomain:
algorithmInstance = (IAlgorithm)assembly.CreateInstance(types[0], true);
if (algorithmInstance != null)
{
Log.Trace("Loader.TryCreateILAlgorithm(): Loaded " + algorithmInstance.GetType().Name);
}
}
catch (ReflectionTypeLoadException err)
{
Log.Error(err);
Log.Error("Loader.TryCreateILAlgorithm(1): " + err.LoaderExceptions[0]);
if (err.InnerException != null) errorMessage = err.InnerException.Message;
}
catch (Exception err)
{
errorMessage = "Algorithm type name not found, or unable to resolve multiple algorithm types to a single type. Please verify algorithm type name matches the algorithm name in the configuration file and that there is one and only one class derived from QCAlgorithm.";
Log.Error($"Loader.TryCreateILAlgorithm(): {errorMessage}\n{err.InnerException ?? err}");
return false;
}
return true;
}
/// <summary>
/// Get a list of all the matching type names in this DLL assembly:
/// </summary>
/// <param name="assembly">Assembly dll we're loading.</param>
/// <returns>String list of types available.</returns>
public static List<string> GetExtendedTypeNames(Assembly assembly)
{
var types = new List<string>();
try
{
Type[] assemblyTypes;
try
{
assemblyTypes = assembly.GetTypes();
}
catch (ReflectionTypeLoadException e)
{
// We may want to exclude possible null values
// See https://stackoverflow.com/questions/7889228/how-to-prevent-reflectiontypeloadexception-when-calling-assembly-gettypes
assemblyTypes = e.Types.Where(t => t != null).ToArray();
var countTypesNotLoaded = e.LoaderExceptions.Length;
Log.Error($"Loader.GetExtendedTypeNames(): Unable to load {countTypesNotLoaded} of the requested types, " +
"see below for more details on what causes an issue:");
foreach (Exception inner in e.LoaderExceptions)
{
Log.Error($"Loader.GetExtendedTypeNames(): {inner.Message}");
}
}
if (assemblyTypes != null && assemblyTypes.Length > 0)
{
types = (from t in assemblyTypes
where t.IsClass // require class
where !t.IsAbstract // require concrete impl
where AlgorithmInterfaceType.IsAssignableFrom(t) // require derived from IAlgorithm
where t.FullName != AlgorithmBaseTypeFullName // require not equal to QuantConnect.QCAlgorithm
where t.FullName != FrameworkBaseTypeFullName // require not equal to QuantConnect.QCAlgorithmFramework
where t.GetConstructor(Type.EmptyTypes) != null // require default ctor
select t.FullName).ToList();
}
else
{
Log.Error("API.GetExtendedTypeNames(): No types found in assembly.");
}
}
catch (Exception err)
{
Log.Error(err);
}
return types;
}
/// <summary>
/// Creates a new instance of the class in the library, safely.
/// </summary>
/// <param name="assemblyPath">Location of the DLL</param>
/// <param name="ramLimit">Limit of the RAM for this process</param>
/// <param name="algorithmInstance">Output algorithm instance</param>
/// <param name="errorMessage">Output error message on failure</param>
/// <returns>bool success</returns>
public bool TryCreateAlgorithmInstanceWithIsolator(string assemblyPath, int ramLimit, out IAlgorithm algorithmInstance, out string errorMessage)
{
IAlgorithm instance = null;
var error = string.Empty;
var success = false;
var isolator = new Isolator();
var complete = isolator.ExecuteWithTimeLimit(_loaderTimeLimit, () =>
{
success = TryCreateAlgorithmInstance(assemblyPath, out instance, out error);
}, ramLimit, sleepIntervalMillis:100, workerThread:_workerThread);
algorithmInstance = instance;
errorMessage = error;
// if the isolator stopped us early add that to our error message
if (!complete)
{
errorMessage = "Failed to create algorithm instance within 10 seconds. Try re-building algorithm. " + error;
}
return complete && success && algorithmInstance != null;
}
#pragma warning disable CS1574
/// <summary>
/// Unload this factory's appDomain.
/// </summary>
/// <remarks>Not used in lean engine. Running the library in an app domain is 10x slower.</remarks>
/// <seealso cref="AppDomain.CreateDomain(string, Evidence, string, string, bool, AppDomainInitializer, string[])"/>
#pragma warning restore CS1574
public void Unload() {
if (appDomain != null)
{
AppDomain.Unload(appDomain);
appDomain = null;
}
}
} // End Algorithm Factory Class
} // End QC Namespace.
@@ -0,0 +1,17 @@
using System.Reflection;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("QuantConnect.AlgorithmFactory")]
[assembly: AssemblyProduct("QuantConnect.AlgorithmFactory")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("a6f998dd-5a08-4a8e-ad94-96749b61df9e")]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,49 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<RootNamespace>QuantConnect.AlgorithmFactory</RootNamespace>
<AssemblyName>QuantConnect.AlgorithmFactory</AssemblyName>
<TargetFramework>net10.0</TargetFramework>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<OutputPath>bin\$(Configuration)\</OutputPath>
<AnalysisMode>AllEnabledByDefault</AnalysisMode>
<DocumentationFile>bin\$(Configuration)\QuantConnect.AlgorithmFactory.xml</DocumentationFile>
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
<Description>QuantConnect LEAN Engine: Algorithm.AlgorithmFactory Project - Factory for instantiation of QCAlgorithm algorithm objects to be used with LEAN</Description>
<NoWarn>CA1062</NoWarn>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<DefineConstants>TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup>
<PackageLicenseFile>LICENSE</PackageLicenseFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.60" />
<PackageReference Include="NodaTime" Version="3.0.5" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Algorithm\QuantConnect.Algorithm.csproj" />
<ProjectReference Include="..\Common\QuantConnect.csproj" />
<ProjectReference Include="..\Configuration\QuantConnect.Configuration.csproj" />
<ProjectReference Include="..\Logging\QuantConnect.Logging.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="..\LICENSE">
<Pack>True</Pack>
<PackagePath></PackagePath>
</None>
</ItemGroup>
</Project>