chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,332 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Python.Runtime;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Logging;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Configuration;
|
||||
|
||||
namespace QuantConnect.Python
|
||||
{
|
||||
/// <summary>
|
||||
/// Helper class for Python initialization
|
||||
/// </summary>
|
||||
public static class PythonInitializer
|
||||
{
|
||||
private static bool IncludeSystemPackages;
|
||||
private static string PathToVirtualEnv;
|
||||
|
||||
// Used to allow multiple Python unit and regression tests to be run in the same test run
|
||||
private static bool _isInitialized;
|
||||
|
||||
// Used to hold pending path additions before Initialize is called
|
||||
private static List<string> _pendingPathAdditions = new List<string>();
|
||||
|
||||
private static string _algorithmLocation;
|
||||
|
||||
/// <summary>
|
||||
/// Initialize python.
|
||||
///
|
||||
/// In some cases, we might not need to call BeginAllowThreads, like when we're running
|
||||
/// in a python or non-threaded environment.
|
||||
/// In those cases, we can set the beginAllowThreads parameter to false.
|
||||
/// </summary>
|
||||
public static void Initialize(bool beginAllowThreads = true)
|
||||
{
|
||||
if (!_isInitialized)
|
||||
{
|
||||
Log.Trace($"PythonInitializer.Initialize(): {Messages.PythonInitializer.Start}...");
|
||||
PythonEngine.Initialize();
|
||||
|
||||
if (beginAllowThreads)
|
||||
{
|
||||
// required for multi-threading usage
|
||||
PythonEngine.BeginAllowThreads();
|
||||
}
|
||||
|
||||
_isInitialized = true;
|
||||
|
||||
ConfigurePythonPaths();
|
||||
|
||||
TryInitPythonVirtualEnvironment();
|
||||
Log.Trace($"PythonInitializer.Initialize(): {Messages.PythonInitializer.Ended}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Shutdown python
|
||||
/// </summary>
|
||||
public static void Shutdown()
|
||||
{
|
||||
if (_isInitialized)
|
||||
{
|
||||
Log.Trace($"PythonInitializer.Shutdown(): {Messages.PythonInitializer.Start}");
|
||||
_isInitialized = false;
|
||||
|
||||
try
|
||||
{
|
||||
var pyLock = Py.GIL();
|
||||
Log.Trace($"PythonInitializer.Shutdown(): calling engine shutdown...");
|
||||
PythonEngine.Shutdown();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error(ex);
|
||||
}
|
||||
|
||||
Log.Trace($"PythonInitializer.Shutdown(): {Messages.PythonInitializer.Ended}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds directories to the python path at runtime
|
||||
/// </summary>
|
||||
public static bool AddPythonPaths(IEnumerable<string> paths)
|
||||
{
|
||||
// Filter out any paths that are already on our Python path
|
||||
if (paths.IsNullOrEmpty())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Add these paths to our pending additions
|
||||
_pendingPathAdditions.AddRange(paths.Where(x => !_pendingPathAdditions.Contains(x)));
|
||||
|
||||
if (_isInitialized)
|
||||
{
|
||||
using (Py.GIL())
|
||||
{
|
||||
using dynamic sys = Py.Import("sys");
|
||||
using var locals = new PyDict();
|
||||
locals.SetItem("sys", sys);
|
||||
|
||||
// Filter out any already paths that already exist on our current PythonPath
|
||||
using var pythonCurrentPath = PythonEngine.Eval("sys.path", locals: locals);
|
||||
var currentPath = pythonCurrentPath.As<List<string>>();
|
||||
_pendingPathAdditions = _pendingPathAdditions.Where(x => !currentPath.Contains(x.Replace('\\', '/'))).ToList();
|
||||
|
||||
// Algorithm location most always be before any other path added through this method
|
||||
var insertionIndex = 0;
|
||||
if (!_algorithmLocation.IsNullOrEmpty())
|
||||
{
|
||||
insertionIndex = currentPath.IndexOf(_algorithmLocation.Replace('\\', '/')) + 1;
|
||||
|
||||
if (insertionIndex == 0)
|
||||
{
|
||||
// The algorithm location is not in the current path so it must be in the pending additions list.
|
||||
// Let's move it to the back so it ends up added at the beginning of the path list
|
||||
_pendingPathAdditions.Remove(_algorithmLocation);
|
||||
_pendingPathAdditions.Add(_algorithmLocation);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert any pending path additions
|
||||
if (!_pendingPathAdditions.IsNullOrEmpty())
|
||||
{
|
||||
var code = string.Join(";", _pendingPathAdditions
|
||||
.Select(s => $"sys.path.insert({insertionIndex}, '{s}')")).Replace('\\', '/');
|
||||
PythonEngine.Exec(code, locals: locals);
|
||||
|
||||
_pendingPathAdditions.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds the algorithm location to the python path.
|
||||
/// This will make sure that <see cref="AddPythonPaths" /> keeps the algorithm location path
|
||||
/// at the beginning of the pythonpath.
|
||||
/// </summary>
|
||||
public static void AddAlgorithmLocationPath(string algorithmLocation)
|
||||
{
|
||||
if (!_algorithmLocation.IsNullOrEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Directory.Exists(algorithmLocation))
|
||||
{
|
||||
Log.Error($@"PythonInitializer.AddAlgorithmLocationPath(): {
|
||||
Messages.PythonInitializer.UnableToLocateAlgorithm(algorithmLocation)}");
|
||||
return;
|
||||
}
|
||||
|
||||
_algorithmLocation = algorithmLocation;
|
||||
AddPythonPaths(new[] { _algorithmLocation });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resets the algorithm location path so another can be set
|
||||
/// </summary>
|
||||
public static void ResetAlgorithmLocationPath()
|
||||
{
|
||||
_algorithmLocation = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// "Activate" a virtual Python environment by prepending its library storage to Pythons
|
||||
/// path. This allows the libraries in this venv to be selected prior to our base install.
|
||||
/// Requires PYTHONNET_PYDLL to be set to base install.
|
||||
/// </summary>
|
||||
/// <remarks>If a module is already loaded, Python will use its cached version first
|
||||
/// these modules must be reloaded by reload() from importlib library</remarks>
|
||||
public static bool ActivatePythonVirtualEnvironment(string pathToVirtualEnv)
|
||||
{
|
||||
if (string.IsNullOrEmpty(pathToVirtualEnv))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if(!Directory.Exists(pathToVirtualEnv))
|
||||
{
|
||||
Log.Error($@"PythonIntializer.ActivatePythonVirtualEnvironment(): {
|
||||
Messages.PythonInitializer.VirutalEnvironmentNotFound(pathToVirtualEnv)}");
|
||||
return false;
|
||||
}
|
||||
|
||||
PathToVirtualEnv = pathToVirtualEnv;
|
||||
|
||||
bool? includeSystemPackages = null;
|
||||
var configFile = new FileInfo(Path.Combine(PathToVirtualEnv, "pyvenv.cfg"));
|
||||
if(configFile.Exists)
|
||||
{
|
||||
foreach (var line in File.ReadAllLines(configFile.FullName))
|
||||
{
|
||||
if (line.Contains("include-system-site-packages", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
// format: include-system-site-packages = false (or true)
|
||||
var equalsIndex = line.IndexOf('=', StringComparison.InvariantCultureIgnoreCase);
|
||||
if(equalsIndex != -1 && line.Length > (equalsIndex + 1) && bool.TryParse(line.Substring(equalsIndex + 1).Trim(), out var result))
|
||||
{
|
||||
includeSystemPackages = result;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!includeSystemPackages.HasValue)
|
||||
{
|
||||
includeSystemPackages = true;
|
||||
Log.Error($@"PythonIntializer.ActivatePythonVirtualEnvironment(): {
|
||||
Messages.PythonInitializer.FailedToFindSystemPackagesConfiguration(pathToVirtualEnv, configFile)}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Log.Trace($@"PythonIntializer.ActivatePythonVirtualEnvironment(): {
|
||||
Messages.PythonInitializer.SystemPackagesConfigurationFound(pathToVirtualEnv, includeSystemPackages.Value)}");
|
||||
}
|
||||
|
||||
if (!includeSystemPackages.Value)
|
||||
{
|
||||
PythonEngine.SetNoSiteFlag();
|
||||
}
|
||||
|
||||
IncludeSystemPackages = includeSystemPackages.Value;
|
||||
|
||||
TryInitPythonVirtualEnvironment();
|
||||
return true;
|
||||
}
|
||||
|
||||
private static void TryInitPythonVirtualEnvironment()
|
||||
{
|
||||
if (!_isInitialized || string.IsNullOrEmpty(PathToVirtualEnv))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using (Py.GIL())
|
||||
{
|
||||
using dynamic sys = Py.Import("sys");
|
||||
using var locals = new PyDict();
|
||||
locals.SetItem("sys", sys);
|
||||
|
||||
if (!IncludeSystemPackages)
|
||||
{
|
||||
var currentPath = (List<string>)sys.path.As<List<string>>();
|
||||
var toRemove = new List<string>(currentPath.Where(s => s.Contains("site-packages", StringComparison.InvariantCultureIgnoreCase)));
|
||||
if (toRemove.Count > 0)
|
||||
{
|
||||
var code = string.Join(";", toRemove.Select(s => $"sys.path.remove('{s}')"));
|
||||
PythonEngine.Exec(code, locals: locals);
|
||||
}
|
||||
}
|
||||
|
||||
// fix the prefixes to point to our venv
|
||||
sys.prefix = PathToVirtualEnv;
|
||||
sys.exec_prefix = PathToVirtualEnv;
|
||||
|
||||
using dynamic site = Py.Import("site");
|
||||
// This has to be overwritten because site module may already have been loaded by the interpreter (but not run yet)
|
||||
site.PREFIXES = new List<PyObject> { sys.prefix, sys.exec_prefix };
|
||||
// Run site path modification with tweaked prefixes
|
||||
site.main();
|
||||
|
||||
if (IncludeSystemPackages)
|
||||
{
|
||||
// let's make sure our site packages is at the start so that we support overriding system libraries with a version in the env
|
||||
PythonEngine.Exec(@$"if sys.path[-1].startswith('{PathToVirtualEnv}'):
|
||||
sys.path.insert(0, sys.path.pop())", locals: locals);
|
||||
}
|
||||
|
||||
if (Log.DebuggingEnabled)
|
||||
{
|
||||
using dynamic os = Py.Import("os");
|
||||
var path = new List<string>();
|
||||
foreach (var p in sys.path)
|
||||
{
|
||||
path.Add((string)p);
|
||||
}
|
||||
|
||||
Log.Debug($"PythonIntializer.InitPythonVirtualEnvironment(): PYTHONHOME: {os.getenv("PYTHONHOME")}." +
|
||||
$" PYTHONPATH: {os.getenv("PYTHONPATH")}." +
|
||||
$" sys.executable: {sys.executable}." +
|
||||
$" sys.prefix: {sys.prefix}." +
|
||||
$" sys.base_prefix: {sys.base_prefix}." +
|
||||
$" sys.exec_prefix: {sys.exec_prefix}." +
|
||||
$" sys.base_exec_prefix: {sys.base_exec_prefix}." +
|
||||
$" sys.path: [{string.Join(",", path)}]");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the python additional paths from the config and adds them to Python using the PythonInitializer
|
||||
/// </summary>
|
||||
private static void ConfigurePythonPaths()
|
||||
{
|
||||
var pythonAdditionalPaths = new List<string> { Environment.CurrentDirectory };
|
||||
pythonAdditionalPaths.AddRange(Config.GetValue("python-additional-paths", Enumerable.Empty<string>()));
|
||||
AddPythonPaths(pythonAdditionalPaths.Where(path =>
|
||||
{
|
||||
var pathExists = Directory.Exists(path);
|
||||
if (!pathExists)
|
||||
{
|
||||
Log.Error($"PythonInitializer.ConfigurePythonPaths(): {Messages.PythonInitializer.PythonPathNotFound(path)}");
|
||||
}
|
||||
|
||||
return pathExists;
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user