chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
+621
View File
@@ -0,0 +1,621 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using Python.Runtime;
using System.Collections.Generic;
namespace QuantConnect.Python
{
/// <summary>
/// Base class for Python wrapper classes
/// </summary>
public class BasePythonWrapper<TInterface> : IEquatable<BasePythonWrapper<TInterface>>, IDisposable
{
private PyObject _instance;
private object _underlyingClrObject;
private Dictionary<string, PyObject> _pythonMethods;
private Dictionary<string, string> _pythonPropertyNames;
private bool _validateInterface;
/// <summary>
/// Gets the underlying python instance
/// </summary>
protected PyObject Instance => _instance;
/// <summary>
/// Creates a new instance of the <see cref="BasePythonWrapper{TInterface}" /> class
/// </summary>
/// <param name="validateInterface">Whether to perform validations for interface implementation</param>
public BasePythonWrapper(bool validateInterface = true)
{
_validateInterface = validateInterface;
}
/// <summary>
/// Creates a new instance of the <see cref="BasePythonWrapper{TInterface}"/> class with the specified instance
/// </summary>
/// <param name="instance">The underlying python instance</param>
/// <param name="validateInterface">Whether to perform validations for interface implementation</param>
public BasePythonWrapper(PyObject instance, bool validateInterface = true)
: this(validateInterface)
{
SetPythonInstance(instance);
}
/// <summary>
/// Sets the python instance
/// </summary>
/// <param name="instance">The underlying python instance</param>
public void SetPythonInstance(PyObject instance)
{
InitializeContainers();
_instance = _validateInterface ? instance.ValidateImplementationOf<TInterface>() : instance;
_instance.TryConvert(out _underlyingClrObject);
}
/// <summary>
/// Sets the python instance and sets the validate interface flag
/// </summary>
/// <param name="instance">The underlying python instance</param>
/// <param name="validateInterface">Whether to perform validations for interface implementation</param>
protected void SetPythonInstance(PyObject instance, bool validateInterface)
{
_validateInterface = validateInterface;
SetPythonInstance(instance);
}
private void InitializeContainers()
{
if (_pythonMethods != null && _pythonPropertyNames != null)
{
_pythonMethods.Clear();
_pythonPropertyNames.Clear();
return;
}
_pythonMethods = new();
_pythonPropertyNames = new();
}
/// <summary>
/// Gets the Python instance property with the specified name
/// </summary>
/// <param name="propertyName">The name of the property</param>
public T GetProperty<T>(string propertyName)
{
using var _ = Py.GIL();
return PythonRuntimeChecker.ConvertAndDispose<T>(GetProperty(propertyName), propertyName, isMethod: false);
}
/// <summary>
/// Gets the Python instance property with the specified name
/// </summary>
/// <param name="propertyName">The name of the property</param>
public PyObject GetProperty(string propertyName)
{
using var _ = Py.GIL();
return _instance.GetAttr(GetPropertyName(propertyName));
}
/// <summary>
/// Sets the Python instance property with the specified name
/// </summary>
/// <param name="propertyName">The name of the property</param>
/// <param name="value">The property value</param>
public void SetProperty(string propertyName, object value)
{
using var _ = Py.GIL();
_instance.SetAttr(GetPropertyName(propertyName), value.ToPython());
}
/// <summary>
/// Gets the Python instance event with the specified name
/// </summary>
/// <param name="name">The name of the event</param>
public dynamic GetEvent(string name)
{
using var _ = Py.GIL();
return _instance.GetAttr(GetPropertyName(name, true));
}
/// <summary>
/// Determines whether the Python instance has the specified attribute
/// </summary>
/// <param name="name">The attribute name</param>
/// <returns>Whether the Python instance has the specified attribute</returns>
public bool HasAttr(string name)
{
using var _ = Py.GIL();
return _instance.HasAttr(name) || _instance.HasAttr(name.ToSnakeCase());
}
/// <summary>
/// Gets the Python instances method with the specified name and caches it
/// </summary>
/// <param name="methodName">The name of the method</param>
/// <param name="pythonOnly">Whether to only return python methods</param>
/// <returns>The matched method</returns>
public PyObject GetMethod(string methodName, bool pythonOnly = false)
{
if (!_pythonMethods.TryGetValue(methodName, out var method))
{
method = pythonOnly ? _instance.GetPythonMethod(methodName) : _instance.GetMethod(methodName);
_pythonMethods = AddToDictionary(_pythonMethods, methodName, method);
}
return method;
}
/// <summary>
/// Invokes the specified method with the specified arguments
/// </summary>
/// <param name="methodName">The name of the method</param>
/// <param name="args">The arguments to call the method with</param>
/// <returns>The returned valued converted to the given type</returns>
public T InvokeMethod<T>(string methodName, params object[] args)
{
var method = GetMethod(methodName);
return PythonRuntimeChecker.InvokeMethod<T>(method, methodName, args);
}
/// <summary>
/// Invokes the specified method with the specified arguments
/// </summary>
/// <param name="methodName">The name of the method</param>
/// <param name="args">The arguments to call the method with</param>
public PyObject InvokeMethod(string methodName, params object[] args)
{
using var _ = Py.GIL();
var method = GetMethod(methodName);
return method.Invoke(args);
}
/// <summary>
/// Invokes the specified method with the specified arguments without returning a value
/// </summary>
/// <param name="methodName">The name of the method</param>
/// <param name="args">The arguments to call the method with</param>
public void InvokeVoidMethod(string methodName, params object[] args)
{
InvokeMethod(methodName, args).Dispose();
}
/// <summary>
/// Invokes the specified method with the specified arguments and iterates over the returned values
/// </summary>
/// <param name="methodName">The name of the method</param>
/// <param name="args">The arguments to call the method with</param>
/// <returns>The returned valued converted to the given type</returns>
public IEnumerable<T> InvokeMethodAndEnumerate<T>(string methodName, params object[] args)
{
var method = GetMethod(methodName);
return PythonRuntimeChecker.InvokeMethodAndEnumerate<T>(method, methodName, args);
}
/// <summary>
/// Invokes the specified method with the specified arguments and iterates over the returned values
/// </summary>
/// <param name="methodName">The name of the method</param>
/// <param name="args">The arguments to call the method with</param>
/// <returns>The returned valued converted to the given type</returns>
public Dictionary<TKey, TValue> InvokeMethodAndGetDictionary<TKey, TValue>(string methodName, params object[] args)
{
var method = GetMethod(methodName);
return PythonRuntimeChecker.InvokeMethodAndGetDictionary<TKey, TValue>(method, methodName, args);
}
/// <summary>
/// Invokes the specified method with the specified arguments and out parameters
/// </summary>
/// <param name="methodName">The name of the method</param>
/// <param name="outParametersTypes">The types of the out parameters</param>
/// <param name="outParameters">The out parameters values</param>
/// <param name="args">The arguments to call the method with</param>
/// <returns>The returned valued converted to the given type</returns>
public T InvokeMethodWithOutParameters<T>(string methodName, Type[] outParametersTypes, out object[] outParameters, params object[] args)
{
var method = GetMethod(methodName);
return PythonRuntimeChecker.InvokeMethodAndGetOutParameters<T>(method, methodName, outParametersTypes, out outParameters, args);
}
/// <summary>
/// Invokes the specified method with the specified arguments and wraps the result
/// by calling the given function if the result is not a C# object
/// </summary>
/// <param name="methodName">The name of the method</param>
/// <param name="wrapResult">Method that wraps a Python object in the corresponding Python Wrapper</param>
/// <param name="args">The arguments to call the method with</param>
/// <returns>The returned value wrapped using the given method if the result is not a C# object</returns>
public T InvokeMethodAndWrapResult<T>(string methodName, Func<PyObject, T> wrapResult, params object[] args)
{
var method = GetMethod(methodName);
return PythonRuntimeChecker.InvokeMethodAndWrapResult(method, methodName, wrapResult, args);
}
private string GetPropertyName(string propertyName, bool isEvent = false)
{
if (!_pythonPropertyNames.TryGetValue(propertyName, out var pythonPropertyName))
{
var snakeCasedPropertyName = propertyName.ToSnakeCase();
// If the object is actually a C# object (e.g. a child class of a C# class),
// we check which property was defined in the Python class (if any), either the snake-cased or the original name.
if (!isEvent && _underlyingClrObject != null)
{
var underlyingClrObjectType = _underlyingClrObject.GetType();
var property = underlyingClrObjectType.GetProperty(propertyName);
if (property != null)
{
var clrPropertyValue = property.GetValue(_underlyingClrObject);
var pyObjectSnakeCasePropertyValue = _instance.GetAttr(snakeCasedPropertyName);
if (!pyObjectSnakeCasePropertyValue.TryConvert(out object pyObjectSnakeCasePropertyClrValue, true) ||
!ReferenceEquals(clrPropertyValue, pyObjectSnakeCasePropertyClrValue))
{
pythonPropertyName = snakeCasedPropertyName;
}
else
{
pythonPropertyName = propertyName;
}
}
}
if (pythonPropertyName == null)
{
pythonPropertyName = snakeCasedPropertyName;
if (!_instance.HasAttr(pythonPropertyName))
{
pythonPropertyName = propertyName;
}
}
_pythonPropertyNames = AddToDictionary(_pythonPropertyNames, propertyName, pythonPropertyName);
}
return pythonPropertyName;
}
/// <summary>
/// Adds a key-value pair to the dictionary by copying the original one first and returning a new dictionary
/// containing the new key-value pair along with the original ones.
/// We do this in order to avoid the overhead of using locks or concurrent dictionaries and still be thread-safe.
/// </summary>
private static Dictionary<string, T> AddToDictionary<T>(Dictionary<string, T> dictionary, string key, T value)
{
return new Dictionary<string, T>(dictionary)
{
[key] = value
};
}
/// <summary>
/// Determines whether the specified instance wraps the same Python object reference as this instance,
/// which would indicate that they are equal.
/// </summary>
/// <param name="other">The other object to compare this with</param>
/// <returns>True if both instances are equal, that is if both wrap the same Python object reference</returns>
public virtual bool Equals(BasePythonWrapper<TInterface> other)
{
return other is not null && (ReferenceEquals(this, other) || Equals(other._instance));
}
/// <summary>
/// Determines whether the specified object is an instance of <see cref="BasePythonWrapper{TInterface}"/>
/// and wraps the same Python object reference as this instance, which would indicate that they are equal.
/// </summary>
/// <param name="obj">The other object to compare this with</param>
/// <returns>True if both instances are equal, that is if both wrap the same Python object reference</returns>
public override bool Equals(object obj)
{
return Equals(obj as PyObject) || Equals(obj as BasePythonWrapper<TInterface>);
}
/// <summary>
/// Gets the hash code for the current instance
/// </summary>
/// <returns>The hash code of the current instance</returns>
public override int GetHashCode()
{
using var _ = Py.GIL();
return PythonReferenceComparer.Instance.GetHashCode(_instance);
}
/// <summary>
/// Determines whether the specified <see cref="PyObject"/> is equal to the current instance's underlying Python object.
/// </summary>
private bool Equals(PyObject other)
{
if (other is null) return false;
if (ReferenceEquals(_instance, other)) return true;
using var _ = Py.GIL();
// We only care about the Python object reference, not the underlying C# object reference for comparison
return PythonReferenceComparer.Instance.Equals(_instance, other);
}
/// <summary>
/// Dispose of this instance
/// </summary>
public virtual void Dispose()
{
using var _ = Py.GIL();
if (_pythonMethods != null)
{
foreach (var methods in _pythonMethods.Values)
{
methods.Dispose();
}
_pythonMethods.Clear();
}
_instance?.Dispose();
}
/// <summary>
/// Attempts to invoke the method if it has been overridden in Python.
/// </summary>
/// <typeparam name="T">The expected return type of the Python method.</typeparam>
/// <param name="methodName">The name of the method to call on the Python instance.</param>
/// <param name="result">When this method returns, contains the method result if the call succeeded.</param>
/// <param name="args">The arguments to pass to the Python method.</param>
/// <returns>true if the Python method was successfully invoked, otherwise, false.</returns>
protected bool TryInvokePythonOverride<T>(string methodName, out T result, params object[] args)
{
if (_instance != null)
{
var method = GetMethod(methodName, true);
if (method != null)
{
result = PythonRuntimeChecker.InvokeMethod<T>(method, methodName, args);
return true;
}
}
result = default;
return false;
}
/// <summary>
/// Set of helper methods to invoke Python methods with runtime checks for return values and out parameter's conversions.
/// </summary>
public class PythonRuntimeChecker
{
/// <summary>
/// Invokes method <paramref name="method"/> and converts the returned value to type <typeparamref name="TResult"/>
/// </summary>
public static TResult InvokeMethod<TResult>(PyObject method, string pythonMethodName, params object[] args)
{
using var _ = Py.GIL();
using var result = method.Invoke(args);
return Convert<TResult>(result, pythonMethodName);
}
/// <summary>
/// Invokes method <paramref name="method"/>, expecting an enumerable or generator as return value,
/// converting each item to type <typeparamref name="TItem"/> on demand.
/// </summary>
public static IEnumerable<TItem> InvokeMethodAndEnumerate<TItem>(PyObject method, string pythonMethodName, params object[] args)
{
using var _ = Py.GIL();
var result = method.Invoke(args);
foreach (var item in EnumerateAndDisposeItems<TItem>(result, pythonMethodName))
{
yield return item;
}
result.Dispose();
}
/// <summary>
/// Invokes method <paramref name="method"/>, expecting a dictionary as return value,
/// which then will be converted to a managed dictionary, with type checking on each item conversion.
/// </summary>
public static Dictionary<TKey, TValue> InvokeMethodAndGetDictionary<TKey, TValue>(PyObject method, string pythonMethodName, params object[] args)
{
using var _ = Py.GIL();
using var result = method.Invoke(args);
Dictionary<TKey, TValue> dict;
if (result.TryConvert(out dict))
{
// this is required if the python implementation is actually returning a C# dict, not common,
// but could happen if its actually calling a base C# implementation
return dict;
}
dict = new();
Func<PyObject, string> keyErrorMessageFunc =
(pyItem) => Messages.BasePythonWrapper.InvalidDictionaryKeyType(pythonMethodName, typeof(TKey), pyItem.GetPythonType());
foreach (var (managedKey, pyKey) in Enumerate<TKey>(result, pythonMethodName, keyErrorMessageFunc))
{
var pyValue = result.GetItem(pyKey);
try
{
dict[managedKey] = pyValue.GetAndDispose<TValue>();
}
catch (InvalidCastException ex)
{
throw new InvalidCastException(
Messages.BasePythonWrapper.InvalidDictionaryValueType(pythonMethodName, typeof(TValue), pyValue.GetPythonType()),
ex);
}
}
return dict;
}
/// <summary>
/// Invokes method <paramref name="method"/> and tries to convert the returned value to type <typeparamref name="TResult"/>.
/// If conversion is not possible, the returned PyObject is passed to the provided <paramref name="wrapResult"/> method,
/// which should try to do the proper conversion, wrapping or handling of the PyObject.
/// </summary>
public static TResult InvokeMethodAndWrapResult<TResult>(PyObject method, string pythonMethodName, Func<PyObject, TResult> wrapResult,
params object[] args)
{
using var _ = Py.GIL();
var result = method.Invoke(args);
if (!result.TryConvert<TResult>(out var managedResult))
{
return wrapResult(result);
}
result.Dispose();
return managedResult;
}
/// <summary>
/// Invokes method <paramref name="method"/> and converts the returned value to type <typeparamref name="TResult"/>.
/// It also makes sure the Python method returns values for the out parameters, converting them into the expected types
/// in <paramref name="outParametersTypes"/> and placing them in the <paramref name="outParameters"/> array.
/// </summary>
public static TResult InvokeMethodAndGetOutParameters<TResult>(PyObject method, string pythonMethodName, Type[] outParametersTypes,
out object[] outParameters, params object[] args)
{
using var _ = Py.GIL();
using var result = method.Invoke(args);
// Since pythonnet does not support out parameters, the methods return
// a tuple where the out parameter come after the other returned values
if (!PyTuple.IsTupleType(result))
{
throw new ArgumentException(
Messages.BasePythonWrapper.InvalidReturnTypeForMethodWithOutParameters(pythonMethodName, result.GetPythonType()));
}
if (result.Length() < outParametersTypes.Length + 1)
{
throw new ArgumentException(Messages.BasePythonWrapper.InvalidReturnTypeTupleSizeForMethodWithOutParameters(
pythonMethodName, outParametersTypes.Length + 1, result.Length()));
}
var managedResult = Convert<TResult>(result[0], pythonMethodName);
outParameters = new object[outParametersTypes.Length];
var i = 0;
try
{
for (; i < outParametersTypes.Length; i++)
{
outParameters[i] = result[i + 1].AsManagedObject(outParametersTypes[i]);
}
}
catch (InvalidCastException exception)
{
throw new InvalidCastException(
Messages.BasePythonWrapper.InvalidOutParameterType(pythonMethodName, i, outParametersTypes[i], result[i + 1].GetPythonType()),
exception);
}
return managedResult;
}
/// <summary>
/// Converts the given PyObject into the provided <typeparamref name="T"/> type,
/// generating an exception with a user-friendly message if conversion is not possible.
/// </summary>
public static T Convert<T>(PyObject pyObject, string pythonName, bool isMethod = true)
{
var type = typeof(T);
try
{
if (type == typeof(void))
{
return default;
}
if (type == typeof(PyObject))
{
return (T)(object)pyObject;
}
return (T)pyObject.AsManagedObject(type);
}
catch (InvalidCastException e)
{
throw new InvalidCastException(Messages.BasePythonWrapper.InvalidReturnType(pythonName, type, pyObject.GetPythonType(), isMethod), e);
}
}
/// <summary>
/// Converts the given PyObject into the provided <typeparamref name="T"/> type,
/// generating an exception with a user-friendly message if conversion is not possible.
/// It will dispose of the source PyObject.
/// </summary>
public static T ConvertAndDispose<T>(PyObject pyObject, string pythonName, bool isMethod = true)
{
try
{
return Convert<T>(pyObject, pythonName, isMethod);
}
finally
{
pyObject.Dispose();
}
}
/// <summary>
/// Verifies that the <paramref name="result"/> value is iterable and converts each item into the <typeparamref name="TItem"/> type,
/// returning also the corresponding source PyObject for each one of them.
/// </summary>
private static IEnumerable<(TItem, PyObject)> Enumerate<TItem>(PyObject result, string pythonMethodName,
Func<PyObject, string> getInvalidCastExceptionMessage = null)
{
if (!result.IsIterable())
{
throw new InvalidCastException(Messages.BasePythonWrapper.InvalidIterable(pythonMethodName, typeof(TItem), result.GetPythonType()));
}
using var iterator = result.GetIterator();
foreach (PyObject item in iterator)
{
TItem managedItem;
try
{
managedItem = item.As<TItem>();
}
catch (InvalidCastException ex)
{
var message = getInvalidCastExceptionMessage?.Invoke(item) ??
Messages.BasePythonWrapper.InvalidMethodIterableItemType(pythonMethodName, typeof(TItem), item.GetPythonType());
throw new InvalidCastException(message, ex);
}
yield return (managedItem, item);
}
}
/// <summary>
/// Verifies that the <paramref name="result"/> value is iterable and converts each item into the <typeparamref name="TItem"/> type.
/// </summary>
private static IEnumerable<TItem> EnumerateAndDisposeItems<TItem>(PyObject result, string pythonMethodName)
{
foreach (var (managedItem, pyItem) in Enumerate<TItem>(result, pythonMethodName))
{
pyItem.Dispose();
yield return managedItem;
}
}
}
}
}
+46
View File
@@ -0,0 +1,46 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Benchmarks;
using System;
namespace QuantConnect.Python
{
/// <summary>
/// Provides an implementation of <see cref="IBenchmark"/> that wraps a <see cref="PyObject"/> object
/// </summary>
public class BenchmarkPythonWrapper : BasePythonWrapper<IBenchmark>, IBenchmark
{
/// <summary>
/// Constructor for initialising the <see cref="BenchmarkPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Python benchmark model</param>
public BenchmarkPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Evaluates this benchmark at the specified time using the method defined in the Python class
/// </summary>
/// <param name="time">The time to evaluate the benchmark at</param>
/// <returns>The value of the benchmark at the specified time</returns>
public decimal Evaluate(DateTime time)
{
return InvokeMethod<decimal>(nameof(Evaluate), time);
}
}
}
@@ -0,0 +1,54 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Brokerages;
namespace QuantConnect.Python
{
/// <summary>
/// Provides a wrapper for <see cref="IBrokerageMessageHandler"/> implementations written in python
/// </summary>
public class BrokerageMessageHandlerPythonWrapper : BasePythonWrapper<IBrokerageMessageHandler>, IBrokerageMessageHandler
{
/// <summary>
/// Initializes a new instance of the <see cref="BrokerageMessageHandlerPythonWrapper"/> class
/// </summary>
/// <param name="model">The python implementation of <see cref="IBrokerageMessageHandler"/></param>
public BrokerageMessageHandlerPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Handles the message
/// </summary>
/// <param name="message">The message to be handled</param>
public void HandleMessage(BrokerageMessageEvent message)
{
InvokeMethod(nameof(HandleMessage), message);
}
/// <summary>
/// Handles a new order placed manually in the brokerage side
/// </summary>
/// <param name="eventArgs">The new order event</param>
/// <returns>Whether the order should be added to the transaction handler</returns>
public bool HandleOrder(NewBrokerageOrderNotificationEventArgs eventArgs)
{
return InvokeMethod<bool>(nameof(HandleOrder), eventArgs);
}
}
}
@@ -0,0 +1,305 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using Python.Runtime;
using QuantConnect.Benchmarks;
using QuantConnect.Brokerages;
using QuantConnect.Data.Market;
using QuantConnect.Data.Shortable;
using QuantConnect.Interfaces;
using QuantConnect.Orders;
using QuantConnect.Orders.Fees;
using QuantConnect.Orders.Fills;
using QuantConnect.Orders.Slippage;
using QuantConnect.Securities;
namespace QuantConnect.Python
{
/// <summary>
/// Provides an implementation of <see cref="IBrokerageModel"/> that wraps a <see cref="PyObject"/> object
/// </summary>
public class BrokerageModelPythonWrapper : BasePythonWrapper<IBrokerageModel>, IBrokerageModel
{
/// <summary>
/// Constructor for initialising the <see cref="BrokerageModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Models brokerage transactions, fees, and order</param>
public BrokerageModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Gets or sets the account type used by this model
/// </summary>
public AccountType AccountType
{
get
{
return GetProperty<AccountType>(nameof(AccountType));
}
}
/// <summary>
/// Gets the brokerages model percentage factor used to determine the required unused buying power for the account.
/// From 1 to 0. Example: 0 means no unused buying power is required. 0.5 means 50% of the buying power should be left unused.
/// </summary>
public decimal RequiredFreeBuyingPowerPercent
{
get
{
return GetProperty<decimal>(nameof(RequiredFreeBuyingPowerPercent));
}
}
/// <summary>
/// Gets a map of the default markets to be used for each security type
/// </summary>
public IReadOnlyDictionary<SecurityType, string> DefaultMarkets
{
get
{
using (Py.GIL())
{
var markets = GetProperty(nameof(DefaultMarkets)) as dynamic;
if ((markets as PyObject).TryConvert(out IReadOnlyDictionary<SecurityType, string> csharpDic))
{
return csharpDic;
}
var dic = new Dictionary<SecurityType, string>();
foreach (var item in markets)
{
using var pyItem = item as PyObject;
var market = pyItem.As<SecurityType>();
dic[market] = markets[item];
}
(markets as PyObject).Dispose();
return dic;
}
}
}
/// <summary>
/// Applies the split to the specified order ticket
/// </summary>
/// <param name="tickets">The open tickets matching the split event</param>
/// <param name="split">The split event data</param>
public void ApplySplit(List<OrderTicket> tickets, Split split)
{
InvokeMethod(nameof(ApplySplit), tickets, split);
}
/// <summary>
/// Returns true if the brokerage would be able to execute this order at this time assuming
/// market prices are sufficient for the fill to take place. This is used to emulate the
/// brokerage fills in backtesting and paper trading. For example some brokerages may not perform
/// executions during extended market hours. This is not intended to be checking whether or not
/// the exchange is open, that is handled in the Security.Exchange property.
/// </summary>
/// <param name="security">The security being ordered</param>
/// <param name="order">The order to test for execution</param>
/// <returns>True if the brokerage would be able to perform the execution, false otherwise</returns>
public bool CanExecuteOrder(Security security, Order order)
{
return InvokeMethod<bool>(nameof(CanExecuteOrder), security, order);
}
/// <summary>
/// Returns true if the brokerage could accept this order. This takes into account
/// order type, security type, and order size limits.
/// </summary>
/// <remarks>
/// For example, a brokerage may have no connectivity at certain times, or an order rate/size limit
/// </remarks>
/// <param name="security">The security being ordered</param>
/// <param name="order">The order to be processed</param>
/// <param name="message">If this function returns false, a brokerage message detailing why the order may not be submitted</param>
/// <returns>True if the brokerage could process the order, false otherwise</returns>
public bool CanSubmitOrder(Security security, Order order, out BrokerageMessageEvent message)
{
message = null;
var result = InvokeMethodWithOutParameters<bool>(nameof(CanSubmitOrder), new[] { typeof(BrokerageMessageEvent) },
out var outParameters, security, order, message);
message = outParameters[0] as BrokerageMessageEvent;
return result;
}
/// <summary>
/// Returns true if the brokerage would allow updating the order as specified by the request
/// </summary>
/// <param name="security">The security of the order</param>
/// <param name="order">The order to be updated</param>
/// <param name="request">The requested updated to be made to the order</param>
/// <param name="message">If this function returns false, a brokerage message detailing why the order may not be updated</param>
/// <returns>True if the brokerage would allow updating the order, false otherwise</returns>
public bool CanUpdateOrder(Security security, Order order, UpdateOrderRequest request, out BrokerageMessageEvent message)
{
message = null;
var result = InvokeMethodWithOutParameters<bool>(nameof(CanUpdateOrder), new[] { typeof(BrokerageMessageEvent) }, out var outParameters,
security, order, request, message);
message = outParameters[0] as BrokerageMessageEvent;
return result;
}
/// <summary>
/// Get the benchmark for this model
/// </summary>
/// <param name="securities">SecurityService to create the security with if needed</param>
/// <returns>The benchmark for this brokerage</returns>
public IBenchmark GetBenchmark(SecurityManager securities)
{
return InvokeMethodAndWrapResult<IBenchmark>(nameof(GetBenchmark), (pyInstance) => new BenchmarkPythonWrapper(pyInstance), securities);
}
/// <summary>
/// Gets a new fee model that represents this brokerage's fee structure
/// </summary>
/// <param name="security">The security to get a fee model for</param>
/// <returns>The new fee model for this brokerage</returns>
public IFeeModel GetFeeModel(Security security)
{
return InvokeMethodAndWrapResult<IFeeModel>(nameof(GetFeeModel), (pyInstance) => new FeeModelPythonWrapper(pyInstance), security);
}
/// <summary>
/// Gets a new fill model that represents this brokerage's fill behavior
/// </summary>
/// <param name="security">The security to get fill model for</param>
/// <returns>The new fill model for this brokerage</returns>
public IFillModel GetFillModel(Security security)
{
return InvokeMethodAndWrapResult<IFillModel>(nameof(GetFillModel), (pyInstance) => new FillModelPythonWrapper(pyInstance), security);
}
/// <summary>
/// Gets the brokerage's leverage for the specified security
/// </summary>
/// <param name="security">The security's whose leverage we seek</param>
/// <returns>The leverage for the specified security</returns>
public decimal GetLeverage(Security security)
{
return InvokeMethod<decimal>(nameof(GetLeverage), security);
}
/// <summary>
/// Gets a new settlement model for the security
/// </summary>
/// <param name="security">The security to get a settlement model for</param>
/// <returns>The settlement model for this brokerage</returns>
public ISettlementModel GetSettlementModel(Security security)
{
return InvokeMethodAndWrapResult<ISettlementModel>(nameof(GetSettlementModel),
(pyInstance) => new SettlementModelPythonWrapper(pyInstance), security);
}
/// <summary>
/// Gets a new settlement model for the security
/// </summary>
/// <param name="security">The security to get a settlement model for</param>
/// <param name="accountType">The account type</param>
/// <returns>The settlement model for this brokerage</returns>
[Obsolete("Flagged deprecated and will remove December 1st 2018")]
public ISettlementModel GetSettlementModel(Security security, AccountType accountType)
{
return InvokeMethod<ISettlementModel>(nameof(GetSettlementModel), security, accountType);
}
/// <summary>
/// Gets a new slippage model that represents this brokerage's fill slippage behavior
/// </summary>
/// <param name="security">The security to get a slippage model for</param>
/// <returns>The new slippage model for this brokerage</returns>
public ISlippageModel GetSlippageModel(Security security)
{
return InvokeMethodAndWrapResult<ISlippageModel>(nameof(GetSlippageModel),
(pyInstance) => new SlippageModelPythonWrapper(pyInstance), security);
}
/// <summary>
/// Determine if this symbol is shortable
/// </summary>
/// <param name="algorithm">The algorithm running</param>
/// <param name="symbol">The symbol to short</param>
/// <param name="quantity">The amount to short</param>
/// <returns></returns>
public bool Shortable(IAlgorithm algorithm, Symbol symbol, decimal quantity)
{
return InvokeMethod<bool>(nameof(Shortable), algorithm, symbol, quantity);
}
/// <summary>
/// Gets a new buying power model for the security, returning the default model with the security's configured leverage.
/// For cash accounts, leverage = 1 is used.
/// </summary>
/// <param name="security">The security to get a buying power model for</param>
/// <returns>The buying power model for this brokerage/security</returns>
public IBuyingPowerModel GetBuyingPowerModel(Security security)
{
return InvokeMethodAndWrapResult<IBuyingPowerModel>(nameof(GetBuyingPowerModel),
(pyInstance) => new BuyingPowerModelPythonWrapper(pyInstance), security);
}
/// <summary>
/// Gets a new buying power model for the security
/// </summary>
/// <param name="security">The security to get a buying power model for</param>
/// <param name="accountType">The account type</param>
/// <returns>The buying power model for this brokerage/security</returns>
[Obsolete("Flagged deprecated and will remove December 1st 2018")]
public IBuyingPowerModel GetBuyingPowerModel(Security security, AccountType accountType)
{
return InvokeMethod<IBuyingPowerModel>(nameof(GetBuyingPowerModel), security, accountType);
}
/// <summary>
/// Gets the shortable provider
/// </summary>
/// <returns>Shortable provider</returns>
public IShortableProvider GetShortableProvider(Security security)
{
return InvokeMethodAndWrapResult<IShortableProvider>(nameof(GetShortableProvider),
(pyInstance) => new ShortableProviderPythonWrapper(pyInstance), security);
}
/// <summary>
/// Convenience method to get the underlying <see cref="IBrokerageModel"/> object from the wrapper.
/// </summary>
/// <returns>Underlying <see cref="IBrokerageModel"/> object</returns>
public IBrokerageModel GetModel()
{
using (Py.GIL())
{
return Instance.As<IBrokerageModel>();
}
}
/// <summary>
/// Gets a new margin interest rate model for the security
/// </summary>
/// <param name="security">The security to get a margin interest rate model for</param>
/// <returns>The margin interest rate model for this brokerage</returns>
public IMarginInterestRateModel GetMarginInterestRateModel(Security security)
{
return InvokeMethodAndWrapResult<IMarginInterestRateModel>(nameof(GetMarginInterestRateModel),
(pyInstance) => new MarginInterestRateModelPythonWrapper(pyInstance), security);
}
}
}
@@ -0,0 +1,141 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Securities;
namespace QuantConnect.Python
{
/// <summary>
/// Wraps a <see cref="PyObject"/> object that represents a security's model of buying power
/// </summary>
public class BuyingPowerModelPythonWrapper : BasePythonWrapper<IBuyingPowerModel>, IBuyingPowerModel
{
/// <summary>
/// Constructor for initializing the <see cref="BuyingPowerModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Represents a security's model of buying power</param>
public BuyingPowerModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Gets the buying power available for a trade
/// </summary>
/// <param name="parameters">A parameters object containing the algorithm's potrfolio, security, and order direction</param>
/// <returns>The buying power available for the trade</returns>
public BuyingPower GetBuyingPower(BuyingPowerParameters parameters)
{
return InvokeMethod<BuyingPower>(nameof(GetBuyingPower), parameters);
}
/// <summary>
/// Gets the current leverage of the security
/// </summary>
/// <param name="security">The security to get leverage for</param>
/// <returns>The current leverage in the security</returns>
public decimal GetLeverage(Security security)
{
return InvokeMethod<decimal>(nameof(GetLeverage), security);
}
/// <summary>
/// Get the maximum market order quantity to obtain a position with a given buying power percentage.
/// Will not take into account free buying power.
/// </summary>
/// <param name="parameters">An object containing the portfolio, the security and the target signed buying power percentage</param>
/// <returns>Returns the maximum allowed market order quantity and if zero, also the reason</returns>
public GetMaximumOrderQuantityResult GetMaximumOrderQuantityForTargetBuyingPower(GetMaximumOrderQuantityForTargetBuyingPowerParameters parameters)
{
return InvokeMethod<GetMaximumOrderQuantityResult>(nameof(GetMaximumOrderQuantityForTargetBuyingPower), parameters);
}
/// <summary>
/// Get the maximum market order quantity to obtain a delta in the buying power used by a security.
/// The deltas sign defines the position side to apply it to, positive long, negative short.
/// </summary>
/// <param name="parameters">An object containing the portfolio, the security and the delta buying power</param>
/// <returns>Returns the maximum allowed market order quantity and if zero, also the reason</returns>
public GetMaximumOrderQuantityResult GetMaximumOrderQuantityForDeltaBuyingPower(
GetMaximumOrderQuantityForDeltaBuyingPowerParameters parameters)
{
return InvokeMethod<GetMaximumOrderQuantityResult>(nameof(GetMaximumOrderQuantityForDeltaBuyingPower), parameters);
}
/// <summary>
/// Gets the amount of buying power reserved to maintain the specified position
/// </summary>
/// <param name="parameters">A parameters object containing the security</param>
/// <returns>The reserved buying power in account currency</returns>
public ReservedBuyingPowerForPosition GetReservedBuyingPowerForPosition(ReservedBuyingPowerForPositionParameters parameters)
{
return InvokeMethod<ReservedBuyingPowerForPosition>(nameof(GetReservedBuyingPowerForPosition), parameters);
}
/// <summary>
/// Check if there is sufficient buying power to execute this order.
/// </summary>
/// <param name="parameters">An object containing the portfolio, the security and the order</param>
/// <returns>Returns buying power information for an order</returns>
public HasSufficientBuyingPowerForOrderResult HasSufficientBuyingPowerForOrder(HasSufficientBuyingPowerForOrderParameters parameters)
{
return InvokeMethod<HasSufficientBuyingPowerForOrderResult>(nameof(HasSufficientBuyingPowerForOrder), parameters);
}
/// <summary>
/// Sets the leverage for the applicable securities, i.e, equities
/// </summary>
/// <remarks>
/// This is added to maintain backwards compatibility with the old margin/leverage system
/// </remarks>
/// <param name="security">The security to set leverage for</param>
/// <param name="leverage">The new leverage</param>
public void SetLeverage(Security security, decimal leverage)
{
InvokeMethod(nameof(SetLeverage), security, leverage);
}
/// <summary>
/// Gets the margin currently allocated to the specified holding
/// </summary>
/// <param name="parameters">An object containing the security</param>
/// <returns>The maintenance margin required for the provided holdings quantity/cost/value</returns>
public MaintenanceMargin GetMaintenanceMargin(MaintenanceMarginParameters parameters)
{
return InvokeMethod<MaintenanceMargin>(nameof(GetMaintenanceMargin), parameters);
}
/// <summary>
/// The margin that must be held in order to increase the position by the provided quantity
/// </summary>
/// <param name="parameters">An object containing the security and quantity</param>
/// <returns>The initial margin required for the provided security and quantity</returns>
public InitialMargin GetInitialMarginRequirement(InitialMarginParameters parameters)
{
return InvokeMethod<InitialMargin>(nameof(GetInitialMarginRequirement), parameters);
}
/// <summary>
/// Gets the total margin required to execute the specified order in units of the account currency including fees
/// </summary>
/// <param name="parameters">An object containing the portfolio, the security and the order</param>
/// <returns>The total margin in terms of the currency quoted in the order</returns>
public InitialMargin GetInitialMarginRequiredForOrder(InitialMarginRequiredForOrderParameters parameters)
{
return InvokeMethod<InitialMargin>(nameof(GetInitialMarginRequiredForOrder), parameters);
}
}
}
+122
View File
@@ -0,0 +1,122 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using QuantConnect.Commands;
using QuantConnect.Interfaces;
using System.Collections.Generic;
namespace QuantConnect.Python
{
/// <summary>
/// Python wrapper for a python defined command type
/// </summary>
public class CommandPythonWrapper : BasePythonWrapper<Command>
{
private static PyObject _linkSerializationMethod;
/// <summary>
/// Constructor for initialising the <see cref="CommandPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="type">Python command type</param>
/// <param name="data">Command data</param>
public CommandPythonWrapper(PyObject type, string data = null)
: base()
{
using var _ = Py.GIL();
var instance = type.Invoke();
SetPythonInstance(instance);
if (!string.IsNullOrEmpty(data))
{
if (HasAttr("PayloadData"))
{
SetProperty("PayloadData", data);
}
foreach (var kvp in JsonConvert.DeserializeObject<Dictionary<string, object>>(data))
{
if (kvp.Value is JArray jArray)
{
SetProperty(kvp.Key, jArray.ToObject<List<object>>());
}
else if (kvp.Value is JObject jobject)
{
SetProperty(kvp.Key, jobject.ToObject<Dictionary<string, object>>());
}
else
{
SetProperty(kvp.Key, kvp.Value);
}
}
}
}
/// <summary>
/// Run this command using the target algorithm
/// </summary>
/// <param name="algorithm">The algorithm instance</param>
/// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
public bool? Run(IAlgorithm algorithm)
{
using var _ = Py.GIL();
var result = InvokeMethod(nameof(Run), algorithm);
return result.GetAndDispose<bool?>();
}
/// <summary>
/// Helper method to serialize a command instance
/// </summary>
public static string Serialize(PyObject command)
{
if (command == null)
{
return string.Empty;
}
using var _ = Py.GIL();
if (_linkSerializationMethod == null)
{
var module = PyModule.FromString("python_serialization", @"from json import dumps
from inspect import getmembers
def serialize(target):
if isinstance(target, dict):
# dictionary
return dumps(target)
if not hasattr(target, '__dict__') or not target.__dict__:
# python command inheriting base Command
members = getmembers(target)
result = {}
for name, value in members:
if value and not name.startswith('__'):
potential_entry = str(value)
if not potential_entry.startswith('<bound '):
result[name] = value
return dumps(result)
# pure python command object
return dumps(target.__dict__)
");
_linkSerializationMethod = module.GetAttr("serialize");
}
using var strResult = _linkSerializationMethod.Invoke(command);
return strResult.As<string>();
}
}
}
@@ -0,0 +1,140 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using Python.Runtime;
using QuantConnect.Data;
using QuantConnect.Data.Consolidators;
namespace QuantConnect.Python
{
/// <summary>
/// Provides an Data Consolidator that wraps a <see cref="PyObject"/> object that represents a custom Python consolidator
/// </summary>
public class DataConsolidatorPythonWrapper : ConsolidatorBase
{
private readonly BasePythonWrapper<IDataConsolidator> _pythonWrapper;
private readonly DataConsolidatedHandler _pythonDataConsolidated;
private bool _disposed;
/// <summary>
/// Gets a clone of the data being currently consolidated
/// </summary>
public override IBaseData WorkingData
{
get { return _pythonWrapper.GetProperty<IBaseData>(nameof(WorkingData)); }
}
/// <summary>
/// Gets the type consumed by this consolidator
/// </summary>
public override Type InputType
{
get { return _pythonWrapper.GetProperty<Type>(nameof(InputType)); }
}
/// <summary>
/// Gets the type produced by this consolidator
/// </summary>
public override Type OutputType
{
get { return _pythonWrapper.GetProperty<Type>(nameof(OutputType)); }
}
/// <summary>
/// Constructor for initialising the <see cref="DataConsolidatorPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="consolidator">Represents a custom python consolidator</param>
public DataConsolidatorPythonWrapper(PyObject consolidator)
{
_pythonWrapper = new BasePythonWrapper<IDataConsolidator>(consolidator, true);
_pythonDataConsolidated = (_, bar) => OnDataConsolidated(bar);
// GetEvent releases the GIL before returning, so the event subscription must
// hold it explicitly, otherwise pythonnet mutates the Python object without the GIL
using (Py.GIL())
{
dynamic pythonEvent = _pythonWrapper.GetEvent("DataConsolidated");
pythonEvent += _pythonDataConsolidated;
}
}
/// <summary>
/// Scans this consolidator to see if it should emit a bar due to time passing
/// </summary>
/// <param name="currentLocalTime">The current time in the local time zone (same as <see cref="BaseData.Time"/>)</param>
public override void Scan(DateTime currentLocalTime)
{
_pythonWrapper.InvokeMethod(nameof(Scan), currentLocalTime);
}
/// <summary>
/// Updates this consolidator with the specified data
/// </summary>
/// <param name="data">The new data for the consolidator</param>
public override void Update(IBaseData data)
{
_pythonWrapper.InvokeMethod(nameof(Update), data);
}
/// <summary>
/// Resets the consolidator
/// </summary>
public override void Reset()
{
_pythonWrapper.InvokeMethod(nameof(Reset));
base.Reset();
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
public override void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
// Dispose can run from engine threads that do not hold the GIL, so acquire it before
// unsubscribing, otherwise the pythonnet event mutation can crash the runtime
using (Py.GIL())
{
dynamic pythonEvent = _pythonWrapper.GetEvent("DataConsolidated");
pythonEvent -= _pythonDataConsolidated;
}
_pythonWrapper.Dispose();
base.Dispose();
}
/// <summary>
/// Two wrappers are equal if they wrap the same Python object reference.
/// </summary>
public override bool Equals(object obj)
{
if (obj is DataConsolidatorPythonWrapper other)
{
return _pythonWrapper.Equals(other._pythonWrapper);
}
return _pythonWrapper.Equals(obj);
}
/// <summary>
/// Hash code based on the underlying Python object reference.
/// </summary>
public override int GetHashCode() => _pythonWrapper.GetHashCode();
}
}
@@ -0,0 +1,73 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using Python.Runtime;
using QuantConnect.Data;
using QuantConnect.Util;
namespace QuantConnect.Python
{
/// <summary>
/// Wraps a <see cref="PyObject"/> object that represents a dividend yield model
/// </summary>
public class DividendYieldModelPythonWrapper : BasePythonWrapper<IDividendYieldModel>, IDividendYieldModel
{
/// <summary>
/// Constructor for initializing the <see cref="DividendYieldModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Represents a security's model of dividend yield</param>
public DividendYieldModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Get dividend yield by a given date of a given symbol
/// </summary>
/// <param name="date">The date</param>
/// <returns>Dividend yield on the given date of the given symbol</returns>
public decimal GetDividendYield(DateTime date)
{
return InvokeMethod<decimal>(nameof(GetDividendYield), date);
}
/// <summary>
/// Get dividend yield at given date and security price
/// </summary>
/// <param name="date">The date</param>
/// <param name="securityPrice">The security price at the given date</param>
/// <returns>Dividend yield on the given date of the given symbol</returns>
/// <remarks>Price data must be raw (<see cref="DataNormalizationMode.Raw"/>)</remarks>
public decimal GetDividendYield(DateTime date, decimal securityPrice)
{
return InvokeMethod<decimal>(nameof(GetDividendYield), date, securityPrice);
}
/// <summary>
/// Converts a <see cref="PyObject"/> object into a <see cref="IDividendYieldModel"/> object, wrapping it if necessary
/// </summary>
/// <param name="model">The Python model</param>
/// <returns>The converted <see cref="IDividendYieldModel"/> instance</returns>
public static IDividendYieldModel FromPyObject(PyObject model)
{
var dividendYieldModel = PythonUtil.CreateInstanceOrWrapper<IDividendYieldModel>(
model,
py => new DividendYieldModelPythonWrapper(py)
);
return dividendYieldModel;
}
}
}
+65
View File
@@ -0,0 +1,65 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Orders.Fees;
using QuantConnect.Securities;
namespace QuantConnect.Python
{
/// <summary>
/// Provides an order fee model that wraps a <see cref="PyObject"/> object that represents a model that simulates order fees
/// </summary>
public class FeeModelPythonWrapper : FeeModel
{
private readonly BasePythonWrapper<FeeModel> _model;
private bool _extendedVersion = true;
/// <summary>
/// Constructor for initialising the <see cref="FeeModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Represents a model that simulates order fees</param>
public FeeModelPythonWrapper(PyObject model)
{
_model = new BasePythonWrapper<FeeModel>(model, false);
}
/// <summary>
/// Get the fee for this order
/// </summary>
/// <param name="parameters">A <see cref="OrderFeeParameters"/> object
/// containing the security and order</param>
/// <returns>The cost of the order in units of the account currency</returns>
public override OrderFee GetOrderFee(OrderFeeParameters parameters)
{
using (Py.GIL())
{
if (_extendedVersion)
{
try
{
return _model.InvokeMethod<OrderFee>(nameof(GetOrderFee), parameters);
}
catch (PythonException)
{
_extendedVersion = false;
}
}
var fee = _model.InvokeMethod<decimal>(nameof(GetOrderFee), parameters.Security, parameters.Order);
return new OrderFee(new CashAmount(fee, "USD"));
}
}
}
}
+197
View File
@@ -0,0 +1,197 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Orders;
using QuantConnect.Orders.Fills;
using QuantConnect.Securities;
using System.Collections.Generic;
namespace QuantConnect.Python
{
/// <summary>
/// Wraps a <see cref="PyObject"/> object that represents a model that simulates order fill events
/// </summary>
public class FillModelPythonWrapper : FillModel
{
private readonly BasePythonWrapper<FillModel> _model;
/// <summary>
/// Constructor for initialising the <see cref="FillModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Represents a model that simulates order fill events</param>
public FillModelPythonWrapper(PyObject model)
{
_model = new BasePythonWrapper<FillModel>(model, false);
using (Py.GIL())
{
(model as dynamic).SetPythonWrapper(this);
}
}
/// <summary>
/// Return an order event with the fill details
/// </summary>
/// <param name="parameters">A parameters object containing the security and order</param>
/// <returns>Order fill information detailing the average price and quantity filled.</returns>
public override Fill Fill(FillModelParameters parameters)
{
Parameters = parameters;
return _model.InvokeMethod<Fill>(nameof(Fill), parameters);
}
/// <summary>
/// Limit Fill Model. Return an order event with the fill details.
/// </summary>
/// <param name="asset">Stock Object to use to help model limit fill</param>
/// <param name="order">Order to fill. Alter the values directly if filled.</param>
/// <returns>Order fill information detailing the average price and quantity filled.</returns>
public override OrderEvent LimitFill(Security asset, LimitOrder order)
{
return _model.InvokeMethod<OrderEvent>(nameof(LimitFill), asset, order);
}
/// <summary>
/// Limit if Touched Fill Model. Return an order event with the fill details.
/// </summary>
/// <param name="asset">Asset we're trading this order</param>
/// <param name="order"><see cref="LimitIfTouchedOrder"/> Order to Check, return filled if true</param>
/// <returns>Order fill information detailing the average price and quantity filled.</returns>
public override OrderEvent LimitIfTouchedFill(Security asset, LimitIfTouchedOrder order)
{
return _model.InvokeMethod<OrderEvent>(nameof(LimitIfTouchedFill), asset, order);
}
/// <summary>
/// Model the slippage on a market order: fixed percentage of order price
/// </summary>
/// <param name="asset">Asset we're trading this order</param>
/// <param name="order">Order to update</param>
/// <returns>Order fill information detailing the average price and quantity filled.</returns>
public override OrderEvent MarketFill(Security asset, MarketOrder order)
{
return _model.InvokeMethod<OrderEvent>(nameof(MarketFill), asset, order);
}
/// <summary>
/// Market on Close Fill Model. Return an order event with the fill details
/// </summary>
/// <param name="asset">Asset we're trading with this order</param>
/// <param name="order">Order to be filled</param>
/// <returns>Order fill information detailing the average price and quantity filled.</returns>
public override OrderEvent MarketOnCloseFill(Security asset, MarketOnCloseOrder order)
{
return _model.InvokeMethod<OrderEvent>(nameof(MarketOnCloseFill), asset, order);
}
/// <summary>
/// Market on Open Fill Model. Return an order event with the fill details
/// </summary>
/// <param name="asset">Asset we're trading with this order</param>
/// <param name="order">Order to be filled</param>
/// <returns>Order fill information detailing the average price and quantity filled.</returns>
public override OrderEvent MarketOnOpenFill(Security asset, MarketOnOpenOrder order)
{
return _model.InvokeMethod<OrderEvent>(nameof(MarketOnOpenFill), asset, order);
}
/// <summary>
/// Stop Limit Fill Model. Return an order event with the fill details.
/// </summary>
/// <param name="asset">Asset we're trading this order</param>
/// <param name="order">Stop Limit Order to Check, return filled if true</param>
/// <returns>Order fill information detailing the average price and quantity filled.</returns>
public override OrderEvent StopLimitFill(Security asset, StopLimitOrder order)
{
return _model.InvokeMethod<OrderEvent>(nameof(StopLimitFill), asset, order);
}
/// <summary>
/// Stop Market Fill Model. Return an order event with the fill details.
/// </summary>
/// <param name="asset">Asset we're trading this order</param>
/// <param name="order">Trailing Stop Order to check, return filled if true</param>
/// <returns>Order fill information detailing the average price and quantity filled.</returns>
public override OrderEvent StopMarketFill(Security asset, StopMarketOrder order)
{
return _model.InvokeMethod<OrderEvent>(nameof(StopMarketFill), asset, order);
}
/// <summary>
/// Trailing Stop Fill Model. Return an order event with the fill details.
/// </summary>
/// <param name="asset">Asset we're trading this order</param>
/// <param name="order">Stop Order to Check, return filled if true</param>
/// <returns>Order fill information detailing the average price and quantity filled.</returns>
public override OrderEvent TrailingStopFill(Security asset, TrailingStopOrder order)
{
return _model.InvokeMethod<OrderEvent>(nameof(TrailingStopFill), asset, order);
}
/// <summary>
/// Default combo market fill model for the base security class. Fills at the last traded price for each leg.
/// </summary>
/// <param name="order">Order to fill</param>
/// <param name="parameters">Fill parameters for the order</param>
/// <returns>Order fill information detailing the average price and quantity filled for each leg. If any of the fills fails, none of the orders will be filled and the returned list will be empty</returns>
public override List<OrderEvent> ComboMarketFill(Order order, FillModelParameters parameters)
{
return _model.InvokeMethod<List<OrderEvent>>(nameof(ComboMarketFill), order, parameters);
}
/// <summary>
/// Default combo limit fill model for the base security class. Fills at the sum of prices for the assets of every leg.
/// </summary>
/// <param name="order">Order to fill</param>
/// <param name="parameters">Fill parameters for the order</param>
/// <returns>Order fill information detailing the average price and quantity filled for each leg. If any of the fills fails, none of the orders will be filled and the returned list will be empty</returns>
public override List<OrderEvent> ComboLimitFill(Order order, FillModelParameters parameters)
{
return _model.InvokeMethod<List<OrderEvent>>(nameof(ComboLimitFill), order, parameters);
}
/// <summary>
/// Default combo limit fill model for the base security class. Fills at the limit price for each leg
/// </summary>
/// <param name="order">Order to fill</param>
/// <param name="parameters">Fill parameters for the order</param>
/// <returns>Order fill information detailing the average price and quantity filled for each leg. If any of the fills fails, none of the orders will be filled and the returned list will be empty</returns>
public override List<OrderEvent> ComboLegLimitFill(Order order, FillModelParameters parameters)
{
return _model.InvokeMethod<List<OrderEvent>>(nameof(ComboLegLimitFill), order, parameters);
}
/// <summary>
/// Get the minimum and maximum price for this security in the last bar:
/// </summary>
/// <param name="asset">Security asset we're checking</param>
/// <param name="direction">The order direction, decides whether to pick bid or ask</param>
protected override Prices GetPrices(Security asset, OrderDirection direction)
{
return _model.InvokeMethod<Prices>(nameof(GetPrices), asset, direction);
}
/// <summary>
/// Get the minimum and maximum price for this security in the last bar:
/// </summary>
/// <param name="asset">Security asset we're checking</param>
/// <param name="direction">The order direction, decides whether to pick bid or ask</param>
/// <remarks>This method was implemented temporarily to help the refactoring of fill models (GH #4567)</remarks>
internal Prices GetPricesInternal(Security asset, OrderDirection direction)
{
return GetPrices(asset, direction);
}
}
}
@@ -0,0 +1,66 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Orders;
using QuantConnect.Securities;
using System;
using System.Collections.Generic;
using System.Linq;
namespace QuantConnect.Python
{
/// <summary>
/// Provides a margin call model that wraps a <see cref="PyObject"/> object that represents the model responsible for picking which orders should be executed during a margin call
/// </summary>
public class MarginCallModelPythonWrapper : BasePythonWrapper<IMarginCallModel>, IMarginCallModel
{
/// <summary>
/// Constructor for initialising the <see cref="MarginCallModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Represents the model responsible for picking which orders should be executed during a margin call</param>
public MarginCallModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Executes synchronous orders to bring the account within margin requirements.
/// </summary>
/// <param name="generatedMarginCallOrders">These are the margin call orders that were generated
/// by individual security margin models.</param>
/// <returns>The list of orders that were actually executed</returns>
public List<OrderTicket> ExecuteMarginCall(IEnumerable<SubmitOrderRequest> generatedMarginCallOrders)
{
return InvokeMethod<List<OrderTicket>>(nameof(ExecuteMarginCall), generatedMarginCallOrders);
}
/// <summary>
/// Scan the portfolio and the updated data for a potential margin call situation which may get the holdings below zero!
/// If there is a margin call, liquidate the portfolio immediately before the portfolio gets sub zero.
/// </summary>
/// <param name="issueMarginCallWarning">Set to true if a warning should be issued to the algorithm</param>
/// <returns>True for a margin call on the holdings.</returns>
public List<SubmitOrderRequest> GetMarginCallOrders(out bool issueMarginCallWarning)
{
issueMarginCallWarning = false;
var requests = InvokeMethodWithOutParameters<List<SubmitOrderRequest>>(nameof(GetMarginCallOrders), new[] { typeof(bool) },
out var outParameters, issueMarginCallWarning);
issueMarginCallWarning = (bool)outParameters[0] || requests.Count > 0;
return requests;
}
}
}
@@ -0,0 +1,44 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Securities;
namespace QuantConnect.Python
{
/// <summary>
/// Wraps a <see cref="PyObject"/> object that represents a security's margin interest rate model
/// </summary>
public class MarginInterestRateModelPythonWrapper : BasePythonWrapper<IMarginInterestRateModel>, IMarginInterestRateModel
{
/// <summary>
/// Constructor for initializing the <see cref="MarginInterestRateModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Represents a security's model of buying power</param>
public MarginInterestRateModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Apply margin interest rates to the portfolio
/// </summary>
/// <param name="marginInterestRateParameters">The parameters to use</param>
public void ApplyMarginInterestRate(MarginInterestRateParameters marginInterestRateParameters)
{
InvokeMethod(nameof(ApplyMarginInterestRate), marginInterestRateParameters);
}
}
}
@@ -0,0 +1,45 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Securities.Option;
namespace QuantConnect.Python
{
/// <summary>
/// Python wrapper for custom option assignment models
/// </summary>
public class OptionAssignmentModelPythonWrapper : BasePythonWrapper<IOptionAssignmentModel>, IOptionAssignmentModel
{
/// <summary>
/// Creates a new instance
/// </summary>
/// <param name="model">The python model to wrapp</param>
public OptionAssignmentModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Get's the option assignments to generate if any
/// </summary>
/// <param name="parameters">The option assignment parameters data transfer class</param>
/// <returns>The option assignment result</returns>
public OptionAssignmentResult GetAssignment(OptionAssignmentParameters parameters)
{
return InvokeMethod<OptionAssignmentResult>(nameof(GetAssignment), parameters);
}
}
}
@@ -0,0 +1,49 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using Python.Runtime;
using QuantConnect.Python;
using System;
namespace QuantConnect.Securities.Option
{
/// <summary>
/// Provides an implementation of <see cref="IOptionPriceModel"/> that wraps a <see cref="PyObject"/> object
/// </summary>
public class OptionPriceModelPythonWrapper : BasePythonWrapper<IOptionPriceModel>, IOptionPriceModel
{
/// <summary>
/// Creates a new instance
/// </summary>
/// <param name="model">The python model to wrap</param>
public OptionPriceModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Evaluates the specified option contract to compute a theoretical price, IV and greeks
/// </summary>
/// <param name="parameters">A <see cref="OptionPriceModelParameters"/> object
/// containing the security, slice and contract</param>
/// <returns>An instance of <see cref="OptionPriceModelResult"/> containing the theoretical
/// price of the specified option contract</returns>
public OptionPriceModelResult Evaluate(OptionPriceModelParameters parameters)
{
return InvokeMethod<OptionPriceModelResult>(nameof(Evaluate), parameters);
}
}
}
+40
View File
@@ -0,0 +1,40 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
namespace QuantConnect.Python
{
/// <summary>
/// Attribute to rename a property or field when converting an instance to a pandas DataFrame row.
/// </summary>
[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field)]
public sealed class PandasColumnAttribute : Attribute
{
/// <summary>
/// The name of the column in the pandas DataFrame.
/// </summary>
public string Name { get; }
/// <summary>
/// Initializes a new instance of the <see cref="PandasColumnAttribute"/> class.
/// </summary>
/// <param name="name">The name of the column in the pandas DataFrame</param>
public PandasColumnAttribute(string name)
{
Name = name;
}
}
}
@@ -0,0 +1,326 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Data;
using QuantConnect.Data.Market;
using QuantConnect.Data.UniverseSelection;
using QuantConnect.Util;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
namespace QuantConnect.Python
{
public partial class PandasConverter
{
/// <summary>
/// Helper class to generate data frames from slices
/// </summary>
private class DataFrameGenerator
{
private static readonly string[] MultiBaseDataCollectionDataFrameNames = new[] { "collection_symbol", "time" };
private static readonly string[] MultiCanonicalSymbolsDataFrameNames = new[] { "canonical", "time" };
private static readonly string[] SingleBaseDataCollectionDataFrameNames = new[] { "time" };
private readonly Type _dataType;
private readonly bool _requestedTick;
private readonly bool _requestedQuoteBar;
private readonly bool _requestedTradeBar;
private readonly bool _timeAsColumn;
/// <summary>
/// PandasData instances for each symbol. Does not hold BaseDataCollection instances.
/// </summary>
private Dictionary<Symbol, PandasData> _pandasData;
private List<(Symbol Symbol, DateTime Time, IEnumerable<ISymbolProvider> Data)> _collections;
private int _maxLevels;
private bool _shouldUseSymbolOnlyIndex;
private readonly bool _flatten;
protected DataFrameGenerator(Type dataType = null, bool timeAsColumn = false, bool flatten = false)
{
_dataType = dataType;
// if no data type is requested we check all
_requestedTick = dataType == null || dataType == typeof(Tick) || dataType == typeof(OpenInterest);
_requestedTradeBar = dataType == null || dataType == typeof(TradeBar);
_requestedQuoteBar = dataType == null || dataType == typeof(QuoteBar);
_timeAsColumn = timeAsColumn;
_flatten = flatten;
}
public DataFrameGenerator(IEnumerable<Slice> slices, bool flatten = false, Type dataType = null)
: this(dataType, flatten: flatten)
{
AddData(slices);
}
/// <summary>
/// Extracts the data from the slices and prepares it for DataFrame generation.
/// If the slices contain BaseDataCollection instances, they are added to the collections list for proper handling.
/// For the rest of the data, PandasData instances are created for each symbol and the data is added to them for later processing.
/// </summary>
protected void AddData(IEnumerable<Slice> slices)
{
HashSet<SecurityIdentifier> addedData = null;
foreach (var slice in slices)
{
foreach (var data in slice.AllData)
{
if (_flatten && IsCollection(data.GetType()))
{
AddCollection(data.Symbol, data.EndTime, (data as IEnumerable).Cast<ISymbolProvider>());
continue;
}
var pandasData = GetPandasData(data);
if (pandasData.IsCustomData || (_requestedTick && data is Tick))
{
pandasData.Add(data);
}
else
{
if (!_requestedTradeBar && !_requestedQuoteBar && _dataType != null && data.GetType().IsAssignableTo(_dataType))
{
// support for auxiliary data history requests
pandasData.Add(data);
continue;
}
// we add both quote and trade bars for each symbol at the same time, because they share the row in the data frame else it will generate 2 rows per series
if (_requestedTradeBar && _requestedQuoteBar)
{
addedData ??= new();
if (!addedData.Add(data.Symbol.ID))
{
continue;
}
}
// the slice already has the data organized by symbol so let's take advantage of it using Bars/QuoteBars collections
QuoteBar quoteBar;
var tradeBar = _requestedTradeBar ? data as TradeBar : null;
if (tradeBar != null)
{
slice.QuoteBars.TryGetValue(tradeBar.Symbol, out quoteBar);
}
else
{
quoteBar = _requestedQuoteBar ? data as QuoteBar : null;
if (quoteBar != null)
{
slice.Bars.TryGetValue(quoteBar.Symbol, out tradeBar);
}
}
pandasData.Add(tradeBar, quoteBar);
}
}
addedData?.Clear();
}
}
/// <summary>
/// Adds a collection of data and prepares it for DataFrame generation.
/// If the collection holds BaseDataCollection instances, they are added to the collections list for proper handling.
/// For the rest of the data, PandasData instances are created for each symbol and the data is added to them for later processing.
/// </summary>
protected void AddData<T>(IEnumerable<T> data)
where T : ISymbolProvider
{
var type = typeof(T);
var isCollection = IsCollection(type);
if (_flatten && isCollection)
{
foreach (var collection in data)
{
var baseData = collection as BaseData;
var collectionData = collection as IEnumerable;
AddCollection(baseData.Symbol, baseData.EndTime, collectionData.Cast<ISymbolProvider>());
}
}
else
{
Symbol prevSymbol = null;
PandasData prevPandasData = null;
foreach (var item in data)
{
var pandasData = prevSymbol != null && item.Symbol == prevSymbol ? prevPandasData : GetPandasData(item);
pandasData.Add(item);
prevSymbol = item.Symbol;
prevPandasData = pandasData;
}
// Multiple symbols detected, use symbol only indexing for performance reasons
if (_pandasData != null && _pandasData.Count > 1)
{
_shouldUseSymbolOnlyIndex = true;
}
}
}
/// <summary>
/// Generates the data frame
/// </summary>
/// <param name="levels">The number of level the index should have. If not provided, it will be inferred from the data</param>
/// <param name="sort">Whether to sort the data frames on concatenation</param>
/// <param name="filterMissingValueColumns">Whether to filter missing values. See <see cref="PandasData.ToPandasDataFrame(int, bool)"/></param>
/// <param name="symbolOnlyIndex">Whether to assume the data has multiple symbols and also one data point per symbol.
/// This is used for performance purposes</param>
/// <param name="forceMultiValueSymbol">Useful when the data contains points for multiple symbols.
/// If false and <paramref name="symbolOnlyIndex"/> is true, it will assume there is a single point for each symbol,
/// and will apply performance improvements for the data frame generation.</param>
public PyObject GenerateDataFrame(int? levels = null, bool sort = true, bool filterMissingValueColumns = true,
bool symbolOnlyIndex = false, bool forceMultiValueSymbol = false)
{
using var _ = Py.GIL();
var pandasDataDataFrames = GetPandasDataDataFrames(levels, filterMissingValueColumns, symbolOnlyIndex, forceMultiValueSymbol).ToList();
var collectionsDataFrames = GetCollectionsDataFrames(symbolOnlyIndex, forceMultiValueSymbol).ToList();
try
{
if (collectionsDataFrames.Count == 0)
{
return ConcatDataFrames(pandasDataDataFrames, sort, dropna: true);
}
var dataFrames = collectionsDataFrames.Select(x => x.Item3).Concat(pandasDataDataFrames);
if (symbolOnlyIndex)
{
return ConcatDataFrames(dataFrames, sort, dropna: true);
}
else if (_collections.DistinctBy(x => x.Symbol).Count() > 1)
{
var keys = collectionsDataFrames
.Select(x => new object[] { x.Item1, x.Item2 })
.Concat(pandasDataDataFrames.Select(x => new object[] { x, DateTime.MinValue }));
var names = _collections.Any(x => x.Symbol.IsCanonical())
? MultiCanonicalSymbolsDataFrameNames
: MultiBaseDataCollectionDataFrameNames;
return ConcatDataFrames(dataFrames, keys, names, sort, dropna: true);
}
else
{
var keys = collectionsDataFrames
.Select(x => new object[] { x.Item2 })
.Concat(pandasDataDataFrames.Select(x => new object[] { DateTime.MinValue }));
return ConcatDataFrames(dataFrames, keys, SingleBaseDataCollectionDataFrameNames, sort, dropna: true);
}
}
finally
{
foreach (var df in pandasDataDataFrames.Concat(collectionsDataFrames.Select(x => x.Item3)))
{
df.Dispose();
}
}
}
/// <summary>
/// Creates the data frames for the data stored in the <see cref="_pandasData"/> dictionary
/// </summary>
private IEnumerable<PyObject> GetPandasDataDataFrames(int? levels, bool filterMissingValueColumns, bool symbolOnlyIndex, bool forceMultiValueSymbol)
{
if (_pandasData is null || _pandasData.Count == 0)
{
yield break;
}
if (!forceMultiValueSymbol && (symbolOnlyIndex || _shouldUseSymbolOnlyIndex))
{
yield return PandasData.ToPandasDataFrame(_pandasData.Values, skipTimesColumn: true);
yield break;
}
foreach (var data in _pandasData.Values)
{
yield return data.ToPandasDataFrame(levels ?? _maxLevels, filterMissingValueColumns);
}
}
/// <summary>
/// Generates the data frames for the base data collections
/// </summary>
private IEnumerable<(Symbol, DateTime, PyObject)> GetCollectionsDataFrames(bool symbolOnlyIndex, bool forceMultiValueSymbol)
{
if (_collections is null || _collections.Count == 0)
{
yield break;
}
foreach (var (symbol, time, data) in _collections.GroupBy(x => x.Symbol).SelectMany(x => x))
{
var generator = new DataFrameGenerator(_dataType, timeAsColumn: !symbolOnlyIndex, flatten: _flatten);
generator.AddData(data);
var dataFrame = generator.GenerateDataFrame(symbolOnlyIndex: symbolOnlyIndex, forceMultiValueSymbol: forceMultiValueSymbol);
yield return (symbol, time, dataFrame);
}
}
private PandasData GetPandasData(ISymbolProvider data)
{
_pandasData ??= new();
if (!_pandasData.TryGetValue(data.Symbol, out var pandasData))
{
pandasData = new PandasData(data, _timeAsColumn);
_pandasData[data.Symbol] = pandasData;
_maxLevels = Math.Max(_maxLevels, pandasData.Levels);
}
return pandasData;
}
private void AddCollection(Symbol symbol, DateTime time, IEnumerable<ISymbolProvider> data)
{
_collections ??= new();
_collections.Add((symbol, time, data));
}
/// <summary>
/// Determines whether the type is considered a collection for flattening.
/// Any object that is a <see cref="BaseData"/> and implements <see cref="IEnumerable{ISymbolProvider}"/>
/// is considered a base data collection.
/// This allows detecting collections of cases like <see cref="OptionUniverse"/> (which is a direct subclass of
/// <see cref="BaseDataCollection"/>) and <see cref="OptionChain"/>, which is a collection of <see cref="OptionContract"/>
/// </summary>
private static bool IsCollection(Type type)
{
return type.IsAssignableTo(typeof(BaseData)) &&
type.GetInterfaces().Any(x => x.IsGenericType &&
x.GetGenericTypeDefinition().IsAssignableTo(typeof(IEnumerable<>)) &&
x.GenericTypeArguments[0].IsAssignableTo(typeof(ISymbolProvider)));
}
}
private class DataFrameGenerator<T> : DataFrameGenerator
where T : ISymbolProvider
{
public DataFrameGenerator(IEnumerable<T> data, bool flatten)
: base(flatten: flatten)
{
AddData(data);
}
}
}
}
+329
View File
@@ -0,0 +1,329 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Data;
using QuantConnect.Indicators;
using QuantConnect.Util;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
namespace QuantConnect.Python
{
/// <summary>
/// Collection of methods that converts lists of objects in pandas.DataFrame
/// </summary>
public partial class PandasConverter
{
private static dynamic _pandas;
private static PyObject _concat;
/// <summary>
/// Initializes the <see cref="PandasConverter"/> class
/// </summary>
static PandasConverter()
{
using (Py.GIL())
{
var pandas = Py.Import("pandas");
_pandas = pandas;
// keep it so we don't need to ask for it each time
_concat = pandas.GetAttr("concat");
}
}
/// <summary>
/// Converts an enumerable of <see cref="Slice"/> in a pandas.DataFrame
/// </summary>
/// <param name="data">Enumerable of <see cref="Slice"/></param>
/// <param name="flatten">Whether to flatten collections into rows and columns</param>
/// <param name="dataType">Optional type of bars to add to the data frame
/// If true, the base data items time will be ignored and only the base data collection time will be used in the index</param>
/// <returns><see cref="PyObject"/> containing a pandas.DataFrame</returns>
public PyObject GetDataFrame(IEnumerable<Slice> data, bool flatten = false, Type dataType = null)
{
var generator = new DataFrameGenerator(data, flatten, dataType);
return generator.GenerateDataFrame();
}
/// <summary>
/// Converts an enumerable of <see cref="IBaseData"/> in a pandas.DataFrame
/// </summary>
/// <param name="data">Enumerable of <see cref="Slice"/></param>
/// <param name="symbolOnlyIndex">Whether to make the index only the symbol, without time or any other index levels</param>
/// <param name="forceMultiValueSymbol">Useful when the data contains points for multiple symbols.
/// If false and <paramref name="symbolOnlyIndex"/> is true, it will assume there is a single point for each symbol,
/// and will apply performance improvements for the data frame generation.</param>
/// <param name="flatten">Whether to flatten collections into rows and columns</param>
/// <returns><see cref="PyObject"/> containing a pandas.DataFrame</returns>
/// <remarks>Helper method for testing</remarks>
public PyObject GetDataFrame<T>(IEnumerable<T> data, bool symbolOnlyIndex = false, bool forceMultiValueSymbol = false, bool flatten = false)
where T : ISymbolProvider
{
var generator = new DataFrameGenerator<T>(data, flatten);
return generator.GenerateDataFrame(
// Use 2 instead of maxLevels for backwards compatibility
levels: symbolOnlyIndex ? 1 : 2,
sort: false,
symbolOnlyIndex: symbolOnlyIndex,
forceMultiValueSymbol: forceMultiValueSymbol);
}
/// <summary>
/// Converts a dictionary with a list of <see cref="IndicatorDataPoint"/> in a pandas.DataFrame
/// </summary>
/// <param name="data">Dictionary with a list of <see cref="IndicatorDataPoint"/></param>
/// <param name="extraData">Optional dynamic properties to include in the DataFrame.</param>
/// <returns><see cref="PyObject"/> containing a pandas.DataFrame</returns>
public PyObject GetIndicatorDataFrame(IEnumerable<KeyValuePair<string, List<IndicatorDataPoint>>> data, IEnumerable<KeyValuePair<string, List<(DateTime, object)>>> extraData = null)
{
using (Py.GIL())
{
using var pyDict = new PyDict();
foreach (var kvp in data)
{
AddSeriesToPyDict(kvp.Key, kvp.Value, pyDict);
}
if (extraData != null)
{
foreach (var kvp in extraData)
{
AddDynamicSeriesToPyDict(kvp.Key, kvp.Value, pyDict);
}
}
return MakeIndicatorDataFrame(pyDict);
}
}
/// <summary>
/// Converts a dictionary with a list of <see cref="IndicatorDataPoint"/> in a pandas.DataFrame
/// </summary>
/// <param name="data"><see cref="PyObject"/> that should be a dictionary (convertible to PyDict) of string to list of <see cref="IndicatorDataPoint"/></param>
/// <returns><see cref="PyObject"/> containing a pandas.DataFrame</returns>
public PyObject GetIndicatorDataFrame(PyObject data)
{
using (Py.GIL())
{
using var inputPythonType = data.GetPythonType();
var inputTypeStr = inputPythonType.ToString();
var targetTypeStr = nameof(PyDict);
PyObject currentKvp = null;
try
{
using var pyDictData = new PyDict(data);
using var seriesPyDict = new PyDict();
targetTypeStr = $"{nameof(String)}: {nameof(List<IndicatorDataPoint>)}";
foreach (var kvp in pyDictData.Items())
{
currentKvp = kvp;
AddSeriesToPyDict(kvp[0].As<string>(), kvp[1].As<List<IndicatorDataPoint>>(), seriesPyDict);
}
return MakeIndicatorDataFrame(seriesPyDict);
}
catch (Exception e)
{
if (currentKvp != null)
{
inputTypeStr = $"{currentKvp[0].GetPythonType()}: {currentKvp[1].GetPythonType()}";
}
throw new ArgumentException(Messages.PandasConverter.ConvertToDictionaryFailed(inputTypeStr, targetTypeStr, e.Message), e);
}
}
}
/// <summary>
/// Returns a string that represent the current object
/// </summary>
/// <returns></returns>
public override string ToString()
{
if (_pandas == null)
{
return Messages.PandasConverter.PandasModuleNotImported;
}
using (Py.GIL())
{
return _pandas.Repr();
}
}
/// <summary>
/// Concatenates multiple data frames
/// </summary>
/// <param name="dataFrames">The data frames to concatenate</param>
/// <param name="keys">
/// Optional new keys for a new multi-index level that would be added
/// to index each individual data frame in the resulting one
/// </param>
/// <param name="names">The optional names of the new index level (and the existing ones if they need to be changed)</param>
/// <param name="sort">Whether to sort the resulting data frame</param>
/// <param name="dropna">Whether to drop columns containing NA values only (Nan, None, etc)</param>
/// <returns>A new data frame result from concatenating the input</returns>
public static PyObject ConcatDataFrames<T>(IEnumerable<PyObject> dataFrames, IEnumerable<T> keys, IEnumerable<string> names,
bool sort = true, bool dropna = true)
{
using (Py.GIL())
{
using var pyDataFrames = dataFrames.ToPyListUnSafe();
if (pyDataFrames.Length() == 0)
{
return _pandas.DataFrame();
}
using var kwargs = Py.kw("sort", sort);
PyList pyKeys = null;
PyList pyNames = null;
try
{
if (keys != null && names != null)
{
pyNames = names.ToPyListUnSafe();
pyKeys = ConvertConcatKeys(keys);
using var pyFalse = false.ToPython();
kwargs.SetItem("keys", pyKeys);
kwargs.SetItem("names", pyNames);
kwargs.SetItem("copy", pyFalse);
}
var result = _concat.Invoke(new[] { pyDataFrames }, kwargs);
// Drop columns with only NaN or None values
if (dropna)
{
using var dropnaKwargs = Py.kw("axis", 1, "inplace", true, "how", "all");
result.GetAttr("dropna").Invoke(Array.Empty<PyObject>(), dropnaKwargs);
}
return result;
}
finally
{
pyKeys?.Dispose();
pyNames?.Dispose();
}
}
}
public static PyObject ConcatDataFrames(IEnumerable<PyObject> dataFrames, bool sort = true, bool dropna = true)
{
return ConcatDataFrames<string>(dataFrames, null, null, sort, dropna);
}
/// <summary>
/// Creates the list of keys required for the pd.concat method, making sure that if the items are enumerables,
/// they are converted to Python tuples so that they are used as levels for a multi index
/// </summary>
private static PyList ConvertConcatKeys(IEnumerable<IEnumerable<object>> keys)
{
var keyTuples = keys.Select(x => new PyTuple(x.Select(y => y.ToPython()).ToArray()));
try
{
return keyTuples.ToPyListUnSafe();
}
finally
{
foreach (var tuple in keyTuples)
{
foreach (var x in tuple)
{
x.DisposeSafely();
}
tuple.DisposeSafely();
}
}
}
private static PyList ConvertConcatKeys<T>(IEnumerable<T> keys)
{
if ((typeof(T).IsAssignableTo(typeof(IEnumerable)) && !typeof(T).IsAssignableTo(typeof(string))))
{
return ConvertConcatKeys(keys.Cast<IEnumerable<object>>());
}
return keys.ToPyListUnSafe();
}
/// <summary>
/// Creates a series from a list of <see cref="IndicatorDataPoint"/> and adds it to the
/// <see cref="PyDict"/> as the value of the given <paramref name="key"/>
/// </summary>
/// <param name="key">Key to insert in the <see cref="PyDict"/></param>
/// <param name="points">List of <see cref="IndicatorDataPoint"/> that will make up the resulting series</param>
/// <param name="pyDict"><see cref="PyDict"/> where the resulting key-value pair will be inserted into</param>
private void AddSeriesToPyDict(string key, List<IndicatorDataPoint> points, PyDict pyDict)
{
var index = new List<DateTime>();
var values = new List<double>();
foreach (var point in points)
{
if (point.EndTime != default)
{
index.Add(point.EndTime);
values.Add((double)point.Value);
}
}
pyDict.SetItem(key.ToLowerInvariant(), _pandas.Series(values, index));
}
/// <summary>
/// Builds a timeindexed pandas <see cref="Series"/> from a collection of
/// heterogeneous data (numbers, enums, strings, etc.) and inserts it into the
/// specified <see cref="PyDict"/> under the given <paramref name="key"/>.
/// </summary>
/// <param name="key">Key to insert in the <see cref="PyDict"/></param>
/// <param name="entries">A list of tuples whose first item is the timestamp and whose second item is the value associated with that timestamp.</param>
/// <param name="pyDict"><see cref="PyDict"/> where the resulting key-value pair will be inserted into</param>
private void AddDynamicSeriesToPyDict(string key, List<(DateTime Timestamp, object Value)> entries, PyDict pyDict)
{
var index = new List<DateTime>();
var values = new List<object>();
foreach (var (timestamp, value) in entries)
{
if (timestamp != default)
{
index.Add(timestamp);
values.Add(value is Enum e ? e.ToString() : value);
}
}
pyDict.SetItem(key.ToLowerInvariant(), _pandas.Series(values, index));
}
/// <summary>
/// Converts a <see cref="PyDict"/> of string to pandas.Series in a pandas.DataFrame
/// </summary>
/// <param name="pyDict"><see cref="PyDict"/> of string to pandas.Series</param>
/// <returns><see cref="PyObject"/> containing a pandas.DataFrame</returns>
private PyObject MakeIndicatorDataFrame(PyDict pyDict)
{
return _pandas.DataFrame(pyDict, columns: pyDict.Keys().Select(x => x.As<string>().ToLowerInvariant()).OrderBy(x => x));
}
}
}
+232
View File
@@ -0,0 +1,232 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Data.Market;
using System;
using System.Collections.Generic;
using System.Reflection;
using System.Text;
namespace QuantConnect.Python
{
public partial class PandasData
{
private static DataTypeMember CreateDataTypeMember(MemberInfo member, DataTypeMember[] children = null)
{
return member switch
{
PropertyInfo property => new PropertyMember(property, children),
FieldInfo field => new FieldMember(field, children),
_ => throw new ArgumentException($"Member type {member.MemberType} is not supported")
};
}
/// <summary>
/// Represents a member of a data type, either a property or a field and it's children members in case it's a complex type.
/// It contains logic to get the member name and the children names, taking into account the parent prefixes.
/// </summary>
private abstract class DataTypeMember
{
private static readonly StringBuilder _stringBuilder = new StringBuilder();
private DataTypeMember _parent;
private string _name;
public MemberInfo Member { get; }
public DataTypeMember[] Children { get; }
public abstract bool IsProperty { get; }
public abstract bool IsField { get; }
/// <summary>
/// The prefix to be used for the children members when a class being expanded has multiple properties/fields of the same type
/// </summary>
public string Prefix { get; private set; }
public bool ShouldBeUnwrapped => Children != null && Children.Length > 0;
/// <summary>
/// Whether this member is Tick.LastPrice or OpenInterest.LastPrice.
/// Saved to avoid MemberInfo comparisons in the future
/// </summary>
public bool IsTickLastPrice { get; }
public bool IsTickProperty { get; }
public DataTypeMember(MemberInfo member, DataTypeMember[] children = null)
{
Member = member;
Children = children;
IsTickLastPrice = member == _tickLastPriceMember || member == _openInterestLastPriceMember;
IsTickProperty = IsProperty && member.DeclaringType == typeof(Tick);
if (Children != null)
{
foreach (var child in Children)
{
child._parent = this;
}
}
}
public void SetPrefix()
{
Prefix = Member.Name.ToLowerInvariant();
}
/// <summary>
/// Gets the member name, adding the parent prefixes if necessary.
/// </summary>
/// <param name="customName">If passed, it will be used instead of the <see cref="Member"/>'s name</param>
public string GetMemberName(string customName = null)
{
if (ShouldBeUnwrapped)
{
return string.Empty;
}
if (!string.IsNullOrEmpty(customName))
{
return BuildMemberName(customName);
}
if (string.IsNullOrEmpty(_name))
{
_name = BuildMemberName(GetBaseName());
}
return _name;
}
public IEnumerable<string> GetMemberNames()
{
return GetMemberNames(null);
}
public abstract object GetValue(object instance);
public abstract Type GetMemberType();
public override string ToString()
{
return $"{GetMemberType().Name} {Member.Name}";
}
private string BuildMemberName(string baseName)
{
_stringBuilder.Clear();
while (_parent != null && _parent.ShouldBeUnwrapped)
{
_stringBuilder.Insert(0, _parent.Prefix);
_parent = _parent._parent;
}
_stringBuilder.Append(baseName.ToLowerInvariant());
return _stringBuilder.ToString();
}
private IEnumerable<string> GetMemberNames(string parentPrefix)
{
// If there are no children, return the name of the member. Else ignore the member and return the children names
if (ShouldBeUnwrapped)
{
var prefix = parentPrefix ?? string.Empty;
if (!string.IsNullOrEmpty(Prefix))
{
prefix += Prefix;
}
foreach (var child in Children)
{
foreach (var childName in child.GetMemberNames(prefix))
{
yield return childName;
}
}
yield break;
}
var memberName = GetBaseName();
_name = string.IsNullOrEmpty(parentPrefix) ? memberName : $"{parentPrefix}{memberName}";
yield return _name;
}
private string GetBaseName()
{
var baseName = Member.GetCustomAttribute<PandasColumnAttribute>()?.Name;
if (string.IsNullOrEmpty(baseName))
{
baseName = Member.Name;
}
return baseName.ToLowerInvariant();
}
}
private class PropertyMember : DataTypeMember
{
private PropertyInfo _property;
public override bool IsProperty => true;
public override bool IsField => false;
public PropertyMember(PropertyInfo property, DataTypeMember[] children = null)
: base(property, children)
{
_property = property;
}
public override object GetValue(object instance)
{
return _property.GetValue(instance);
}
public override Type GetMemberType()
{
return _property.PropertyType;
}
}
private class FieldMember : DataTypeMember
{
private FieldInfo _field;
public override bool IsProperty => false;
public override bool IsField => true;
public FieldMember(FieldInfo field, DataTypeMember[] children = null)
: base(field, children)
{
_field = field;
}
public override object GetValue(object instance)
{
return _field.GetValue(instance);
}
public override Type GetMemberType()
{
return _field.FieldType;
}
}
}
}
+753
View File
@@ -0,0 +1,753 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Data;
using QuantConnect.Data.Fundamental;
using QuantConnect.Data.Market;
using QuantConnect.Util;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Reflection;
namespace QuantConnect.Python
{
/// <summary>
/// Organizes a list of data to create pandas.DataFrames
/// </summary>
public partial class PandasData
{
// we keep these so we don't need to ask for them each time
private static PyString _empty;
private static PyObject _pandas;
private static PyObject _pandasColumn;
private static PyObject _seriesFactory;
private static PyObject _dataFrameFactory;
private static PyObject _multiIndexFactory;
private static PyObject _multiIndex;
private static PyObject _indexFactory;
private static PyList _defaultNames;
private static PyList _level1Names;
private static PyList _level2Names;
private static PyList _level3Names;
private readonly static Dictionary<Type, List<DataTypeMember>> _membersCache = new();
private readonly static MemberInfo _tickLastPriceMember = typeof(Tick).GetProperty(nameof(Tick.LastPrice));
private readonly static MemberInfo _openInterestLastPriceMember = typeof(OpenInterest).GetProperty(nameof(Tick.LastPrice));
private static readonly string[] _nonLeanDataTypeForcedMemberNames = new[] { nameof(BaseData.Value) };
private readonly static string[] _quoteTickOnlyPropertes = new[] {
nameof(Tick.AskPrice),
nameof(Tick.AskSize),
nameof(Tick.BidPrice),
nameof(Tick.BidSize)
};
private static readonly Type PandasNonExpandableAttribute = typeof(PandasNonExpandableAttribute);
private static readonly Type PandasIgnoreAttribute = typeof(PandasIgnoreAttribute);
private static readonly Type PandasIgnoreMembersAttribute = typeof(PandasIgnoreMembersAttribute);
private static readonly IReadOnlyCollection<DateTime> EmptySeriesTimesKey = new List<DateTime>();
private static readonly List<DataTypeMember> EmptyDataTypeMembers = new List<DataTypeMember>();
private readonly Symbol _symbol;
private readonly bool _isFundamentalType;
private readonly bool _isBaseData;
private readonly bool _timeAsColumn;
private readonly Dictionary<string, Serie> _series;
private readonly Dictionary<Type, List<DataTypeMember>> _members = new();
/// <summary>
/// Gets true if this is a custom data request, false for normal QC data
/// </summary>
public bool IsCustomData { get; }
/// <summary>
/// Implied levels of a multi index pandas.Series (depends on the security type)
/// </summary>
public int Levels { get; } = 2;
/// <summary>
/// Initializes the static members of the <see cref="PandasData"/> class
/// </summary>
static PandasData()
{
using (Py.GIL())
{
// Use our PandasMapper class that modifies pandas indexing to support tickers, symbols and SIDs
_pandas = Py.Import("PandasMapper");
_pandasColumn = _pandas.GetAttr("PandasColumn");
_seriesFactory = _pandas.GetAttr("Series");
_dataFrameFactory = _pandas.GetAttr("DataFrame");
_multiIndex = _pandas.GetAttr("MultiIndex");
_multiIndexFactory = _multiIndex.GetAttr("from_tuples");
_indexFactory = _pandas.GetAttr("Index");
_empty = new PyString(string.Empty);
var time = new PyString("time");
var symbol = new PyString("symbol");
var expiry = new PyString("expiry");
_defaultNames = new PyList(new PyObject[] { expiry, new PyString("strike"), new PyString("type"), symbol, time });
_level1Names = new PyList(new PyObject[] { symbol });
_level2Names = new PyList(new PyObject[] { symbol, time });
_level3Names = new PyList(new PyObject[] { expiry, symbol, time });
}
}
/// <summary>
/// Initializes an instance of <see cref="PandasData"/>
/// </summary>
public PandasData(object data, bool timeAsColumn = false)
{
_series = new();
var baseData = data as IBaseData;
// in the case we get a list/collection of data we take the first data point to determine the type
// but it's also possible to get a data which supports enumerating we don't care about those cases
if (baseData == null && data is IEnumerable enumerable)
{
foreach (var item in enumerable)
{
data = item;
baseData = data as IBaseData;
break;
}
}
var type = data.GetType();
_isFundamentalType = type == typeof(Fundamental);
_isBaseData = baseData != null;
_timeAsColumn = timeAsColumn && _isBaseData;
_symbol = _isBaseData ? baseData.Symbol : ((ISymbolProvider)data).Symbol;
IsCustomData = Extensions.IsCustomDataType(_symbol, type);
if (baseData == null)
{
Levels = 1;
}
else if (_symbol.SecurityType == SecurityType.Future)
{
Levels = 3;
}
else if (_symbol.SecurityType.IsOption())
{
Levels = 5;
}
}
/// <summary>
/// Adds security data object to the end of the lists
/// </summary>
/// <param name="data"><see cref="IBaseData"/> object that contains security data</param>
public void Add(object data)
{
Add(data, false);
}
private void Add(object data, bool overrideValues)
{
if (data == null)
{
return;
}
var typeMembers = GetInstanceDataTypeMembers(data);
var endTime = default(DateTime);
if (_isBaseData)
{
endTime = ((IBaseData)data).EndTime;
if (_timeAsColumn)
{
AddToSeries("time", endTime, endTime, overrideValues);
}
}
AddMembersData(data, typeMembers, endTime, overrideValues);
if (data is DynamicData dynamicData)
{
var storage = dynamicData.GetStorageDictionary();
var value = dynamicData.Value;
AddToSeries("value", endTime, value, overrideValues);
foreach (var kvp in storage.Where(x => x.Key != "value"
// if this is a PythonData instance we add in '__typename' which we don't want into the data frame
&& !x.Key.StartsWith("__", StringComparison.InvariantCulture)))
{
AddToSeries(kvp.Key, endTime, kvp.Value, overrideValues);
}
}
}
private void AddMemberToSeries(object instance, DateTime endTime, DataTypeMember member, bool overrideValues)
{
var baseName = (string)null;
var tick = member.IsTickProperty ? instance as Tick : null;
if (tick != null && member.IsTickLastPrice && tick.TickType == TickType.OpenInterest)
{
baseName = "OpenInterest";
}
// TODO field/property.GetValue is expensive
var key = member.GetMemberName(baseName);
var value = member.GetValue(instance);
var memberType = member.GetMemberType();
// For DataDictionary instances, we only want to add the values
if (MemberIsDataDictionary(memberType))
{
value = memberType.GetProperty("Values").GetValue(value);
}
else if (member.IsProperty)
{
if (_isFundamentalType && value is FundamentalTimeDependentProperty timeDependentProperty)
{
value = timeDependentProperty.Clone(new FixedTimeProvider(endTime));
}
else if (member.IsTickProperty && tick != null)
{
if (tick.TickType != TickType.Quote && _quoteTickOnlyPropertes.Contains(member.Member.Name))
{
value = null;
}
else if (member.IsTickLastPrice)
{
var nullValueKey = tick.TickType != TickType.OpenInterest
? member.GetMemberName("OpenInterest")
: member.GetMemberName();
AddToSeries(nullValueKey, endTime, null, overrideValues);
}
}
}
AddToSeries(key, endTime, value, overrideValues);
}
/// <summary>
/// Adds Lean data objects to the end of the lists
/// </summary>
/// <param name="tradeBar"><see cref="TradeBar"/> object that contains trade bar information of the security</param>
/// <param name="quoteBar"><see cref="QuoteBar"/> object that contains quote bar information of the security</param>
public void Add(TradeBar tradeBar, QuoteBar quoteBar)
{
// Quote bar first, so if there is a trade bar, OHLC will be overwritten
Add(quoteBar);
Add(tradeBar, overrideValues: true);
}
/// <summary>
/// Get the pandas.DataFrame of the current <see cref="PandasData"/> state
/// </summary>
/// <param name="levels">Number of levels of the multi index</param>
/// <param name="filterMissingValueColumns">If false, make sure columns with "missing" values only are still added to the dataframe</param>
/// <returns>pandas.DataFrame object</returns>
public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns = true)
{
using var _ = Py.GIL();
PyObject[] indexTemplate;
// Create the index labels
var names = _defaultNames;
if (levels == 1)
{
names = _level1Names;
indexTemplate = GetIndexTemplate(_symbol);
}
else if (levels == 2)
{
// symbol, time
names = _level2Names;
indexTemplate = GetIndexTemplate(_symbol, null);
}
else if (levels == 3)
{
// expiry, symbol, time
names = _level3Names;
indexTemplate = GetIndexTemplate(_symbol.ID.Date, _symbol, null);
}
else
{
if (_symbol.SecurityType == SecurityType.Future)
{
indexTemplate = GetIndexTemplate(_symbol.ID.Date, null, null, _symbol, null);
}
else if (_symbol.SecurityType.IsOption())
{
indexTemplate = GetIndexTemplate(_symbol.ID.Date, _symbol.ID.StrikePrice, _symbol.ID.OptionRight, _symbol, null);
}
else
{
indexTemplate = GetIndexTemplate(null, null, null, _symbol, null);
}
}
names = new PyList(names.SkipLast(names.Count() > 1 && _timeAsColumn ? 1 : 0).ToArray());
// creating the pandas MultiIndex is expensive so we keep a cash
var indexCache = new Dictionary<IReadOnlyCollection<DateTime>, PyObject>(new ListComparer<DateTime>());
// Returns a dictionary keyed by column name where values are pandas.Series objects
using var pyDict = new PyDict();
foreach (var (seriesName, serie) in _series)
{
if (filterMissingValueColumns && serie.ShouldFilter) continue;
var key = serie.Times ?? EmptySeriesTimesKey;
if (!indexCache.TryGetValue(key, out var index))
{
PyList indexSource;
if (_timeAsColumn)
{
indexSource = serie.Values.Select(_ => CreateIndexSourceValue(DateTime.MinValue, indexTemplate)).ToPyListUnSafe();
}
else
{
indexSource = serie.Times.Select(time => CreateIndexSourceValue(time, indexTemplate)).ToPyListUnSafe();
}
if (indexTemplate.Length == 1)
{
using var nameDic = Py.kw("name", names[0]);
index = _indexFactory.Invoke(new[] { indexSource }, nameDic);
}
else
{
using var namesDic = Py.kw("names", names);
index = _multiIndexFactory.Invoke(new[] { indexSource }, namesDic);
}
indexCache[key] = index;
foreach (var pyObject in indexSource)
{
pyObject.Dispose();
}
indexSource.Dispose();
}
// Adds pandas.Series value keyed by the column name
using var pyvalues = new PyList();
for (var i = 0; i < serie.Values.Count; i++)
{
using var pyObject = serie.Values[i].ToPython();
pyvalues.Append(pyObject);
}
using var series = _seriesFactory.Invoke(pyvalues, index);
using var pyStrKey = seriesName.ToPython();
using var pyKey = _pandasColumn.Invoke(pyStrKey);
pyDict.SetItem(pyKey, series);
}
_series.Clear();
foreach (var kvp in indexCache)
{
kvp.Value.Dispose();
}
for (var i = 0; i < indexTemplate.Length; i++)
{
DisposeIfNotEmpty(indexTemplate[i]);
}
names.Dispose();
// Create the DataFrame
var result = _dataFrameFactory.Invoke(pyDict);
foreach (var item in pyDict)
{
item.Dispose();
}
return result;
}
/// <summary>
/// Helper method to create a single pandas data frame indexed by symbol
/// </summary>
/// <remarks>Will add a single point per pandas data series (symbol)</remarks>
public static PyObject ToPandasDataFrame(IEnumerable<PandasData> pandasDatas, bool skipTimesColumn = false)
{
using var _ = Py.GIL();
using var list = pandasDatas.Select(x => x._symbol).ToPyListUnSafe();
using var namesDic = Py.kw("name", _level1Names[0]);
using var index = _indexFactory.Invoke(new[] { list }, namesDic);
var valuesPerSeries = new Dictionary<string, PyList>();
var seriesToSkip = new Dictionary<string, bool>();
foreach (var pandasData in pandasDatas)
{
foreach (var kvp in pandasData._series)
{
if (skipTimesColumn && kvp.Key == "time")
{
continue;
}
if (seriesToSkip.ContainsKey(kvp.Key))
{
seriesToSkip[kvp.Key] &= kvp.Value.ShouldFilter;
}
else
{
seriesToSkip[kvp.Key] = kvp.Value.ShouldFilter;
}
if (!valuesPerSeries.TryGetValue(kvp.Key, out PyList value))
{
// Adds pandas.Series value keyed by the column name
value = valuesPerSeries[kvp.Key] = new PyList();
}
if (kvp.Value.Values.Count > 0)
{
// taking only 1 value per symbol
using var valueOfSymbol = kvp.Value.Values[0].ToPython();
value.Append(valueOfSymbol);
}
else
{
value.Append(PyObject.None);
}
}
}
using var pyDict = new PyDict();
foreach (var kvp in valuesPerSeries)
{
if (seriesToSkip.TryGetValue(kvp.Key, out var skip) && skip)
{
continue;
}
using var series = _seriesFactory.Invoke(kvp.Value, index);
using var pyStrKey = kvp.Key.ToPython();
using var pyKey = _pandasColumn.Invoke(pyStrKey);
pyDict.SetItem(pyKey, series);
kvp.Value.Dispose();
}
var result = _dataFrameFactory.Invoke(pyDict);
// Drop columns with only NaN or None values
using var dropnaKwargs = Py.kw("axis", 1, "inplace", true, "how", "all");
result.GetAttr("dropna").Invoke(Array.Empty<PyObject>(), dropnaKwargs);
return result;
}
private List<DataTypeMember> GetInstanceDataTypeMembers(object data)
{
var type = data.GetType();
if (!_members.TryGetValue(type, out var members))
{
HashSet<string> columnNames;
if (data is DynamicData dynamicData)
{
columnNames = (data as DynamicData)?.GetStorageDictionary()
// if this is a PythonData instance we add in '__typename' which we don't want into the data frame
.Where(x => !x.Key.StartsWith("__", StringComparison.InvariantCulture)).ToHashSet(x => x.Key);
columnNames.Add("value");
members = EmptyDataTypeMembers;
}
else
{
members = GetTypeMembers(type);
columnNames = members.SelectMany(x => x.GetMemberNames()).ToHashSet();
// We add openinterest key so the series is created: open interest tick LastPrice is renamed to OpenInterest
if (data is Tick)
{
columnNames.Add("openinterest");
}
}
_members[type] = members;
if (_timeAsColumn)
{
columnNames.Add("time");
}
foreach (var columnName in columnNames)
{
_series.TryAdd(columnName, new Serie(withTimeIndex: !_timeAsColumn));
}
}
return members;
}
/// <summary>
/// Gets or create/adds the <see cref="DataTypeMember"/> instances corresponding to the members of the given type,
/// and returns the names of the members.
/// </summary>
private List<DataTypeMember> GetTypeMembers(Type type)
{
List<DataTypeMember> typeMembers;
lock (_membersCache)
{
if (!_membersCache.TryGetValue(type, out typeMembers))
{
// Contracts (e.g. OptionContract, FuturesContract) expose their own representative price members
// (LastPrice, BidPrice, ...) and mark the BaseData-like aliases (Value, Price, Close) with
// PandasIgnore, so we don't want to force the Value member in as we do for custom data types.
var forcedInclusionMembers = LeanData.IsCommonLeanDataType(type) || typeof(BaseContract).IsAssignableFrom(type)
? Array.Empty<string>()
: _nonLeanDataTypeForcedMemberNames;
typeMembers = GetDataTypeMembers(type, forcedInclusionMembers).ToList();
_membersCache[type] = typeMembers;
}
}
_members[type] = typeMembers;
return typeMembers;
}
/// <summary>
/// Gets the <see cref="DataTypeMember"/> instances corresponding to the members of the given type.
/// It will try to unwrap properties which types are classes unless they are marked either to be ignored or to be added as a whole
/// </summary>
private static IEnumerable<DataTypeMember> GetDataTypeMembers(Type type, string[] forcedInclusionMembers)
{
var members = type
.GetMembers(BindingFlags.Instance | BindingFlags.Public)
.Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property)
.Where(x => forcedInclusionMembers.Contains(x.Name)
|| (!x.IsDefined(PandasIgnoreAttribute) && !x.DeclaringType.IsDefined(PandasIgnoreMembersAttribute)));
return members
.Select(member =>
{
var dataTypeMember = CreateDataTypeMember(member);
var memberType = dataTypeMember.GetMemberType();
// Should we unpack its properties into columns?
if (memberType.IsClass
&& (memberType.Namespace == null
// We only expand members of types in the QuantConnect namespace,
// else we might be expanding types like System.String, NodaTime.DateTimeZone or any other external types
|| (memberType.Namespace.StartsWith("QuantConnect.", StringComparison.InvariantCulture)
&& !memberType.IsDefined(PandasNonExpandableAttribute)
&& !member.IsDefined(PandasNonExpandableAttribute))))
{
dataTypeMember = CreateDataTypeMember(member, GetDataTypeMembers(memberType, forcedInclusionMembers).ToArray());
}
return (memberType, dataTypeMember);
})
// Check if there are multiple properties/fields of the same type,
// in which case we add the property/field name as prefix for the inner members to avoid name conflicts
.GroupBy(x => x.memberType, x => x.dataTypeMember)
.SelectMany(grouping =>
{
var typeProperties = grouping.ToList();
if (typeProperties.Count > 1)
{
var propertiesToExpand = typeProperties.Where(x => x.ShouldBeUnwrapped).ToList();
if (propertiesToExpand.Count > 1)
{
foreach (var property in propertiesToExpand)
{
property.SetPrefix();
}
}
}
return typeProperties;
});
}
/// <summary>
/// Adds the member value to the corresponding series, making sure unwrapped values a properly added
/// by checking the children members and adding their values to their own series
/// </summary>
private void AddMembersData(object instance, IEnumerable<DataTypeMember> members, DateTime endTime, bool overrideValues)
{
foreach (var member in members)
{
if (!member.ShouldBeUnwrapped)
{
AddMemberToSeries(instance, endTime, member, overrideValues);
}
else
{
var memberValue = member.GetValue(instance);
if (memberValue != null)
{
AddMembersData(memberValue, member.Children, endTime, overrideValues);
}
}
}
}
/// <summary>
/// Only dipose of the PyObject if it was set to something different than empty
/// </summary>
private static void DisposeIfNotEmpty(PyObject pyObject)
{
if (!ReferenceEquals(pyObject, _empty))
{
pyObject.Dispose();
}
}
private static bool MemberIsDataDictionary(Type memberType)
{
while (memberType != null && !memberType.IsValueType)
{
if (memberType.IsGenericType && memberType.GetGenericTypeDefinition() == typeof(DataDictionary<>))
{
return true;
}
memberType = memberType.BaseType;
}
return false;
}
private PyObject[] GetIndexTemplate(params object[] args)
{
return args.SkipLast(args.Length > 1 && _timeAsColumn ? 1 : 0).Select(x => x?.ToPython() ?? _empty).ToArray();
}
/// <summary>
/// Create a new tuple index
/// </summary>
private PyObject CreateIndexSourceValue(DateTime index, PyObject[] list)
{
if (!_timeAsColumn && list.Length > 1)
{
DisposeIfNotEmpty(list[^1]);
list[^1] = index.ToPython();
}
if (list.Length > 1)
{
return new PyTuple(list.ToArray());
}
return list[0].ToPython();
}
/// <summary>
/// Adds data to dictionary
/// </summary>
/// <param name="key">The key of the value to get</param>
/// <param name="time"><see cref="DateTime"/> object to add to the value associated with the specific key</param>
/// <param name="input"><see cref="Object"/> to add to the value associated with the specific key. Can be null.</param>
private void AddToSeries(string key, DateTime time, object input, bool overrideValues)
{
if (!_series.TryGetValue(key, out var serie))
{
throw new ArgumentException($"PandasData.AddToSeries(): {Messages.PandasData.KeyNotFoundInSeries(key)}");
}
serie.Add(time, input, overrideValues);
}
private class Serie
{
private static readonly IFormatProvider InvariantCulture = CultureInfo.InvariantCulture;
public bool ShouldFilter { get; private set; }
public List<DateTime> Times { get; }
public List<object> Values { get; }
public Serie(bool withTimeIndex = true)
{
ShouldFilter = true;
Values = new();
if (withTimeIndex)
{
Times = new();
}
}
public void Add(DateTime time, object input, bool overrideValues)
{
var value = input is decimal ? Convert.ToDouble(input, InvariantCulture) : input;
if (ShouldFilter)
{
// we need at least 1 valid entry for the series not to get filtered
if (value is double doubleValue)
{
if (!doubleValue.IsNaNOrZero())
{
ShouldFilter = false;
}
}
else if (value is string stringValue)
{
if (!string.IsNullOrWhiteSpace(stringValue))
{
ShouldFilter = false;
}
}
else if (value is bool boolValue)
{
if (boolValue)
{
ShouldFilter = false;
}
}
else if (value != null)
{
if (value is ICollection enumerable)
{
if (enumerable.Count != 0)
{
ShouldFilter = false;
}
}
else
{
ShouldFilter = false;
}
}
}
if (overrideValues && Times != null && Times.Count > 0 && Times[^1] == time)
{
// If the time is the same as the last one, we overwrite the value
Values[^1] = value;
}
else
{
Values.Add(value);
Times?.Add(time);
}
}
}
private class FixedTimeProvider : ITimeProvider
{
private readonly DateTime _time;
public DateTime GetUtcNow() => _time;
public FixedTimeProvider(DateTime time)
{
_time = time;
}
}
}
}
+28
View File
@@ -0,0 +1,28 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
namespace QuantConnect.Python
{
/// <summary>
/// Attribute to mark a property or field as ignored when converting an instance to a pandas DataFrame row.
/// No column will be created for this property or field.
/// </summary>
[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field)]
public sealed class PandasIgnoreAttribute : Attribute
{
}
}
@@ -0,0 +1,27 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
namespace QuantConnect.Python
{
/// <summary>
/// Attribute to indicate the pandas converter to ignore all members of the class when converting an instance to a pandas DataFrame row.
/// </summary>
[AttributeUsage(AttributeTargets.Class, Inherited = false)]
public sealed class PandasIgnoreMembersAttribute : Attribute
{
}
}
@@ -0,0 +1,28 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
namespace QuantConnect.Python
{
/// <summary>
/// Attribute to mark a class, field or property as non-expandable by the pandas converter.
/// The instance will be added to the dataframe as it is, without unwrapping its fields and properties into columns.
/// </summary>
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Property | AttributeTargets.Field)]
public sealed class PandasNonExpandableAttribute : Attribute
{
}
}
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<!-- For Linux and Mac users: uncomment and fill either of the following: -->
<!-- <dllmap os="linux" dll="python3.6m" target = "/home/{your_user_name}/miniconda3/envs/{qc_environment}/lib/libpython3.6m.so"/> -->
<!-- <dllmap os="osx" dll="python3.6m" target = "/Users/{your_user_name}/anaconda3/lib/libpython3.6m.dylib"/> -->
</configuration>
+55
View File
@@ -0,0 +1,55 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using System;
namespace QuantConnect.Python
{
/// <summary>
/// Provides methods for creating new instances of python custom data objects
/// </summary>
public class PythonActivator
{
/// <summary>
/// <see cref="System.Type"/> of the object we wish to create
/// </summary>
public Type Type { get; }
/// <summary>
/// Method to return an instance of object
/// </summary>
public Func<object[], object> Factory { get; }
/// <summary>
/// Creates a new instance of <see cref="PythonActivator"/>
/// </summary>
/// <param name="type"><see cref="System.Type"/> of the object we wish to create</param>
/// <param name="value"><see cref="PyObject"/> that contains the python type</param>
public PythonActivator(Type type, PyObject value)
{
Type = type;
Factory = x =>
{
using (Py.GIL())
{
var instance = value.Invoke();
return new PythonData(instance);
}
};
}
}
}
+114
View File
@@ -0,0 +1,114 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Data;
using QuantConnect.Data.Consolidators;
using QuantConnect.Indicators;
using System;
namespace QuantConnect.Python
{
/// <summary>
/// Provides a base class for python consolidators, necessary to use event handler.
/// Inherits the built-in rolling window so custom Python consolidators can access their
/// consolidated history through the indexer, Current, Previous and Window members.
/// </summary>
public class PythonConsolidator : WindowBase<IBaseData>, IDataConsolidator
{
/// <summary>
/// Gets the most recently consolidated piece of data. This will be null if this consolidator
/// has not produced any data yet.
/// </summary>
public IBaseData Consolidated
{
get; set;
}
/// <summary>
/// Gets a clone of the data being currently consolidated
/// </summary>
public IBaseData WorkingData
{
get; set;
}
/// <summary>
/// Gets the type consumed by this consolidator
/// </summary>
public Type InputType
{
get; set;
}
/// <summary>
/// Gets the type produced by this consolidator
/// </summary>
public Type OutputType
{
get; set;
}
/// <summary>
/// Event handler that fires when a new piece of data is produced
/// </summary>
public event DataConsolidatedHandler DataConsolidated;
/// <summary>
/// Function to invoke the event handler
/// </summary>
/// <param name="consolidator">Reference to the consolidator itself</param>
/// <param name="data">The finished data from the consolidator</param>
public void OnDataConsolidated(PyObject consolidator, IBaseData data)
{
// populate the rolling window before firing so a handler sees the new bar at index 0
if (data != null)
{
Current = data;
}
DataConsolidated?.Invoke(consolidator, data);
}
/// <summary>
/// Resets the consolidator
/// </summary>
public virtual void Reset()
{
Consolidated = null;
WorkingData = null;
ResetWindow();
}
/// <summary>
/// Scans this consolidator to see if it should emit a bar due to time passing
/// </summary>
/// <param name="currentLocalTime">The current time in the local time zone (same as <see cref="BaseData.Time"/>)</param>
public virtual void Scan(DateTime currentLocalTime)
{
}
/// <summary>
/// Updates this consolidator with the specified data
/// </summary>
/// <param name="data">The new data for the consolidator</param>
public virtual void Update(IBaseData data)
{
}
public void Dispose()
{
}
}
}
+228
View File
@@ -0,0 +1,228 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Data;
using System;
using System.Collections.Generic;
namespace QuantConnect.Python
{
/// <summary>
/// Dynamic data class for Python algorithms.
/// Stores properties of python instances in DynamicData dictionary
/// </summary>
public class PythonData : DynamicData
{
private readonly string _pythonTypeName;
private readonly dynamic _pythonReader;
private readonly dynamic _pythonGetSource;
private readonly dynamic _pythonData;
private readonly dynamic _defaultResolution;
private readonly dynamic _supportedResolutions;
private readonly dynamic _isSparseData;
private readonly dynamic _requiresMapping;
private DateTime _endTime;
/// <summary>
/// The end time of this data. Some data covers spans (trade bars)
/// and as such we want to know the entire time span covered
/// </summary>
/// <remarks>
/// This property is overriden to allow different values for Time and EndTime
/// if they are set in the Reader. In the base implementation EndTime equals Time
/// </remarks>
public override DateTime EndTime
{
get
{
return _endTime == default ? Time : _endTime;
}
set
{
_endTime = value;
if(Time == default)
{
// if Time hasn't been set let's set it, like BaseData does. If the user overrides it that's okay
Time = value;
}
}
}
/// <summary>
/// Constructor for initializing the PythonData class
/// </summary>
public PythonData()
{
//Empty constructor required for fast-reflection initialization
}
/// <summary>
/// Constructor for initializing the PythonData class with wrapped PyObject
/// </summary>
/// <param name="pythonData"></param>
public PythonData(PyObject pythonData)
{
_pythonData = pythonData;
using (Py.GIL())
{
// these methods rely on the Symbol so we can call them yet but we can get them
_requiresMapping = pythonData.GetMethod("RequiresMapping");
_isSparseData = pythonData.GetMethod("IsSparseData");
_defaultResolution = pythonData.GetMethod("DefaultResolution");
_supportedResolutions = pythonData.GetMethod("SupportedResolutions");
_pythonReader = pythonData.GetMethod("Reader");
_pythonGetSource = pythonData.GetMethod("GetSource");
_pythonTypeName = pythonData.GetPythonType().GetAssemblyName().Name;
}
}
/// <summary>
/// Source Locator for algorithm written in Python.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="date">Date of the data file we're looking for</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>STRING API Url.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
using (Py.GIL())
{
var source = _pythonGetSource(config, date, isLiveMode);
return (source as PyObject).GetAndDispose<SubscriptionDataSource>();
}
}
/// <summary>
/// Generic Reader Implementation for Python Custom Data.
/// </summary>
/// <param name="config">Subscription configuration</param>
/// <param name="line">CSV line of data from the source</param>
/// <param name="date">Date of the requested line</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns></returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
using (Py.GIL())
{
var data = _pythonReader(config, line, date, isLiveMode);
var result = (data as PyObject).GetAndDispose<BaseData>();
(result as PythonData)?.SetProperty("__typename", _pythonTypeName);
return result;
}
}
/// <summary>
/// Indicates if there is support for mapping
/// </summary>
/// <returns>True indicates mapping should be used</returns>
public override bool RequiresMapping()
{
if (_requiresMapping == null)
{
return base.RequiresMapping();
}
using (Py.GIL())
{
return _requiresMapping();
}
}
/// <summary>
/// Indicates that the data set is expected to be sparse
/// </summary>
/// <remarks>Relies on the <see cref="Symbol"/> property value</remarks>
/// <returns>True if the data set represented by this type is expected to be sparse</returns>
public override bool IsSparseData()
{
if (_isSparseData == null)
{
return base.IsSparseData();
}
using (Py.GIL())
{
return _isSparseData();
}
}
/// <summary>
/// Gets the default resolution for this data and security type
/// </summary>
/// <remarks>This is a method and not a property so that python
/// custom data types can override it</remarks>
public override Resolution DefaultResolution()
{
if (_defaultResolution == null)
{
return base.DefaultResolution();
}
using (Py.GIL())
{
return _defaultResolution();
}
}
/// <summary>
/// Gets the supported resolution for this data and security type
/// </summary>
/// <remarks>This is a method and not a property so that python
/// custom data types can override it</remarks>
public override List<Resolution> SupportedResolutions()
{
if (_supportedResolutions == null)
{
return base.SupportedResolutions();
}
using (Py.GIL())
{
return _supportedResolutions();
}
}
/// <summary>
/// Indexes into this PythonData, where index is key to the dynamic property
/// </summary>
/// <param name="index">the index</param>
/// <returns>Dynamic property of a given index</returns>
public object this[string index]
{
get
{
return GetProperty(index);
}
set
{
SetProperty(index, value is double ? value.ConvertInvariant<decimal>() : value);
}
}
/// <summary>
/// Helper method to determine if the current instance is of the provided type
/// </summary>
/// <param name="type">Target type to check against</param>
/// <returns>True if this instance is of the provided type</returns>
public bool IsOfType(Type type)
{
if (HasProperty("__typename"))
{
return (string)GetProperty("__typename") == type.FullName;
}
return GetType() == type;
}
}
}
+332
View File
@@ -0,0 +1,332 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.IO;
using System.Linq;
using Python.Runtime;
using QuantConnect.Util;
using QuantConnect.Logging;
using System.Collections.Generic;
using QuantConnect.Configuration;
namespace QuantConnect.Python
{
/// <summary>
/// Helper class for Python initialization
/// </summary>
public static class PythonInitializer
{
private static bool IncludeSystemPackages;
private static string PathToVirtualEnv;
// Used to allow multiple Python unit and regression tests to be run in the same test run
private static bool _isInitialized;
// Used to hold pending path additions before Initialize is called
private static List<string> _pendingPathAdditions = new List<string>();
private static string _algorithmLocation;
/// <summary>
/// Initialize python.
///
/// In some cases, we might not need to call BeginAllowThreads, like when we're running
/// in a python or non-threaded environment.
/// In those cases, we can set the beginAllowThreads parameter to false.
/// </summary>
public static void Initialize(bool beginAllowThreads = true)
{
if (!_isInitialized)
{
Log.Trace($"PythonInitializer.Initialize(): {Messages.PythonInitializer.Start}...");
PythonEngine.Initialize();
if (beginAllowThreads)
{
// required for multi-threading usage
PythonEngine.BeginAllowThreads();
}
_isInitialized = true;
ConfigurePythonPaths();
TryInitPythonVirtualEnvironment();
Log.Trace($"PythonInitializer.Initialize(): {Messages.PythonInitializer.Ended}");
}
}
/// <summary>
/// Shutdown python
/// </summary>
public static void Shutdown()
{
if (_isInitialized)
{
Log.Trace($"PythonInitializer.Shutdown(): {Messages.PythonInitializer.Start}");
_isInitialized = false;
try
{
var pyLock = Py.GIL();
Log.Trace($"PythonInitializer.Shutdown(): calling engine shutdown...");
PythonEngine.Shutdown();
}
catch (Exception ex)
{
Log.Error(ex);
}
Log.Trace($"PythonInitializer.Shutdown(): {Messages.PythonInitializer.Ended}");
}
}
/// <summary>
/// Adds directories to the python path at runtime
/// </summary>
public static bool AddPythonPaths(IEnumerable<string> paths)
{
// Filter out any paths that are already on our Python path
if (paths.IsNullOrEmpty())
{
return false;
}
// Add these paths to our pending additions
_pendingPathAdditions.AddRange(paths.Where(x => !_pendingPathAdditions.Contains(x)));
if (_isInitialized)
{
using (Py.GIL())
{
using dynamic sys = Py.Import("sys");
using var locals = new PyDict();
locals.SetItem("sys", sys);
// Filter out any already paths that already exist on our current PythonPath
using var pythonCurrentPath = PythonEngine.Eval("sys.path", locals: locals);
var currentPath = pythonCurrentPath.As<List<string>>();
_pendingPathAdditions = _pendingPathAdditions.Where(x => !currentPath.Contains(x.Replace('\\', '/'))).ToList();
// Algorithm location most always be before any other path added through this method
var insertionIndex = 0;
if (!_algorithmLocation.IsNullOrEmpty())
{
insertionIndex = currentPath.IndexOf(_algorithmLocation.Replace('\\', '/')) + 1;
if (insertionIndex == 0)
{
// The algorithm location is not in the current path so it must be in the pending additions list.
// Let's move it to the back so it ends up added at the beginning of the path list
_pendingPathAdditions.Remove(_algorithmLocation);
_pendingPathAdditions.Add(_algorithmLocation);
}
}
// Insert any pending path additions
if (!_pendingPathAdditions.IsNullOrEmpty())
{
var code = string.Join(";", _pendingPathAdditions
.Select(s => $"sys.path.insert({insertionIndex}, '{s}')")).Replace('\\', '/');
PythonEngine.Exec(code, locals: locals);
_pendingPathAdditions.Clear();
}
}
}
return true;
}
/// <summary>
/// Adds the algorithm location to the python path.
/// This will make sure that <see cref="AddPythonPaths" /> keeps the algorithm location path
/// at the beginning of the pythonpath.
/// </summary>
public static void AddAlgorithmLocationPath(string algorithmLocation)
{
if (!_algorithmLocation.IsNullOrEmpty())
{
return;
}
if (!Directory.Exists(algorithmLocation))
{
Log.Error($@"PythonInitializer.AddAlgorithmLocationPath(): {
Messages.PythonInitializer.UnableToLocateAlgorithm(algorithmLocation)}");
return;
}
_algorithmLocation = algorithmLocation;
AddPythonPaths(new[] { _algorithmLocation });
}
/// <summary>
/// Resets the algorithm location path so another can be set
/// </summary>
public static void ResetAlgorithmLocationPath()
{
_algorithmLocation = null;
}
/// <summary>
/// "Activate" a virtual Python environment by prepending its library storage to Pythons
/// path. This allows the libraries in this venv to be selected prior to our base install.
/// Requires PYTHONNET_PYDLL to be set to base install.
/// </summary>
/// <remarks>If a module is already loaded, Python will use its cached version first
/// these modules must be reloaded by reload() from importlib library</remarks>
public static bool ActivatePythonVirtualEnvironment(string pathToVirtualEnv)
{
if (string.IsNullOrEmpty(pathToVirtualEnv))
{
return false;
}
if(!Directory.Exists(pathToVirtualEnv))
{
Log.Error($@"PythonIntializer.ActivatePythonVirtualEnvironment(): {
Messages.PythonInitializer.VirutalEnvironmentNotFound(pathToVirtualEnv)}");
return false;
}
PathToVirtualEnv = pathToVirtualEnv;
bool? includeSystemPackages = null;
var configFile = new FileInfo(Path.Combine(PathToVirtualEnv, "pyvenv.cfg"));
if(configFile.Exists)
{
foreach (var line in File.ReadAllLines(configFile.FullName))
{
if (line.Contains("include-system-site-packages", StringComparison.InvariantCultureIgnoreCase))
{
// format: include-system-site-packages = false (or true)
var equalsIndex = line.IndexOf('=', StringComparison.InvariantCultureIgnoreCase);
if(equalsIndex != -1 && line.Length > (equalsIndex + 1) && bool.TryParse(line.Substring(equalsIndex + 1).Trim(), out var result))
{
includeSystemPackages = result;
break;
}
}
}
}
if(!includeSystemPackages.HasValue)
{
includeSystemPackages = true;
Log.Error($@"PythonIntializer.ActivatePythonVirtualEnvironment(): {
Messages.PythonInitializer.FailedToFindSystemPackagesConfiguration(pathToVirtualEnv, configFile)}");
}
else
{
Log.Trace($@"PythonIntializer.ActivatePythonVirtualEnvironment(): {
Messages.PythonInitializer.SystemPackagesConfigurationFound(pathToVirtualEnv, includeSystemPackages.Value)}");
}
if (!includeSystemPackages.Value)
{
PythonEngine.SetNoSiteFlag();
}
IncludeSystemPackages = includeSystemPackages.Value;
TryInitPythonVirtualEnvironment();
return true;
}
private static void TryInitPythonVirtualEnvironment()
{
if (!_isInitialized || string.IsNullOrEmpty(PathToVirtualEnv))
{
return;
}
using (Py.GIL())
{
using dynamic sys = Py.Import("sys");
using var locals = new PyDict();
locals.SetItem("sys", sys);
if (!IncludeSystemPackages)
{
var currentPath = (List<string>)sys.path.As<List<string>>();
var toRemove = new List<string>(currentPath.Where(s => s.Contains("site-packages", StringComparison.InvariantCultureIgnoreCase)));
if (toRemove.Count > 0)
{
var code = string.Join(";", toRemove.Select(s => $"sys.path.remove('{s}')"));
PythonEngine.Exec(code, locals: locals);
}
}
// fix the prefixes to point to our venv
sys.prefix = PathToVirtualEnv;
sys.exec_prefix = PathToVirtualEnv;
using dynamic site = Py.Import("site");
// This has to be overwritten because site module may already have been loaded by the interpreter (but not run yet)
site.PREFIXES = new List<PyObject> { sys.prefix, sys.exec_prefix };
// Run site path modification with tweaked prefixes
site.main();
if (IncludeSystemPackages)
{
// let's make sure our site packages is at the start so that we support overriding system libraries with a version in the env
PythonEngine.Exec(@$"if sys.path[-1].startswith('{PathToVirtualEnv}'):
sys.path.insert(0, sys.path.pop())", locals: locals);
}
if (Log.DebuggingEnabled)
{
using dynamic os = Py.Import("os");
var path = new List<string>();
foreach (var p in sys.path)
{
path.Add((string)p);
}
Log.Debug($"PythonIntializer.InitPythonVirtualEnvironment(): PYTHONHOME: {os.getenv("PYTHONHOME")}." +
$" PYTHONPATH: {os.getenv("PYTHONPATH")}." +
$" sys.executable: {sys.executable}." +
$" sys.prefix: {sys.prefix}." +
$" sys.base_prefix: {sys.base_prefix}." +
$" sys.exec_prefix: {sys.exec_prefix}." +
$" sys.base_exec_prefix: {sys.base_exec_prefix}." +
$" sys.path: [{string.Join(",", path)}]");
}
}
}
/// <summary>
/// Gets the python additional paths from the config and adds them to Python using the PythonInitializer
/// </summary>
private static void ConfigurePythonPaths()
{
var pythonAdditionalPaths = new List<string> { Environment.CurrentDirectory };
pythonAdditionalPaths.AddRange(Config.GetValue("python-additional-paths", Enumerable.Empty<string>()));
AddPythonPaths(pythonAdditionalPaths.Where(path =>
{
var pathExists = Directory.Exists(path);
if (!pathExists)
{
Log.Error($"PythonInitializer.ConfigurePythonPaths(): {Messages.PythonInitializer.PythonPathNotFound(path)}");
}
return pathExists;
}));
}
}
}
+136
View File
@@ -0,0 +1,136 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using Python.Runtime;
namespace QuantConnect.Python
{
/// <summary>
/// Provides extension methods for managing python wrapper classes
/// </summary>
public static class PythonWrapper
{
/// <summary>
/// Validates that the specified <see cref="PyObject"/> completely implements the provided interface type
/// </summary>
/// <typeparam name="TInterface">The interface type</typeparam>
/// <param name="model">The model implementing the interface type</param>
public static PyObject ValidateImplementationOf<TInterface>(this PyObject model)
{
var notInterface = !typeof(TInterface).IsInterface;
var missingMembers = new List<string>();
var members = typeof(TInterface).GetMembers(BindingFlags.Public | BindingFlags.Instance);
using (Py.GIL())
{
foreach (var member in members)
{
var method = member as MethodInfo;
if ((method == null || !method.IsSpecialName) &&
!model.HasAttr(member.Name) && !model.HasAttr(member.Name.ToSnakeCase()))
{
if (notInterface)
{
if (method != null && !method.IsAbstract && (method.IsFinal || !method.IsVirtual || method.DeclaringType != typeof(TInterface)))
{
continue;
}
else if (member is ConstructorInfo)
{
continue;
}
else if (member.Name is "ToString")
{
continue;
}
}
missingMembers.Add(member.Name);
}
}
if (missingMembers.Any())
{
throw new NotImplementedException(
Messages.PythonWrapper.InterfaceNotFullyImplemented(typeof(TInterface).Name, model.GetPythonType().Name, missingMembers));
}
}
return model;
}
/// <summary>
/// Invokes the specified method on the provided <see cref="PyObject"/> instance with the specified arguments
/// </summary>
/// <param name="model">The <see cref="PyObject"/> instance</param>
/// <param name="methodName">The name of the method to invoke</param>
/// <param name="args">The arguments to call the method with</param>
/// <returns>The return value of the called method converted into the <typeparamref name="T"/> type</returns>
public static T InvokeMethod<T>(this PyObject model, string methodName, params object[] args)
{
using var _ = Py.GIL();
return InvokeMethodImpl(model, methodName, args).GetAndDispose<T>();
}
/// <summary>
/// Invokes the specified method on the provided <see cref="PyObject"/> instance with the specified arguments
/// </summary>
/// <param name="model">The <see cref="PyObject"/> instance</param>
/// <param name="methodName">The name of the method to invoke</param>
/// <param name="args">The arguments to call the method with</param>
public static void InvokeMethod(this PyObject model, string methodName, params object[] args)
{
InvokeMethodImpl(model, methodName, args);
}
/// <summary>
/// Invokes the given <see cref="PyObject"/> method with the specified arguments
/// </summary>
/// <param name="method">The method to invoke</param>
/// <param name="args">The arguments to call the method with</param>
/// <returns>The return value of the called method converted into the <typeparamref name="T"/> type</returns>
public static T Invoke<T>(this PyObject method, params object[] args)
{
using var _ = Py.GIL();
return InvokeMethodImpl(method, args).GetAndDispose<T>();
}
/// <summary>
/// Invokes the given <see cref="PyObject"/> method with the specified arguments
/// </summary>
/// <param name="method">The method to invoke</param>
/// <param name="args">The arguments to call the method with</param>
public static PyObject Invoke(this PyObject method, params object[] args)
{
return InvokeMethodImpl(method, args);
}
private static PyObject InvokeMethodImpl(PyObject model, string methodName, params object[] args)
{
using var _ = Py.GIL();
PyObject method = model.GetMethod(methodName);
return InvokeMethodImpl(method, args);
}
private static PyObject InvokeMethodImpl(PyObject method, params object[] args)
{
using var _ = Py.GIL();
return method.Invoke(args.Select(arg => arg.ToPython()).ToArray());
}
}
}
@@ -0,0 +1,61 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using Python.Runtime;
using QuantConnect.Data;
using QuantConnect.Util;
namespace QuantConnect.Python
{
/// <summary>
/// Wraps a <see cref="PyObject"/> object that represents a risk-free interest rate model
/// </summary>
public class RiskFreeInterestRateModelPythonWrapper : BasePythonWrapper<IRiskFreeInterestRateModel>, IRiskFreeInterestRateModel
{
/// <summary>
/// Constructor for initializing the <see cref="RiskFreeInterestRateModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Represents a security's model of buying power</param>
public RiskFreeInterestRateModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Get interest rate by a given date
/// </summary>
/// <param name="date">The date</param>
/// <returns>Interest rate on the given date</returns>
public decimal GetInterestRate(DateTime date)
{
return InvokeMethod<decimal>(nameof(GetInterestRate), date);
}
/// <summary>
/// Converts a <see cref="PyObject"/> object into a <see cref="IRiskFreeInterestRateModel"/> object, wrapping it if necessary
/// </summary>
/// <param name="model">The Python model</param>
/// <returns>The converted <see cref="IRiskFreeInterestRateModel"/> instance</returns>
public static IRiskFreeInterestRateModel FromPyObject(PyObject model)
{
var riskFreeInterestRateModel = PythonUtil.CreateInstanceOrWrapper<IRiskFreeInterestRateModel>(
model,
py => new RiskFreeInterestRateModelPythonWrapper(py)
);
return riskFreeInterestRateModel;
}
}
}
@@ -0,0 +1,44 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Securities;
namespace QuantConnect.Python
{
/// <summary>
/// Wraps a <see cref="PyObject"/> object that represents a type capable of initializing a new security
/// </summary>
public class SecurityInitializerPythonWrapper : BasePythonWrapper<ISecurityInitializer>, ISecurityInitializer
{
/// <summary>
/// Constructor for initialising the <see cref="SecurityInitializerPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Represents a type capable of initializing a new security</param>
public SecurityInitializerPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Initializes the specified security
/// </summary>
/// <param name="security">The security to be initialized</param>
public void Initialize(Security security)
{
InvokeMethod(nameof(Initialize), security);
}
}
}
@@ -0,0 +1,68 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using Python.Runtime;
using QuantConnect.Securities;
namespace QuantConnect.Python
{
/// <summary>
/// Provides an implementation of <see cref="ISettlementModel"/> that wraps a <see cref="PyObject"/> object
/// </summary>
public class SettlementModelPythonWrapper : BasePythonWrapper<ISettlementModel>, ISettlementModel
{
/// <summary>
/// Constructor for initialising the <see cref="SettlementModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Settlement Python Model</param>
public SettlementModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Applies cash settlement rules using the method defined in the Python class
/// </summary>
/// <param name="applyFundsParameters">The funds application parameters</param>
public void ApplyFunds(ApplyFundsSettlementModelParameters applyFundsParameters)
{
InvokeMethod(nameof(ApplyFunds), applyFundsParameters);
}
/// <summary>
/// Scan for pending settlements using the method defined in the Python class
/// </summary>
/// <param name="settlementParameters">The settlement parameters</param>
public void Scan(ScanSettlementModelParameters settlementParameters)
{
InvokeMethod(nameof(Scan), settlementParameters);
}
/// <summary>
/// Gets the unsettled cash amount for the security
/// </summary>
public CashAmount GetUnsettledCash()
{
var result = InvokeMethod<CashAmount?>(nameof(GetUnsettledCash));
if (result == null)
{
return default;
}
return result.Value;
}
}
}
@@ -0,0 +1,49 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Algorithm.Framework.Portfolio.SignalExports;
using QuantConnect.Interfaces;
namespace QuantConnect.Python
{
/// <summary>
/// Provides an implementation of <see cref="ISignalExportTarget"/> that wraps a <see cref="PyObject"/> object
/// </summary>
public class SignalExportTargetPythonWrapper : BasePythonWrapper<ISignalExportTarget>, ISignalExportTarget
{
/// <summary>
/// Constructor for initialising the <see cref="SignalExportTargetPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="instance">The underlying python instance</param>
public SignalExportTargetPythonWrapper(PyObject instance) : base(instance) { }
/// <summary>
/// Interface to send positions holdings to different 3rd party API's
/// </summary>
public bool Send(SignalExportTargetParameters parameters)
{
return InvokeMethod<bool>(nameof(Send), parameters);
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
public void Dispose()
{
InvokeMethod(nameof(Dispose));
}
}
}
@@ -0,0 +1,48 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Orders;
using QuantConnect.Orders.Slippage;
using QuantConnect.Securities;
namespace QuantConnect.Python
{
/// <summary>
/// Wraps a <see cref="PyObject"/> object that represents a model that simulates market order slippage
/// </summary>
public class SlippageModelPythonWrapper : BasePythonWrapper<ISlippageModel>, ISlippageModel
{
/// <summary>
/// Constructor for initialising the <see cref="SlippageModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model">Represents a model that simulates market order slippage</param>
public SlippageModelPythonWrapper(PyObject model)
: base(model)
{
}
/// <summary>
/// Slippage Model. Return a decimal cash slippage approximation on the order.
/// </summary>
/// <param name="asset">The security matching the order</param>
/// <param name="order">The order to compute slippage for</param>
/// <returns>The slippage of the order in units of the account currency</returns>
public decimal GetSlippageApproximation(Security asset, Order order)
{
return InvokeMethod<decimal>(nameof(GetSlippageApproximation), asset, order);
}
}
}
@@ -0,0 +1,88 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Python.Runtime;
using QuantConnect.Data;
using QuantConnect.Securities;
using System;
using System.Collections.Generic;
using QuantConnect.Interfaces;
using QuantConnect.Securities.Volatility;
namespace QuantConnect.Python
{
/// <summary>
/// Provides a volatility model that wraps a <see cref="PyObject"/> object that represents a model that computes the volatility of a security
/// </summary>
public class VolatilityModelPythonWrapper : BaseVolatilityModel
{
private readonly BasePythonWrapper<IVolatilityModel> _model;
/// <summary>
/// Constructor for initialising the <see cref="VolatilityModelPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
/// </summary>
/// <param name="model"> Represents a model that computes the volatility of a security</param>
public VolatilityModelPythonWrapper(PyObject model)
{
_model = new BasePythonWrapper<IVolatilityModel>(model);
}
/// <summary>
/// Gets the volatility of the security as a percentage
/// </summary>
public override decimal Volatility
{
get
{
return _model.GetProperty<decimal>(nameof(Volatility));
}
}
/// <summary>
/// Updates this model using the new price information in
/// the specified security instance
/// </summary>
/// <param name="security">The security to calculate volatility for</param>
/// <param name="data">The new data used to update the model</param>
public override void Update(Security security, BaseData data)
{
_model.InvokeMethod(nameof(Update), security, data).Dispose();
}
/// <summary>
/// Returns history requirements for the volatility model expressed in the form of history request
/// </summary>
/// <param name="security">The security of the request</param>
/// <param name="utcTime">The date/time of the request</param>
/// <returns>History request object list, or empty if no requirements</returns>
public override IEnumerable<HistoryRequest> GetHistoryRequirements(Security security, DateTime utcTime)
{
return _model.InvokeMethodAndEnumerate<HistoryRequest>(nameof(GetHistoryRequirements), security, utcTime);
}
/// <summary>
/// Sets the <see cref="ISubscriptionDataConfigProvider"/> instance to use.
/// </summary>
/// <param name="subscriptionDataConfigProvider">Provides access to registered <see cref="SubscriptionDataConfig"/></param>
public override void SetSubscriptionDataConfigProvider(
ISubscriptionDataConfigProvider subscriptionDataConfigProvider)
{
if (_model.HasAttr(nameof(SetSubscriptionDataConfigProvider)))
{
_model.InvokeMethod(nameof(SetSubscriptionDataConfigProvider), subscriptionDataConfigProvider).Dispose();
}
}
}
}