wehub-resource-sync
418 lines
19 KiB
C#
418 lines
19 KiB
C#
/*
|
|
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
|
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
using Deedle;
|
|
using QuantConnect.Orders;
|
|
using QuantConnect.Brokerages.Backtesting;
|
|
using QuantConnect.Data;
|
|
using QuantConnect.Data.Market;
|
|
using QuantConnect.Interfaces;
|
|
using QuantConnect.Lean.Engine.DataFeeds;
|
|
using QuantConnect.Lean.Engine.Results;
|
|
using QuantConnect.Lean.Engine.Setup;
|
|
using QuantConnect.Lean.Engine.TransactionHandlers;
|
|
using QuantConnect.Logging;
|
|
using QuantConnect.Packets;
|
|
using QuantConnect.Securities;
|
|
using QuantConnect.Util;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using QuantConnect.Lean.Engine.HistoricalData;
|
|
|
|
namespace QuantConnect.Report
|
|
{
|
|
/// <summary>
|
|
/// Runs LEAN to calculate the portfolio at a given time from <see cref="Order"/> objects.
|
|
/// Generates and returns <see cref="PointInTimePortfolio"/> objects that represents
|
|
/// the holdings and other miscellaneous metrics at a point in time by reprocessing the orders
|
|
/// as they were filled.
|
|
/// </summary>
|
|
public class PortfolioLooper : IDisposable
|
|
{
|
|
/// <summary>
|
|
/// Default resolution to read. This will affect the granularity of the results generated for FX and Crypto
|
|
/// </summary>
|
|
private const Resolution _resolution = Resolution.Hour;
|
|
|
|
private SecurityService _securityService;
|
|
private DataManager _dataManager;
|
|
private IResultHandler _resultHandler;
|
|
private IDataCacheProvider _cacheProvider;
|
|
private IEnumerable<Slice> _conversionSlices = new List<Slice>();
|
|
|
|
/// <summary>
|
|
/// QCAlgorithm derived class that sets up internal data feeds for
|
|
/// use with crypto and forex data, as well as managing the <see cref="SecurityPortfolioManager"/>
|
|
/// </summary>
|
|
public PortfolioLooperAlgorithm Algorithm { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// Creates an instance of the PortfolioLooper class
|
|
/// </summary>
|
|
/// <param name="startingCash">Equity curve</param>
|
|
/// <param name="orders">Order events</param>
|
|
/// <param name="resolution">Optional parameter to override default resolution (Hourly)</param>
|
|
/// <param name="algorithmConfiguration">Optional parameter to override default algorithm configuration</param>
|
|
private PortfolioLooper(double startingCash, List<Order> orders, Resolution resolution = _resolution,
|
|
AlgorithmConfiguration algorithmConfiguration = null)
|
|
{
|
|
// Initialize the providers that the HistoryProvider requires
|
|
var factorFileProvider = Composer.Instance.GetExportedValueByTypeName<IFactorFileProvider>("LocalDiskFactorFileProvider");
|
|
var mapFileProvider = Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>("LocalDiskMapFileProvider");
|
|
_cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), false);
|
|
var historyProvider = new SubscriptionDataReaderHistoryProvider();
|
|
|
|
Algorithm = new PortfolioLooperAlgorithm((decimal)startingCash, orders, algorithmConfiguration);
|
|
var dataPermissionManager = new DataPermissionManager();
|
|
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, _cacheProvider, mapFileProvider, factorFileProvider, (_) => { }, false, dataPermissionManager, Algorithm.ObjectStore, Algorithm.Settings));
|
|
Algorithm.SetHistoryProvider(historyProvider);
|
|
|
|
// Dummy LEAN datafeed classes and initializations that essentially do nothing
|
|
var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"");
|
|
var feed = new MockDataFeed();
|
|
|
|
// Create MHDB and Symbol properties DB instances for the DataManager
|
|
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
|
|
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
|
|
_dataManager = new DataManager(feed,
|
|
new UniverseSelection(
|
|
Algorithm,
|
|
new SecurityService(Algorithm.Portfolio.CashBook,
|
|
marketHoursDatabase,
|
|
symbolPropertiesDataBase,
|
|
Algorithm,
|
|
RegisteredSecurityDataTypesProvider.Null,
|
|
new SecurityCacheProvider(Algorithm.Portfolio),
|
|
algorithm: Algorithm),
|
|
dataPermissionManager,
|
|
new DefaultDataProvider()),
|
|
Algorithm,
|
|
Algorithm.TimeKeeper,
|
|
marketHoursDatabase,
|
|
false,
|
|
RegisteredSecurityDataTypesProvider.Null,
|
|
dataPermissionManager);
|
|
|
|
_securityService = new SecurityService(Algorithm.Portfolio.CashBook,
|
|
marketHoursDatabase,
|
|
symbolPropertiesDataBase,
|
|
Algorithm,
|
|
RegisteredSecurityDataTypesProvider.Null,
|
|
new SecurityCacheProvider(Algorithm.Portfolio),
|
|
algorithm: Algorithm);
|
|
|
|
var transactions = new BacktestingTransactionHandler();
|
|
_resultHandler = new BacktestingResultHandler();
|
|
|
|
// Initialize security services and other properties so that we
|
|
// don't get null reference exceptions during our re-calculation
|
|
Algorithm.Securities.SetSecurityService(_securityService);
|
|
Algorithm.SubscriptionManager.SetDataManager(_dataManager);
|
|
|
|
// Initialize the algorithm before adding any securities
|
|
Algorithm.Initialize();
|
|
Algorithm.PostInitialize();
|
|
|
|
// Initializes all the proper Securities from the orders provided by the user
|
|
Algorithm.FromOrders(orders);
|
|
|
|
// More initialization, this time with Algorithm and other misc. classes
|
|
_resultHandler.Initialize(new (job, new Messaging.Messaging(), new Api.Api(), transactions, mapFileProvider));
|
|
_resultHandler.SetAlgorithm(Algorithm, Algorithm.Portfolio.TotalPortfolioValue);
|
|
|
|
Algorithm.Transactions.SetOrderProcessor(transactions);
|
|
|
|
transactions.Initialize(Algorithm, new BacktestingBrokerage(Algorithm), _resultHandler);
|
|
feed.Initialize(Algorithm, job, _resultHandler, null, null, null, _dataManager, null, null);
|
|
|
|
// Begin setting up the currency conversion feed if needed
|
|
var coreSecurities = Algorithm.Securities.Values.ToList();
|
|
|
|
BaseSetupHandler.SetupCurrencyConversions(Algorithm, _dataManager.UniverseSelection);
|
|
var conversionSecurities = Algorithm.Securities.Values.Where(s => !coreSecurities.Contains(s)).ToList();
|
|
|
|
// Skip the history request if we don't need to convert anything
|
|
if (conversionSecurities.Any())
|
|
{
|
|
// Point-in-time Slices to convert FX and Crypto currencies to the portfolio currency
|
|
_conversionSlices = GetHistory(Algorithm, conversionSecurities, resolution);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
|
/// </summary>
|
|
public void Dispose()
|
|
{
|
|
_dataManager.RemoveAllSubscriptions();
|
|
_cacheProvider.DisposeSafely();
|
|
_resultHandler.Exit();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Internal method to get the history for the given securities
|
|
/// </summary>
|
|
/// <param name="algorithm">Algorithm</param>
|
|
/// <param name="securities">Securities to get history for</param>
|
|
/// <param name="resolution">Resolution to retrieve data in</param>
|
|
/// <returns>History of the given securities</returns>
|
|
/// <remarks>Method is static because we want to use it from the constructor as well</remarks>
|
|
private static IEnumerable<Slice> GetHistory(IAlgorithm algorithm, List<Security> securities, Resolution resolution)
|
|
{
|
|
var historyRequests = new List<Data.HistoryRequest>();
|
|
var historyRequestFactory = new HistoryRequestFactory(algorithm);
|
|
|
|
// Create the history requests
|
|
foreach (var security in securities)
|
|
{
|
|
var configs = algorithm.SubscriptionManager
|
|
.SubscriptionDataConfigService
|
|
.GetSubscriptionDataConfigs(security.Symbol, includeInternalConfigs: true);
|
|
|
|
// we need to order and select a specific configuration type
|
|
// so the conversion rate is deterministic
|
|
var configToUse = configs.OrderBy(x => x.TickType).First();
|
|
|
|
var startTime = historyRequestFactory.GetStartTimeAlgoTz(
|
|
security.Symbol,
|
|
1,
|
|
resolution,
|
|
security.Exchange.Hours,
|
|
configToUse.DataTimeZone,
|
|
configToUse.Type);
|
|
var endTime = algorithm.EndDate;
|
|
|
|
historyRequests.Add(historyRequestFactory.CreateHistoryRequest(
|
|
configToUse,
|
|
startTime,
|
|
endTime,
|
|
security.Exchange.Hours,
|
|
resolution
|
|
));
|
|
}
|
|
|
|
return algorithm.HistoryProvider.GetHistory(historyRequests, algorithm.TimeZone).ToList();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the history for the given symbols from the <paramref name="start"/> to the <paramref name="end"/>
|
|
/// </summary>
|
|
/// <param name="symbols">Symbols to request history for</param>
|
|
/// <param name="start">Start date of history request</param>
|
|
/// <param name="end">End date of history request</param>
|
|
/// <param name="resolution">Resolution of history request</param>
|
|
/// <returns>Enumerable of slices</returns>
|
|
public static IEnumerable<Slice> GetHistory(List<Symbol> symbols, DateTime start, DateTime end, Resolution resolution)
|
|
{
|
|
// Handles the conversion of Symbol to Security for us.
|
|
var looper = new PortfolioLooper(0, new List<Order>(), resolution);
|
|
var securities = new List<Security>();
|
|
|
|
looper.Algorithm.SetStartDate(start);
|
|
looper.Algorithm.SetEndDate(end);
|
|
|
|
foreach (var symbol in symbols)
|
|
{
|
|
var configs = looper.Algorithm.SubscriptionManager.SubscriptionDataConfigService.Add(symbol, resolution, false, false);
|
|
securities.Add(looper.Algorithm.Securities.CreateSecurity(symbol, configs));
|
|
}
|
|
|
|
return GetHistory(looper.Algorithm, securities, resolution);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the point in time portfolio over multiple deployments
|
|
/// </summary>
|
|
/// <param name="equityCurve">Equity curve series</param>
|
|
/// <param name="orders">Orders</param>
|
|
/// <param name="algorithmConfiguration">Optional parameter to override default algorithm configuration</param>
|
|
/// <param name="liveSeries">Equity curve series originates from LiveResult</param>
|
|
/// <returns>Enumerable of <see cref="PointInTimePortfolio"/></returns>
|
|
public static IEnumerable<PointInTimePortfolio> FromOrders(Series<DateTime, double> equityCurve, IEnumerable<Order> orders,
|
|
AlgorithmConfiguration algorithmConfiguration = null, bool liveSeries = false)
|
|
{
|
|
// Don't do anything if we have no orders or equity curve to process
|
|
if (!orders.Any() || equityCurve.IsEmpty)
|
|
{
|
|
yield break;
|
|
}
|
|
|
|
// Chunk different deployments into separate Lists for separate processing
|
|
var portfolioDeployments = new List<List<Order>>();
|
|
|
|
// Orders are guaranteed to start counting from 1. This ensures that we have
|
|
// no collision at all with the start of a deployment
|
|
var previousOrderId = 0;
|
|
var currentDeployment = new List<Order>();
|
|
|
|
// Make use of reference semantics to add new deployments to the list
|
|
portfolioDeployments.Add(currentDeployment);
|
|
|
|
foreach (var order in orders)
|
|
{
|
|
// In case we have two different deployments with only a single
|
|
// order in the deployments, <= was chosen because it covers duplicate values
|
|
if (order.Id <= previousOrderId)
|
|
{
|
|
currentDeployment = new List<Order>();
|
|
portfolioDeployments.Add(currentDeployment);
|
|
}
|
|
|
|
currentDeployment.Add(order);
|
|
previousOrderId = order.Id;
|
|
}
|
|
|
|
PortfolioLooper looper = null;
|
|
PointInTimePortfolio prev = null;
|
|
foreach (var deploymentOrders in portfolioDeployments)
|
|
{
|
|
if (deploymentOrders.Count == 0)
|
|
{
|
|
Log.Trace($"PortfolioLooper.FromOrders(): Deployment contains no orders");
|
|
continue;
|
|
}
|
|
var startTime = deploymentOrders.First().Time;
|
|
var deployment = equityCurve.Where(kvp => kvp.Key <= startTime);
|
|
if (deployment.IsEmpty)
|
|
{
|
|
Log.Trace($"PortfolioLooper.FromOrders(): Equity series is empty after filtering with upper bound: {startTime}");
|
|
continue;
|
|
}
|
|
|
|
// Skip any deployments that haven't been ran long enough to be generated in live mode
|
|
if (liveSeries && deploymentOrders.First().Time.Date == deploymentOrders.Last().Time.Date)
|
|
{
|
|
Log.Trace("PortfolioLooper.FromOrders(): Filtering deployment because it has not been deployed for more than one day");
|
|
continue;
|
|
}
|
|
|
|
// For every deployment, we want to start fresh.
|
|
looper = new PortfolioLooper(deployment.LastValue(), deploymentOrders, algorithmConfiguration: algorithmConfiguration);
|
|
|
|
foreach (var portfolio in looper.ProcessOrders(deploymentOrders))
|
|
{
|
|
prev = portfolio;
|
|
yield return portfolio;
|
|
}
|
|
}
|
|
|
|
if (prev != null)
|
|
{
|
|
yield return new PointInTimePortfolio(prev, equityCurve.LastKey());
|
|
}
|
|
|
|
looper.DisposeSafely();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Process the orders
|
|
/// </summary>
|
|
/// <param name="orders">orders</param>
|
|
/// <returns>PointInTimePortfolio</returns>
|
|
private IEnumerable<PointInTimePortfolio> ProcessOrders(IEnumerable<Order> orders)
|
|
{
|
|
// Portfolio.ProcessFill(...) does not filter out invalid orders. We must do so ourselves
|
|
foreach (var order in orders)
|
|
{
|
|
Algorithm.SetDateTime(order.Time);
|
|
|
|
var orderSecurity = Algorithm.Securities[order.Symbol];
|
|
DateTime lastFillTime;
|
|
|
|
if ((order.Type == OrderType.MarketOnOpen || order.Type == OrderType.MarketOnClose) &&
|
|
(order.Status == OrderStatus.Filled || order.Status == OrderStatus.PartiallyFilled) && order.LastFillTime == null)
|
|
{
|
|
lastFillTime = order.Time;
|
|
}
|
|
else if (order.LastFillTime == null)
|
|
{
|
|
Log.Trace($"Order with ID: {order.Id} has been skipped because of null LastFillTime");
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
lastFillTime = order.LastFillTime.Value;
|
|
}
|
|
|
|
var tick = new Tick { Quantity = order.Quantity, AskPrice = order.Price, BidPrice = order.Price, Value = order.Price, EndTime = lastFillTime };
|
|
var tradeBar = new TradeBar
|
|
{
|
|
Open = order.Price,
|
|
High = order.Price,
|
|
Low = order.Price,
|
|
Close = order.Price,
|
|
Volume = order.Quantity,
|
|
|
|
DataType = MarketDataType.TradeBar,
|
|
Period = TimeSpan.Zero,
|
|
Symbol = order.Symbol,
|
|
Time = lastFillTime,
|
|
};
|
|
|
|
// Required for crypto so that the Cache Price is updated accordingly,
|
|
// since its `Security.Price` implementation explicitly requests TradeBars.
|
|
// For most asset types this might be enough as well, but there is the
|
|
// possibility that some trades might get filtered, so we cover that
|
|
// case by setting the market price via Tick as well.
|
|
orderSecurity.SetMarketPrice(tradeBar);
|
|
orderSecurity.SetMarketPrice(tick);
|
|
|
|
// Check if we have a base currency (i.e. forex or crypto that requires currency conversion)
|
|
// to ensure the proper conversion rate is set for them
|
|
var baseCurrency = orderSecurity as IBaseCurrencySymbol;
|
|
|
|
if (baseCurrency != null)
|
|
{
|
|
// We want slices that apply to either this point in time, or the last most recent point in time
|
|
var updateSlices = _conversionSlices.Where(x => x.Time <= order.Time).ToList();
|
|
|
|
// This is put here because there can potentially be no slices
|
|
if (updateSlices.Count != 0)
|
|
{
|
|
var updateSlice = updateSlices.Last();
|
|
|
|
foreach (var quoteBar in updateSlice.QuoteBars.Values)
|
|
{
|
|
Algorithm.Securities[quoteBar.Symbol].SetMarketPrice(quoteBar);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Update our cash holdings before we invalidate the portfolio value
|
|
// to calculate the proper cash value of other assets the algo owns
|
|
foreach (var cash in Algorithm.Portfolio.CashBook.Values.Where(x => x.CurrencyConversion != null))
|
|
{
|
|
cash.Update();
|
|
}
|
|
|
|
// Securities prices might have been updated, so we need to recalculate how much
|
|
// money we have in our portfolio, otherwise we risk being out of date and
|
|
// calculate on stale data.
|
|
Algorithm.Portfolio.InvalidateTotalPortfolioValue();
|
|
|
|
var ticket = order.ToOrderTicket(Algorithm.Transactions);
|
|
var orderEvent = new OrderEvent(order, order.Time, Orders.Fees.OrderFee.Zero) { FillPrice = order.Price, FillQuantity = order.Quantity, Ticket = ticket };
|
|
|
|
// Process the order
|
|
Algorithm.Portfolio.ProcessFills(new List<OrderEvent> { orderEvent });
|
|
|
|
// Create portfolio statistics and return back to the user
|
|
yield return new PointInTimePortfolio(order, Algorithm.Portfolio);
|
|
}
|
|
}
|
|
}
|
|
}
|