chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
@@ -0,0 +1,91 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using NodaTime;
using QuantConnect.Data;
using QuantConnect.Interfaces;
using QuantConnect.Lean.Engine.DataFeeds;
using HistoryRequest = QuantConnect.Data.HistoryRequest;
namespace QuantConnect.Lean.Engine.HistoricalData
{
/// <summary>
/// Provides an implementation of <see cref="IHistoryProvider"/> that relies on
/// a brokerage connection to retrieve historical data
/// </summary>
public class BrokerageHistoryProvider : SynchronizingHistoryProvider
{
private IDataPermissionManager _dataPermissionManager;
private IBrokerage _brokerage;
private bool _initialized;
/// <summary>
/// Sets the brokerage to be used for historical requests
/// </summary>
/// <param name="brokerage">The brokerage instance</param>
public void SetBrokerage(IBrokerage brokerage)
{
_brokerage = brokerage;
}
/// <summary>
/// Initializes this history provider to work for the specified job
/// </summary>
/// <param name="parameters">The initialization parameters</param>
public override void Initialize(HistoryProviderInitializeParameters parameters)
{
if (_initialized)
{
// let's make sure no one tries to change our parameters values
throw new InvalidOperationException("BrokerageHistoryProvider can only be initialized once");
}
_initialized = true;
_brokerage.Connect();
AlgorithmSettings = parameters.AlgorithmSettings;
_dataPermissionManager = parameters.DataPermissionManager;
}
/// <summary>
/// Gets the history for the requested securities
/// </summary>
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
// create subscription objects from the configs
var subscriptions = new List<Subscription>();
foreach (var request in requests)
{
var history = _brokerage.GetHistory(request);
if (history == null)
{
// doesn't support this history request, that's okay
continue;
}
var subscription = CreateSubscription(request, history);
subscriptions.Add(subscription);
}
if (subscriptions.Count == 0)
{
return null;
}
return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone);
}
}
}
@@ -0,0 +1,106 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using NodaTime;
using System.Linq;
using QuantConnect.Data;
using QuantConnect.Interfaces;
using QuantConnect.Data.Market;
using System.Collections.Generic;
using QuantConnect.Data.Auxiliary;
using QuantConnect.Lean.Engine.DataFeeds.Queues;
namespace QuantConnect.Lean.Engine.HistoricalData
{
/// <summary>
/// Provides FAKE implementation of <see cref="IHistoryProvider"/> used for testing. <see cref="FakeDataQueue"/>
/// </summary>
public class FakeHistoryProvider : HistoryProviderBase
{
private int _historyCount;
/// <summary>
/// Gets the total number of data points emitted by this history provider
/// </summary>
public override int DataPointCount => _historyCount;
/// <summary>
/// Initializes this history provider to work for the specified job
/// </summary>
/// <param name="parameters">The initialization parameters</param>
public override void Initialize(HistoryProviderInitializeParameters parameters)
{
}
/// <summary>
/// Gets the history for the requested securities
/// </summary>
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
var single = requests.FirstOrDefault();
if (single == null)
{
yield break;
}
var currentLocalTime = single.StartTimeLocal;
while (currentLocalTime < single.EndTimeLocal)
{
if (single.ExchangeHours.IsOpen(currentLocalTime, single.IncludeExtendedMarketHours))
{
_historyCount++;
BaseData data;
if (single.DataType == typeof(TradeBar))
{
data = new TradeBar
{
Symbol = single.Symbol,
Time = currentLocalTime,
Open = _historyCount,
Low = _historyCount,
High = _historyCount,
Close = _historyCount,
Volume = _historyCount,
Period = single.Resolution.ToTimeSpan()
};
}
else if (single.DataType == typeof(QuoteBar))
{
data = new QuoteBar
{
Symbol = single.Symbol,
Time = currentLocalTime,
Ask = new Bar(_historyCount, _historyCount, _historyCount, _historyCount),
Bid = new Bar(_historyCount, _historyCount, _historyCount, _historyCount),
Period = single.Resolution.ToTimeSpan()
};
}
else
{
yield break;
}
yield return new Slice(data.EndTime, new BaseData[] { data }, data.EndTime.ConvertFromUtc(single.ExchangeHours.TimeZone));
}
currentLocalTime = currentLocalTime.Add(single.Resolution.ToTimeSpan());
}
}
}
}
@@ -0,0 +1,218 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using NodaTime;
using QuantConnect.Configuration;
using QuantConnect.Data;
using QuantConnect.Interfaces;
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
using QuantConnect.Logging;
using QuantConnect.Packets;
using QuantConnect.Util;
using System;
using System.Collections.Generic;
using System.Linq;
using HistoryRequest = QuantConnect.Data.HistoryRequest;
namespace QuantConnect.Lean.Engine.HistoricalData
{
/// <summary>
/// Provides an implementation of <see cref="IHistoryProvider"/> which
/// acts as a wrapper to use multiple history providers together
/// </summary>
public class HistoryProviderManager : HistoryProviderBase
{
private AlgorithmNodePacket _job;
private IDataPermissionManager _dataPermissionManager;
private IBrokerage _brokerage;
private bool _initialized;
private bool _loggedEquityShortcutWarning;
/// <summary>
/// Collection of history providers being used
/// </summary>
/// <remarks>Protected for testing purposes</remarks>
private List<IHistoryProvider> _historyProviders = new();
/// <summary>
/// Gets the total number of data points emitted by this history provider
/// </summary>
public override int DataPointCount => GetDataPointCount();
/// <summary>
/// Sets the brokerage to be used for historical requests
/// </summary>
/// <param name="brokerage">The brokerage instance</param>
public void SetBrokerage(IBrokerage brokerage)
{
_brokerage = brokerage;
}
/// <summary>
/// Initializes this history provider to work for the specified job
/// </summary>
/// <param name="parameters">The initialization parameters</param>
public override void Initialize(HistoryProviderInitializeParameters parameters)
{
if (_initialized)
{
// let's make sure no one tries to change our parameters values
throw new InvalidOperationException("BrokerageHistoryProvider can only be initialized once");
}
_initialized = true;
_job = parameters.Job;
var dataProvidersList = parameters.Job?.HistoryProvider.DeserializeList() ?? new List<string>();
if (dataProvidersList.IsNullOrEmpty())
{
dataProvidersList.AddRange(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider").DeserializeList());
}
_dataPermissionManager = parameters.DataPermissionManager;
foreach (var historyProviderName in dataProvidersList)
{
IHistoryProvider historyProvider;
if (HistoryExtensions.TryGetBrokerageName(historyProviderName, out var brokerageName))
{
// we get the data queue handler if it already exists
var dataQueueHandler = Composer.Instance.GetPart<IDataQueueHandler>((x) => x.GetType().Name == brokerageName);
if (dataQueueHandler == null)
{
// we need to create the brokerage/data queue handler
dataQueueHandler = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(brokerageName);
// initialize it
dataQueueHandler.SetJob((Packets.LiveNodePacket)parameters.Job);
Log.Trace($"HistoryProviderManager.Initialize(): Created and wrapped '{brokerageName}' as '{typeof(BrokerageHistoryProvider).Name}'");
}
else
{
Log.Trace($"HistoryProviderManager.Initialize(): Wrapping '{brokerageName}' instance as '{typeof(BrokerageHistoryProvider).Name}'");
}
// wrap it
var brokerageHistoryProvider = new BrokerageHistoryProvider();
brokerageHistoryProvider.SetBrokerage((IBrokerage)dataQueueHandler);
historyProvider = brokerageHistoryProvider;
}
else
{
historyProvider = Composer.Instance.GetExportedValueByTypeName<IHistoryProvider>(historyProviderName);
if (historyProvider is BrokerageHistoryProvider)
{
(historyProvider as BrokerageHistoryProvider).SetBrokerage(_brokerage);
}
}
historyProvider.Initialize(parameters);
historyProvider.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
historyProvider.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
historyProvider.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
historyProvider.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
historyProvider.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
_historyProviders.Add(historyProvider);
}
Log.Trace($"HistoryProviderManager.Initialize(): history providers [{string.Join(",", _historyProviders.Select(x => x.GetType().Name))}]");
}
/// <summary>
/// Gets the history for the requested securities
/// </summary>
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
List<IEnumerator<Slice>> historyEnumerators = new(_historyProviders.Count);
var historyRequests = new List<HistoryRequest>();
foreach (var request in requests)
{
var config = request.ToSubscriptionDataConfig();
_dataPermissionManager?.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal);
historyRequests.Add(request);
}
foreach (var historyProvider in _historyProviders)
{
try
{
var history = historyProvider.GetHistory(historyRequests, sliceTimeZone);
if (history == null)
{
// doesn't support this history request, that's okay
continue;
}
historyEnumerators.Add(history.GetEnumerator());
if (_job != null && _job.DeploymentTarget == DeploymentTarget.CloudPlatform
&& _historyProviders.Count > 1 && historyRequests.All(x => x.Symbol.SecurityType == SecurityType.Equity))
{
if (!_loggedEquityShortcutWarning)
{
_loggedEquityShortcutWarning = true;
Log.Trace($"HistoryProviderManager.GetHistory(): using {_historyProviders[0].GetType().Name} provider for equity," +
$" skipping: [{string.Join(",", _historyProviders.Skip(1).Select(x => x.GetType().Name))}]");
}
break;
}
}
catch (Exception e)
{
// ignore
}
}
using var synchronizer = new SynchronizingSliceEnumerator(historyEnumerators);
Slice latestMergeSlice = null;
while (synchronizer.MoveNext())
{
if (synchronizer.Current == null)
{
continue;
}
if (latestMergeSlice == null)
{
latestMergeSlice = synchronizer.Current;
continue;
}
if (synchronizer.Current.UtcTime > latestMergeSlice.UtcTime)
{
// a newer slice we emit the old and keep a reference of the new
// so in the next loop we merge if required
yield return latestMergeSlice;
latestMergeSlice = synchronizer.Current;
}
else
{
// a new slice with same time we merge them into 'latestMergeSlice'
latestMergeSlice.MergeSlice(synchronizer.Current);
}
}
if (latestMergeSlice != null)
{
yield return latestMergeSlice;
}
}
private int GetDataPointCount()
{
var dataPointCount = 0;
foreach (var historyProvider in _historyProviders)
{
dataPointCount += historyProvider.DataPointCount;
}
return dataPointCount;
}
}
}
@@ -0,0 +1,78 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using NodaTime;
using System.Linq;
using QuantConnect.Data;
using QuantConnect.Util;
using QuantConnect.Interfaces;
using System.Collections.Generic;
using QuantConnect.Lean.Engine.DataFeeds;
namespace QuantConnect.Lean.Engine.HistoricalData
{
/// <summary>
/// Base class for history providers that resolve symbol mappings
/// and synchronize multiple data streams into time-aligned slices.
/// </summary>
public abstract class MappedSynchronizingHistoryProvider : SynchronizingHistoryProvider
{
/// <summary>
/// Resolves map files to correctly handle current and historical ticker symbols.
/// </summary>
private static readonly Lazy<IMapFileProvider> _mapFileProvider = new(Composer.Instance.GetPart<IMapFileProvider>);
/// <summary>
/// Gets historical data for a single resolved history request.
/// Implementations should assume the symbol is already correctly mapped.
/// </summary>
/// <param name="request">The resolved history request.</param>
/// <returns>The historical data.</returns>
public abstract IEnumerable<BaseData>? GetHistory(HistoryRequest request);
/// <summary>
/// Gets the history for the requested securities
/// </summary>
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable<Slice>? GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
var subscriptions = new List<Subscription>();
foreach (var request in requests)
{
var history = request
.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider.Value)
.SelectMany(x => GetHistory(x) ?? []);
var subscription = CreateSubscription(request, history);
if (!subscription.MoveNext())
{
continue;
}
subscriptions.Add(subscription);
}
if (subscriptions.Count == 0)
{
return null;
}
// Ownership of subscription is transferred to CreateSliceEnumerableFromSubscriptions
return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone);
}
}
}
@@ -0,0 +1,137 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using NodaTime;
using QuantConnect.Data;
using QuantConnect.Data.Market;
using QuantConnect.Data.UniverseSelection;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Securities;
using System;
using System.Collections.Generic;
using System.Linq;
using HistoryRequest = QuantConnect.Data.HistoryRequest;
namespace QuantConnect.Lean.Engine.HistoricalData
{
/// <summary>
/// Implements a History provider that always return a IEnumerable of Slice with prices following a sine function
/// </summary>
public class SineHistoryProvider : HistoryProviderBase
{
private readonly SecurityChanges _securityChanges = SecurityChanges.None;
private readonly SecurityManager _securities;
/// <summary>
/// Gets the total number of data points emitted by this history provider
/// </summary>
public override int DataPointCount => 0;
/// <summary>
/// Initializes a new instance of the <see cref="SineHistoryProvider"/> class
/// </summary>
/// <param name="securities">Collection of securities that a history request can return</param>
public SineHistoryProvider(SecurityManager securities)
{
_securities = securities;
}
/// <summary>
/// Initializes this history provider to work for the specified job
/// </summary>
/// <param name="parameters">The initialization parameters</param>
public override void Initialize(HistoryProviderInitializeParameters parameters)
{
}
/// <summary>
/// Gets the history for the requested securities
/// </summary>
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
var configsByDateTime = GetSubscriptionDataConfigByDateTime(requests);
var count = configsByDateTime.Count;
var i = 0;
var timeSliceFactory = new TimeSliceFactory(sliceTimeZone);
foreach (var kvp in configsByDateTime)
{
var utcDateTime = kvp.Key;
var configs = kvp.Value;
var last = Convert.ToDecimal(100 + 10 * Math.Sin(Math.PI * (360 - count + i) / 180.0));
var high = last * 1.005m;
var low = last / 1.005m;
var packets = new List<DataFeedPacket>();
foreach (var config in configs)
{
Security security;
if (!_securities.TryGetValue(config.Symbol, out security))
{
continue;
}
var period = config.Resolution.ToTimeSpan();
var time = (utcDateTime - period).ConvertFromUtc(config.DataTimeZone);
var data = new TradeBar(time, config.Symbol, last, high, last, last, 1000, period);
security.SetMarketPrice(data);
packets.Add(new DataFeedPacket(security, config, new List<BaseData> { data }));
}
i++;
yield return timeSliceFactory.Create(utcDateTime, packets, _securityChanges, new Dictionary<Universe, BaseDataCollection>()).Slice;
}
}
private Dictionary<DateTime, List<SubscriptionDataConfig>> GetSubscriptionDataConfigByDateTime(
IEnumerable<HistoryRequest> requests)
{
var dictionary = new Dictionary<DateTime, List<SubscriptionDataConfig>>();
var barSize = requests.Select(x => x.Resolution.ToTimeSpan()).Min();
var startUtc = requests.Min(x => x.StartTimeUtc);
var endUtc = requests.Max(x => x.EndTimeUtc);
for (var utcDateTime = startUtc; utcDateTime < endUtc; utcDateTime += barSize)
{
var subscriptionDataConfig = new List<SubscriptionDataConfig>();
foreach (var request in requests)
{
var exchange = request.ExchangeHours;
var extendedMarket = request.IncludeExtendedMarketHours;
var localDateTime = utcDateTime.ConvertFromUtc(exchange.TimeZone);
if (!exchange.IsOpen(localDateTime, extendedMarket))
{
continue;
}
var config = request.ToSubscriptionDataConfig();
subscriptionDataConfig.Add(config);
}
if (subscriptionDataConfig.Count > 0)
{
dictionary.Add(utcDateTime.Add(barSize), subscriptionDataConfig);
}
}
return dictionary;
}
}
}
@@ -0,0 +1,226 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.Collections.Generic;
using NodaTime;
using QuantConnect.Data;
using QuantConnect.Data.Market;
using QuantConnect.Data.UniverseSelection;
using QuantConnect.Interfaces;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
using QuantConnect.Securities;
using QuantConnect.Util;
using HistoryRequest = QuantConnect.Data.HistoryRequest;
namespace QuantConnect.Lean.Engine.HistoricalData
{
/// <summary>
/// Provides an implementation of <see cref="IHistoryProvider"/> that uses <see cref="BaseData"/>
/// instances to retrieve historical data
/// </summary>
public class SubscriptionDataReaderHistoryProvider : SynchronizingHistoryProvider
{
private SymbolProperties _nullSymbolProperties;
private SecurityCache _nullCache;
private Cash _nullCash;
private IDataProvider _dataProvider;
private IMapFileProvider _mapFileProvider;
private IFactorFileProvider _factorFileProvider;
private IDataCacheProvider _dataCacheProvider;
private IObjectStore _objectStore;
private bool _parallelHistoryRequestsEnabled;
private bool _initialized;
/// <summary>
/// Manager used to allow or deny access to a requested datasource for specific users
/// </summary>
protected IDataPermissionManager DataPermissionManager { get; set; }
/// <summary>
/// Initializes this history provider to work for the specified job
/// </summary>
/// <param name="parameters">The initialization parameters</param>
public override void Initialize(HistoryProviderInitializeParameters parameters)
{
if (_initialized)
{
return;
}
_initialized = true;
_dataProvider = parameters.DataProvider;
_mapFileProvider = parameters.MapFileProvider;
_dataCacheProvider = parameters.DataCacheProvider;
_factorFileProvider = parameters.FactorFileProvider;
_objectStore = parameters.ObjectStore;
AlgorithmSettings = parameters.AlgorithmSettings;
DataPermissionManager = parameters.DataPermissionManager;
_parallelHistoryRequestsEnabled = parameters.ParallelHistoryRequestsEnabled;
_nullCache = new SecurityCache();
_nullCash = new Cash(Currencies.NullCurrency, 0, 1m);
_nullSymbolProperties = SymbolProperties.GetDefault(Currencies.NullCurrency);
}
/// <summary>
/// Gets the history for the requested securities
/// </summary>
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
// create subscription objects from the configs
var subscriptions = new List<Subscription>();
foreach (var request in requests)
{
var subscription = CreateSubscription(request);
subscriptions.Add(subscription);
}
return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone);
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request)
{
var config = request.ToSubscriptionDataConfig();
// this security is internal only we do not need to worry about a few of it's properties
// TODO: we don't need fee/fill/BPM/etc either. Even better we should refactor & remove the need for the security
var security = new Security(
request.ExchangeHours,
config,
_nullCash,
_nullSymbolProperties,
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
_nullCache
);
var dataReader = new SubscriptionDataReader(config,
request,
_mapFileProvider,
_factorFileProvider,
_dataCacheProvider,
_dataProvider,
_objectStore);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator<BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
var useDailyStrictEndTimes = LeanData.UseDailyStrictEndTimes(AlgorithmSettings, request, config.Symbol, config.Increment);
if (useDailyStrictEndTimes)
{
// before corporate events which might yield data and we synchronize both feeds
reader = new StrictDailyEndTimesEnumerator(reader, request.ExchangeHours, request.StartTimeLocal);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
_mapFileProvider,
request.StartTimeLocal,
request.EndTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
var exchange = GetSecurityExchange(security.Exchange, request.DataType, request.Symbol);
reader = new FillForwardEnumerator(reader, exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.StartTimeLocal, request.EndTimeLocal, config.Increment, config.DataTimeZone, useDailyStrictEndTimes, request.DataType);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours);
// allow all ticks
if (config.Resolution != Resolution.Tick)
{
var timeBasedFilter = new TimeBasedFilter(request);
reader = new FilterEnumerator<BaseData>(reader, timeBasedFilter.Filter);
}
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false, AlgorithmSettings.DailyPreciseEndTime);
}
return SubscriptionUtils.Create(subscriptionRequest, reader, AlgorithmSettings.DailyPreciseEndTime);
}
/// <summary>
/// Gets the intraday data enumerator if any
/// </summary>
protected virtual IEnumerator<BaseData> GetIntradayDataEnumerator(IEnumerator<BaseData> rawData, HistoryRequest request)
{
return null;
}
/// <summary>
/// Internal helper class to filter data based on requested times
/// </summary>
private class TimeBasedFilter
{
public Type RequestedType { get; set; }
public DateTime EndTimeLocal { get; set; }
public DateTime StartTimeLocal { get; set; }
public TimeBasedFilter(HistoryRequest request)
{
RequestedType = request.DataType;
EndTimeLocal = request.EndTimeLocal;
StartTimeLocal = request.StartTimeLocal;
}
public bool Filter(BaseData data)
{
// filter out all aux data, unless if we are asking for aux data
if (data.DataType == MarketDataType.Auxiliary && data.GetType() != RequestedType) return false;
// filter out future data
if (data.EndTime > EndTimeLocal) return false;
// filter out data before the start
return data.EndTime > StartTimeLocal;
}
}
}
}
@@ -0,0 +1,189 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using NodaTime;
using QuantConnect.Data;
using QuantConnect.Data.Market;
using QuantConnect.Data.UniverseSelection;
using QuantConnect.Interfaces;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
using QuantConnect.Securities;
using QuantConnect.Util;
namespace QuantConnect.Lean.Engine.HistoricalData
{
/// <summary>
/// Provides an abstract implementation of <see cref="IHistoryProvider"/>
/// which provides synchronization of multiple history results
/// </summary>
public abstract class SynchronizingHistoryProvider : HistoryProviderBase
{
/// <summary>
/// The market hours database
/// </summary>
protected static readonly MarketHoursDatabase MarketHours = MarketHoursDatabase.FromDataFolder();
private int _dataPointCount;
/// <summary>
/// The algorithm settings instance to use
/// </summary>
public IAlgorithmSettings AlgorithmSettings { get; set; } = new AlgorithmSettings();
/// <summary>
/// Gets the total number of data points emitted by this history provider
/// </summary>
public override int DataPointCount => _dataPointCount;
/// <summary>
/// Enumerates the subscriptions into slices
/// </summary>
protected IEnumerable<Slice> CreateSliceEnumerableFromSubscriptions(List<Subscription> subscriptions, DateTimeZone sliceTimeZone)
{
// required by TimeSlice.Create, but we don't need it's behavior
var frontier = DateTime.MinValue;
// never changes, there's no selection during a history request
var universeSelectionData = new Dictionary<Universe, BaseDataCollection>();
var timeSliceFactory = new TimeSliceFactory(sliceTimeZone);
while (true)
{
var earlyBirdTicks = long.MaxValue;
var data = new List<DataFeedPacket>();
foreach (var subscription in subscriptions.Where(subscription => !subscription.EndOfStream))
{
if (subscription.Current == null && !subscription.MoveNext())
{
// initial pump. We do it here and not when creating the subscriptions so
// that parallel workers can all start as fast as possible
continue;
}
DataFeedPacket packet = null;
while (subscription.Current.EmitTimeUtc <= frontier)
{
if (packet == null)
{
// for performance, lets be selfish about creating a new instance
packet = new DataFeedPacket(subscription.Security, subscription.Configuration);
// only add if we have data
data.Add(packet);
}
packet.Add(subscription.Current.Data);
Interlocked.Increment(ref _dataPointCount);
if (!subscription.MoveNext())
{
break;
}
}
// update our early bird ticks (next frontier time)
if (subscription.Current != null)
{
// take the earliest between the next piece of data or the next tz discontinuity
earlyBirdTicks = Math.Min(earlyBirdTicks, subscription.Current.EmitTimeUtc.Ticks);
}
}
if (data.Count != 0)
{
// reuse the slice construction code from TimeSlice.Create
yield return timeSliceFactory.Create(frontier, data, SecurityChanges.None, universeSelectionData).Slice;
}
// end of subscriptions, after we emit, else we might drop a data point
if (earlyBirdTicks == long.MaxValue) break;
frontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc);
}
// make sure we clean up after ourselves
foreach (var subscription in subscriptions)
{
subscription.Dispose();
}
}
/// <summary>
/// Retrieves the appropriate <see cref="SecurityExchange"/> based on the data type and symbol.
/// </summary>
/// <param name="exchange">The default exchange instance.</param>
/// <param name="dataType">The type of data being processed.</param>
/// <param name="symbol">The security symbol.</param>
/// <returns>The security exchange with appropriate market hours.</returns>
protected static SecurityExchange GetSecurityExchange(SecurityExchange exchange, Type dataType, Symbol symbol)
{
if (dataType == typeof(OpenInterest))
{
// Retrieve the original market hours, which include holidays and closed days.
var originalExchangeHours = MarketHours.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
// Use the original market hours to prevent fill-forwarding on non-trading hours.
return new SecurityExchange(originalExchangeHours);
}
return exchange;
}
/// <summary>
/// Creates a subscription to process the history request
/// </summary>
protected Subscription CreateSubscription(HistoryRequest request, IEnumerable<BaseData> history)
{
var config = request.ToSubscriptionDataConfig();
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var reader = history.GetEnumerator();
var useDailyStrictEndTimes = LeanData.UseDailyStrictEndTimes(AlgorithmSettings, request, config.Symbol, config.Increment);
if (useDailyStrictEndTimes)
{
reader = new StrictDailyEndTimesEnumerator(reader, request.ExchangeHours, request.StartTimeLocal);
}
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// FillForwardEnumerator expects these values in local times
var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
var exchange = GetSecurityExchange(security.Exchange, request.DataType, request.Symbol);
reader = new FillForwardEnumerator(reader, exchange, readOnlyRef, request.IncludeExtendedMarketHours, start, end, config.Increment, config.DataTimeZone, useDailyStrictEndTimes, request.DataType);
}
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
return SubscriptionUtils.Create(subscriptionRequest, reader, AlgorithmSettings.DailyPreciseEndTime);
}
}
}