wehub-resource-sync
614 lines
32 KiB
C#
614 lines
32 KiB
C#
/*
|
|
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
|
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*
|
|
*/
|
|
|
|
using System;
|
|
using System.Linq;
|
|
using QuantConnect.Data;
|
|
using QuantConnect.Util;
|
|
using QuantConnect.Logging;
|
|
using QuantConnect.Packets;
|
|
using QuantConnect.Interfaces;
|
|
using QuantConnect.Securities;
|
|
using System.Collections.Generic;
|
|
using QuantConnect.Configuration;
|
|
using QuantConnect.Data.Auxiliary;
|
|
using QuantConnect.Data.Custom.Tiingo;
|
|
using QuantConnect.Lean.Engine.Results;
|
|
using QuantConnect.Data.UniverseSelection;
|
|
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
|
using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
|
|
using QuantConnect.Data.Fundamental;
|
|
|
|
namespace QuantConnect.Lean.Engine.DataFeeds
|
|
{
|
|
/// <summary>
|
|
/// Provides an implementation of <see cref="IDataFeed"/> that is designed to deal with
|
|
/// live, remote data sources
|
|
/// </summary>
|
|
public class LiveTradingDataFeed : FileSystemDataFeed
|
|
{
|
|
private static readonly int MaximumWarmupHistoryDaysLookBack = Config.GetInt("maximum-warmup-history-days-look-back", 5);
|
|
|
|
private LiveNodePacket _job;
|
|
|
|
// used to get current time
|
|
private ITimeProvider _timeProvider;
|
|
private IAlgorithm _algorithm;
|
|
private ITimeProvider _frontierTimeProvider;
|
|
private IDataProvider _dataProvider;
|
|
private IMapFileProvider _mapFileProvider;
|
|
private IDataQueueHandler _dataQueueHandler;
|
|
private BaseDataExchange _customExchange;
|
|
private SubscriptionCollection _subscriptions;
|
|
private IFactorFileProvider _factorFileProvider;
|
|
private IDataChannelProvider _channelProvider;
|
|
private readonly HashSet<string> _unsupportedConfigurations = new();
|
|
|
|
// in live trading we delay scheduled universe selection to 8 am NY, but NY goes from -4/-5 UTC time, so we adjust it
|
|
private static ReferenceWrapper<DateTime> _lastUtcDateShiftUpdate;
|
|
private static ReferenceWrapper<TimeSpan> _scheduledUniverseUtcTimeShift;
|
|
|
|
/// <summary>
|
|
/// Public flag indicator that the thread is still busy.
|
|
/// </summary>
|
|
public bool IsActive
|
|
{
|
|
get; private set;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Initializes the data feed for the specified job and algorithm
|
|
/// </summary>
|
|
public override void Initialize(IAlgorithm algorithm,
|
|
AlgorithmNodePacket job,
|
|
IResultHandler resultHandler,
|
|
IMapFileProvider mapFileProvider,
|
|
IFactorFileProvider factorFileProvider,
|
|
IDataProvider dataProvider,
|
|
IDataFeedSubscriptionManager subscriptionManager,
|
|
IDataFeedTimeProvider dataFeedTimeProvider,
|
|
IDataChannelProvider dataChannelProvider)
|
|
{
|
|
if (!(job is LiveNodePacket))
|
|
{
|
|
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
|
|
}
|
|
|
|
_algorithm = algorithm;
|
|
_job = (LiveNodePacket)job;
|
|
_timeProvider = dataFeedTimeProvider.TimeProvider;
|
|
_dataProvider = dataProvider;
|
|
_mapFileProvider = mapFileProvider;
|
|
_factorFileProvider = factorFileProvider;
|
|
_channelProvider = dataChannelProvider;
|
|
_frontierTimeProvider = dataFeedTimeProvider.FrontierTimeProvider;
|
|
_customExchange = GetBaseDataExchange();
|
|
_subscriptions = subscriptionManager.DataFeedSubscriptions;
|
|
|
|
_dataQueueHandler = GetDataQueueHandler();
|
|
_dataQueueHandler?.SetJob(_job);
|
|
|
|
// run the custom data exchange
|
|
_customExchange.Start();
|
|
|
|
IsActive = true;
|
|
|
|
base.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, subscriptionManager, dataFeedTimeProvider, dataChannelProvider);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a new subscription to provide data for the specified security.
|
|
/// </summary>
|
|
/// <param name="request">Defines the subscription to be added, including start/end times the universe and security</param>
|
|
/// <returns>The created <see cref="Subscription"/> if successful, null otherwise</returns>
|
|
public override Subscription CreateSubscription(SubscriptionRequest request)
|
|
{
|
|
Subscription subscription = null;
|
|
try
|
|
{
|
|
// create and add the subscription to our collection
|
|
subscription = request.IsUniverseSubscription
|
|
? CreateUniverseSubscription(request)
|
|
: CreateDataSubscription(request);
|
|
}
|
|
catch (Exception err)
|
|
{
|
|
Log.Error(err, $"CreateSubscription(): Failed configuration: '{request.Configuration}'");
|
|
// kill the algorithm, this shouldn't happen
|
|
_algorithm.SetRuntimeError(err, $"Failed to subscribe to {request.Configuration.Symbol}");
|
|
}
|
|
|
|
return subscription;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Removes the subscription from the data feed, if it exists
|
|
/// </summary>
|
|
/// <param name="subscription">The subscription to remove</param>
|
|
public override void RemoveSubscription(Subscription subscription)
|
|
{
|
|
var symbol = subscription.Configuration.Symbol;
|
|
|
|
// remove the subscriptions
|
|
if (!_channelProvider.ShouldStreamSubscription(subscription.Configuration))
|
|
{
|
|
_customExchange.RemoveEnumerator(symbol);
|
|
}
|
|
else
|
|
{
|
|
_dataQueueHandler.UnsubscribeWithMapping(subscription.Configuration);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// External controller calls to signal a terminate of the thread.
|
|
/// </summary>
|
|
public override void Exit()
|
|
{
|
|
if (IsActive)
|
|
{
|
|
IsActive = false;
|
|
Log.Trace("LiveTradingDataFeed.Exit(): Start. Setting cancellation token...");
|
|
if (_dataQueueHandler is DataQueueHandlerManager manager)
|
|
{
|
|
manager.UnsupportedConfiguration -= HandleUnsupportedConfigurationEvent;
|
|
}
|
|
_customExchange?.Stop();
|
|
Log.Trace("LiveTradingDataFeed.Exit(): Exit Finished.");
|
|
|
|
base.Exit();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the <see cref="IDataQueueHandler"/> to use by default <see cref="DataQueueHandlerManager"/>
|
|
/// </summary>
|
|
/// <remarks>Useful for testing</remarks>
|
|
/// <returns>The loaded <see cref="IDataQueueHandler"/></returns>
|
|
protected virtual IDataQueueHandler GetDataQueueHandler()
|
|
{
|
|
var result = new DataQueueHandlerManager(_algorithm.Settings);
|
|
result.UnsupportedConfiguration += HandleUnsupportedConfigurationEvent;
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the <see cref="BaseDataExchange"/> to use
|
|
/// </summary>
|
|
/// <remarks>Useful for testing</remarks>
|
|
protected virtual BaseDataExchange GetBaseDataExchange()
|
|
{
|
|
return new BaseDataExchange("CustomDataExchange") { SleepInterval = 100 };
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a new subscription for the specified security
|
|
/// </summary>
|
|
/// <param name="request">The subscription request</param>
|
|
/// <returns>A new subscription instance of the specified security</returns>
|
|
private Subscription CreateDataSubscription(SubscriptionRequest request)
|
|
{
|
|
Subscription subscription = null;
|
|
|
|
// let's keep track of the last point we got from the file based enumerator and start our history enumeration from this point
|
|
// this is much more efficient since these duplicated points will be dropped by the filter righ away causing memory usage spikes
|
|
var lastPointTracker = new LastPointTracker();
|
|
|
|
var localStartTime = request.StartTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
|
|
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
|
|
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
|
|
|
|
IEnumerator<BaseData> enumerator = null;
|
|
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
|
|
{
|
|
if (!Tiingo.IsAuthCodeSet)
|
|
{
|
|
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
|
|
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
|
|
}
|
|
|
|
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _algorithm.ObjectStore);
|
|
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
|
|
|
|
var enqueable = new EnqueueableEnumerator<BaseData>();
|
|
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack, handleData: data =>
|
|
{
|
|
enqueable.Enqueue(data);
|
|
|
|
subscription?.OnNewDataAvailable();
|
|
});
|
|
|
|
enumerator = enqueable;
|
|
}
|
|
else
|
|
{
|
|
var auxEnumerators = new List<IEnumerator<BaseData>>();
|
|
|
|
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, request.Security.Cache, _mapFileProvider,
|
|
_factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
|
|
{
|
|
auxEnumerators.Add(auxDataEnumator);
|
|
}
|
|
|
|
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
|
|
enumerator = Subscribe(request.Configuration, handler, IsExpired);
|
|
|
|
if (auxEnumerators.Count > 0)
|
|
{
|
|
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
|
|
}
|
|
}
|
|
|
|
// scale prices before 'SubscriptionFilterEnumerator' since it updates securities realtime price
|
|
// and before fill forwarding so we don't happen to apply twice the factor
|
|
if (request.Configuration.PricesShouldBeScaled(liveMode: true))
|
|
{
|
|
enumerator = new PriceScaleFactorEnumerator(
|
|
enumerator,
|
|
request.Configuration,
|
|
_factorFileProvider,
|
|
liveMode: true);
|
|
}
|
|
|
|
if (request.Configuration.FillDataForward)
|
|
{
|
|
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
|
|
// Pass the security exchange hours explicitly to avoid using the ones in the request, since
|
|
// those could be different. e.g. when requests are created for open interest data the exchange
|
|
// hours are set to always open to avoid OI data being filtered out due to the exchange being closed.
|
|
var useDailyStrictEndTimes = LeanData.UseDailyStrictEndTimes(_algorithm.Settings, request, request.Configuration.Symbol, request.Configuration.Increment, request.Security.Exchange.Hours);
|
|
|
|
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution,
|
|
request.Configuration.ExtendedMarketHours, localStartTime, localEndTime, request.Configuration.Resolution, request.Configuration.DataTimeZone,
|
|
useDailyStrictEndTimes, request.Configuration.Type, lastPointTracker);
|
|
}
|
|
|
|
// make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
|
|
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
|
|
|
|
// define market hours and user filters to incoming data after the frontier enumerator so during warmup we avoid any realtime data making it's way into the securities
|
|
if (request.Configuration.IsFilteredSubscription)
|
|
{
|
|
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
|
|
}
|
|
|
|
enumerator = GetWarmupEnumerator(request, enumerator, lastPointTracker);
|
|
|
|
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider,
|
|
enumerator, request.IsUniverseSubscription, _algorithm.Settings.DailyPreciseEndTime);
|
|
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
|
|
|
|
return subscription;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper method to determine if the symbol associated with the requested configuration is expired or not
|
|
/// </summary>
|
|
/// <remarks>This is useful during warmup where we can be requested to add some already expired asset. We want to skip sending it
|
|
/// to our live <see cref="_dataQueueHandler"/> instance to avoid explosions. But we do want to add warmup enumerators</remarks>
|
|
private bool IsExpired(SubscriptionDataConfig dataConfig)
|
|
{
|
|
var mapFile = _mapFileProvider.ResolveMapFile(dataConfig);
|
|
var delistingDate = dataConfig.Symbol.GetDelistingDate(mapFile);
|
|
return _timeProvider.GetUtcNow().Date > delistingDate.ConvertToUtc(dataConfig.ExchangeTimeZone);
|
|
}
|
|
|
|
private IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler, Func<SubscriptionDataConfig, bool> isExpired)
|
|
{
|
|
return new LiveSubscriptionEnumerator(dataConfig, _dataQueueHandler, newDataAvailableHandler, isExpired);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a new subscription for universe selection
|
|
/// </summary>
|
|
/// <param name="request">The subscription request</param>
|
|
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
|
|
{
|
|
Subscription subscription = null;
|
|
|
|
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
|
|
|
|
// grab the relevant exchange hours
|
|
var config = request.Universe.Configuration;
|
|
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
|
|
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
|
|
|
|
IEnumerator<BaseData> enumerator = null;
|
|
if (request.Universe is ITimeTriggeredUniverse timeTriggered)
|
|
{
|
|
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
|
|
|
|
// spoof a tick on the requested interval to trigger the universe selection function
|
|
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
|
|
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
|
|
|
|
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
|
|
if (request.Universe is not UserDefinedUniverse)
|
|
{
|
|
var enqueueable = new EnqueueableEnumerator<BaseData>();
|
|
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
|
|
enumerator = enqueueable;
|
|
}
|
|
}
|
|
else if (config.Type.IsAssignableTo(typeof(ETFConstituentUniverse)) ||
|
|
config.Type.IsAssignableTo(typeof(FundamentalUniverse)) ||
|
|
request.Universe is OptionChainUniverse ||
|
|
request.Universe is FuturesChainUniverse)
|
|
{
|
|
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating {config.Type.Name} universe: {config.Symbol.ID}");
|
|
|
|
// Will try to pull data from the data folder every 10min, file with yesterdays date.
|
|
// If lean is started today it will trigger initial coarse universe selection
|
|
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
|
|
_algorithm.ObjectStore,
|
|
// we adjust time to the previous tradable date
|
|
time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone, _algorithm.Settings.DailyPreciseEndTime),
|
|
TimeSpan.FromMinutes(10)
|
|
);
|
|
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
|
|
|
|
// aggregates each coarse data point into a single BaseDataCollection
|
|
var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, true);
|
|
var enqueable = new EnqueueableEnumerator<BaseData>();
|
|
_customExchange.AddEnumerator(config.Symbol, aggregator, handleData: data =>
|
|
{
|
|
enqueable.Enqueue(data);
|
|
subscription?.OnNewDataAvailable();
|
|
});
|
|
|
|
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
|
|
// advance time if before 23pm or after 5am and not on Saturdays
|
|
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
|
|
}
|
|
else
|
|
{
|
|
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);
|
|
|
|
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _algorithm.ObjectStore);
|
|
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
|
|
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
|
|
|
|
var enqueueable = new EnqueueableEnumerator<BaseData>();
|
|
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
|
|
enumerator = enqueueable;
|
|
}
|
|
|
|
enumerator = AddScheduleWrapper(request, enumerator, new PredicateTimeProvider(_frontierTimeProvider, (currentUtcDateTime) => {
|
|
// will only let time advance after it's passed the live time shift frontier
|
|
return currentUtcDateTime.TimeOfDay > GetScheduledUniverseUtcTimeShift(currentUtcDateTime);
|
|
}));
|
|
|
|
enumerator = GetWarmupEnumerator(request, enumerator);
|
|
|
|
// create the subscription
|
|
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider,
|
|
enumerator, request.IsUniverseSubscription, _algorithm.Settings.DailyPreciseEndTime);
|
|
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
|
|
|
|
return subscription;
|
|
}
|
|
|
|
public static TimeSpan GetScheduledUniverseUtcTimeShift(DateTime currentUtcDateTime)
|
|
{
|
|
var currentDate = currentUtcDateTime.Date;
|
|
if (currentDate != _lastUtcDateShiftUpdate?.Value)
|
|
{
|
|
// on every date change, we will update the scheduled time shift to be 8am NY time, which is 12-13 UTC depending on DST
|
|
_lastUtcDateShiftUpdate = new(currentDate);
|
|
_scheduledUniverseUtcTimeShift = new(currentDate.ConvertFromUtc(TimeZones.NewYork).Date.AddHours(8).ConvertToUtc(TimeZones.NewYork).TimeOfDay
|
|
// some minor randomness
|
|
+ TimeSpan.FromSeconds(DateTime.UtcNow.Second));
|
|
}
|
|
return _scheduledUniverseUtcTimeShift.Value;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Build and apply the warmup enumerators when required
|
|
/// </summary>
|
|
private IEnumerator<BaseData> GetWarmupEnumerator(SubscriptionRequest request, IEnumerator<BaseData> liveEnumerator, LastPointTracker lastPointTracker = null)
|
|
{
|
|
if (_algorithm.IsWarmingUp)
|
|
{
|
|
var warmupRequest = new SubscriptionRequest(request, endTimeUtc: _timeProvider.GetUtcNow(),
|
|
// we will not fill forward each warmup enumerators separately but concatenated bellow
|
|
configuration: new SubscriptionDataConfig(request.Configuration, fillForward: false,
|
|
resolution: _algorithm.Settings.WarmupResolution));
|
|
if (warmupRequest.TradableDaysInDataTimeZone.Any()
|
|
// make sure there is at least room for a single bar of the requested resolution, else can cause issues with some history providers
|
|
// this could happen when we create some internal subscription whose start time is 'Now', which we don't really want to warmup
|
|
&& warmupRequest.EndTimeUtc - warmupRequest.StartTimeUtc >= warmupRequest.Configuration.Resolution.ToTimeSpan()
|
|
// since we change the resolution, let's validate it's still valid configuration (example daily equity quotes are not!)
|
|
&& LeanData.IsValidConfiguration(warmupRequest.Configuration.SecurityType, warmupRequest.Configuration.Resolution, warmupRequest.Configuration.TickType))
|
|
{
|
|
// since we will source data locally and from the history provider, let's limit the history request size
|
|
// by setting a start date respecting the 'MaximumWarmupHistoryDaysLookBack'
|
|
var historyWarmup = warmupRequest;
|
|
var warmupHistoryStartDate = warmupRequest.EndTimeUtc.AddDays(-MaximumWarmupHistoryDaysLookBack);
|
|
if (warmupHistoryStartDate > warmupRequest.StartTimeUtc)
|
|
{
|
|
historyWarmup = new SubscriptionRequest(warmupRequest, startTimeUtc: warmupHistoryStartDate);
|
|
}
|
|
|
|
lastPointTracker ??= new LastPointTracker();
|
|
|
|
var synchronizedWarmupEnumerator = TryAddFillForwardEnumerator(warmupRequest,
|
|
// we concatenate the file based and history based warmup enumerators, dropping duplicate time stamps
|
|
new ConcatEnumerator(true, GetFileBasedWarmupEnumerator(warmupRequest), GetHistoryWarmupEnumerator(historyWarmup, lastPointTracker)) { CanEmitNull = false },
|
|
// if required by the original request, we will fill forward the Synced warmup data
|
|
request.Configuration.FillDataForward,
|
|
_algorithm.Settings.WarmupResolution);
|
|
synchronizedWarmupEnumerator = ConfigureLastPointTracker(synchronizedWarmupEnumerator, lastPointTracker, isWarmUpEnumerator: true);
|
|
synchronizedWarmupEnumerator = AddScheduleWrapper(warmupRequest, synchronizedWarmupEnumerator, null);
|
|
|
|
// don't let future data past. We let null pass because that's letting the next enumerator know we've ended because we always return true in live
|
|
synchronizedWarmupEnumerator = new FilterEnumerator<BaseData>(synchronizedWarmupEnumerator, data => data == null || data.EndTime <= warmupRequest.EndTimeLocal);
|
|
|
|
// the order here is important, concat enumerator will keep the last enumerator given and dispose of the rest
|
|
liveEnumerator = new ConcatEnumerator(true, synchronizedWarmupEnumerator, liveEnumerator);
|
|
}
|
|
}
|
|
return liveEnumerator;
|
|
}
|
|
|
|
/// <summary>
|
|
/// File based warmup enumerator
|
|
/// </summary>
|
|
private IEnumerator<BaseData> GetFileBasedWarmupEnumerator(SubscriptionRequest warmup)
|
|
{
|
|
IEnumerator<BaseData> result = null;
|
|
try
|
|
{
|
|
var enumerator = CreateEnumerator(warmup);
|
|
if (warmup.Configuration.PricesShouldBeScaled())
|
|
{
|
|
enumerator = new PriceScaleFactorEnumerator(enumerator, warmup.Configuration, _factorFileProvider);
|
|
}
|
|
result = new FilterEnumerator<BaseData>(enumerator,
|
|
// don't let future data past, nor fill forward, that will be handled after merging with the history request response
|
|
data => data == null || data.EndTime < warmup.EndTimeLocal && !data.IsFillForward);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Log.Error(e, $"File based warmup: {warmup.Configuration}");
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// History based warmup enumerator
|
|
/// </summary>
|
|
private IEnumerator<BaseData> GetHistoryWarmupEnumerator(SubscriptionRequest warmup, LastPointTracker lastPointTracker)
|
|
{
|
|
IEnumerator<BaseData> result;
|
|
if (warmup.IsUniverseSubscription)
|
|
{
|
|
// we ignore the fill forward time span argument because we will fill forwared the concatenated file and history based enumerators next in the stack
|
|
result = CreateUniverseEnumerator(warmup);
|
|
}
|
|
else
|
|
{
|
|
// we create an enumerable of which we get the enumerator to defer the creation of the history request until the file based enumeration ended
|
|
// and potentially the 'lastPointTracker' is available to adjust our start time
|
|
result = new[] { warmup }.SelectMany(_ =>
|
|
{
|
|
var startTimeUtc = warmup.StartTimeUtc;
|
|
if (lastPointTracker != null && lastPointTracker.LastDataPoint != null)
|
|
{
|
|
var lastPointExchangeTime = lastPointTracker.LastDataPoint.Time;
|
|
if (warmup.Configuration.Resolution == Resolution.Daily)
|
|
{
|
|
// time could be 9.30 for example using strict daily end times, but we just want the date in this case
|
|
lastPointExchangeTime = lastPointExchangeTime.Date;
|
|
}
|
|
|
|
var utcLastPointTime = lastPointExchangeTime.ConvertToUtc(warmup.ExchangeHours.TimeZone);
|
|
if (utcLastPointTime > startTimeUtc)
|
|
{
|
|
if (Log.DebuggingEnabled)
|
|
{
|
|
Log.Debug($"LiveTradingDataFeed.GetHistoryWarmupEnumerator(): Adjusting history warmup start time to {utcLastPointTime} from {startTimeUtc} for {warmup.Configuration}");
|
|
}
|
|
startTimeUtc = utcLastPointTime;
|
|
}
|
|
}
|
|
var historyRequest = new Data.HistoryRequest(warmup.Configuration, warmup.ExchangeHours, startTimeUtc, warmup.EndTimeUtc);
|
|
try
|
|
{
|
|
return _algorithm.HistoryProvider.GetHistory(new[] { historyRequest }, _algorithm.TimeZone).Select(slice =>
|
|
{
|
|
try
|
|
{
|
|
var data = slice.Get(historyRequest.DataType);
|
|
return (BaseData)data[warmup.Configuration.Symbol];
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Log.Error(e, $"History warmup: {warmup.Configuration}");
|
|
}
|
|
return null;
|
|
});
|
|
}
|
|
catch
|
|
{
|
|
// some history providers could throw if they do not support a type
|
|
}
|
|
return Enumerable.Empty<BaseData>();
|
|
}).GetEnumerator();
|
|
}
|
|
|
|
return new FilterEnumerator<BaseData>(result,
|
|
// don't let future data past, nor fill forward, that will be handled after merging with the file based enumerator
|
|
data => data == null || data.EndTime < warmup.EndTimeLocal && !data.IsFillForward);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Will wrap the provided enumerator with a <see cref="FrontierAwareEnumerator"/>
|
|
/// using a <see cref="PredicateTimeProvider"/> that will advance time based on the provided
|
|
/// function
|
|
/// </summary>
|
|
/// <remarks>Won't advance time if now.Hour is bigger or equal than 23pm, less or equal than 5am or Saturday.
|
|
/// This is done to prevent universe selection occurring in those hours so that the subscription changes
|
|
/// are handled correctly.</remarks>
|
|
private IEnumerator<BaseData> GetConfiguredFrontierAwareEnumerator(
|
|
IEnumerator<BaseData> enumerator,
|
|
TimeZoneOffsetProvider tzOffsetProvider,
|
|
Func<DateTime, bool> customStepEvaluator)
|
|
{
|
|
var stepTimeProvider = new PredicateTimeProvider(_frontierTimeProvider, customStepEvaluator);
|
|
|
|
return new FrontierAwareEnumerator(enumerator, stepTimeProvider, tzOffsetProvider);
|
|
}
|
|
|
|
private IDataQueueUniverseProvider GetUniverseProvider(SecurityType securityType)
|
|
{
|
|
if (_dataQueueHandler is not IDataQueueUniverseProvider or DataQueueHandlerManager { HasUniverseProvider: false })
|
|
{
|
|
throw new NotSupportedException($"The DataQueueHandler does not support {securityType}.");
|
|
}
|
|
return (IDataQueueUniverseProvider)_dataQueueHandler;
|
|
}
|
|
|
|
private void HandleUnsupportedConfigurationEvent(object _, SubscriptionDataConfig config)
|
|
{
|
|
if (_algorithm != null)
|
|
{
|
|
lock (_unsupportedConfigurations)
|
|
{
|
|
var key = $"{config.Symbol.ID.Market} {config.Symbol.ID.SecurityType} {config.Type.Name}";
|
|
if (_unsupportedConfigurations.Add(key))
|
|
{
|
|
Log.Trace($"LiveTradingDataFeed.HandleUnsupportedConfigurationEvent(): detected unsupported configuration: {config}");
|
|
|
|
_algorithm.Debug($"Warning: {key} data not supported. Please consider reviewing the data providers selection.");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Overrides methods of the base data exchange implementation
|
|
/// </summary>
|
|
private class EnumeratorHandler : BaseDataExchange.EnumeratorHandler
|
|
{
|
|
public EnumeratorHandler(Symbol symbol, IEnumerator<BaseData> enumerator, EnqueueableEnumerator<BaseData> enqueueable)
|
|
: base(symbol, enumerator, handleData: enqueueable.Enqueue)
|
|
{
|
|
EnumeratorFinished += (_, _) => enqueueable.Stop();
|
|
}
|
|
}
|
|
}
|
|
}
|