chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,301 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using DateTime = System.DateTime;
|
||||
using Tick = QuantConnect.Data.Market.Tick;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.All)]
|
||||
public class AggregationManagerTests
|
||||
{
|
||||
[Test]
|
||||
public void PassesTicksStraightThrough()
|
||||
{
|
||||
using var aggregator = GetDataAggregator();
|
||||
var reference = DateTime.Today;
|
||||
var config = GetSubscriptionDataConfig<Tick>(Symbols.SPY, Resolution.Tick);
|
||||
|
||||
var count = 0;
|
||||
aggregator.Add(config, (s, e) => { count++; });
|
||||
|
||||
aggregator.Update(new Tick(reference.AddSeconds(1), Symbols.SPY, 30, 30) { TickType = TickType.Trade });
|
||||
aggregator.Update(new Tick(reference.AddSeconds(2), Symbols.SPY, 20, 20) { TickType = TickType.Trade });
|
||||
|
||||
Assert.AreEqual(count, 2);
|
||||
|
||||
aggregator.Update(new Tick(reference.AddSeconds(3), Symbols.AAPL, 200, 200) { TickType = TickType.Trade });
|
||||
Assert.AreEqual(count, 2);
|
||||
|
||||
aggregator.Remove(config);
|
||||
aggregator.Update(new Tick(reference.AddSeconds(4), Symbols.SPY, 20, 20) { TickType = TickType.Trade });
|
||||
Assert.AreEqual(count, 2);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void BadTicksIgnored()
|
||||
{
|
||||
using var aggregator = GetDataAggregator();
|
||||
var reference = DateTime.Today;
|
||||
var config = GetSubscriptionDataConfig<Tick>(Symbols.SPY, Resolution.Tick);
|
||||
|
||||
var count = 0;
|
||||
aggregator.Add(config, (s, e) => { count++; });
|
||||
|
||||
aggregator.Update(new Tick(reference.AddSeconds(1), Symbols.AAPL, 200, 200));
|
||||
Assert.AreEqual(count, 0);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TickTypeRespected()
|
||||
{
|
||||
using var aggregator = GetDataAggregator();
|
||||
var reference = DateTime.Today;
|
||||
var config = GetSubscriptionDataConfig<Tick>(Symbols.SPY, Resolution.Tick);
|
||||
|
||||
var count = 0;
|
||||
aggregator.Add(config, (s, e) => { count++; });
|
||||
|
||||
aggregator.Update(new Tick(reference.AddSeconds(3), Symbols.SPY, 200, 200) { TickType = TickType.Trade });
|
||||
Assert.AreEqual(count, 1);
|
||||
|
||||
aggregator.Update(new Tick(reference.AddSeconds(4), Symbols.SPY, 20, 20) { TickType = TickType.Quote });
|
||||
Assert.AreEqual(count, 1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UnknownSubscriptionIgnored()
|
||||
{
|
||||
using var aggregator = GetDataAggregator();
|
||||
var reference = DateTime.Today;
|
||||
var config = GetSubscriptionDataConfig<Tick>(Symbols.SPY, Resolution.Tick);
|
||||
|
||||
var count = 0;
|
||||
|
||||
aggregator.Update(new Tick(reference.AddSeconds(1), Symbols.SPY, 30, 30) { TickType = TickType.Trade });
|
||||
aggregator.Update(new Tick(reference.AddSeconds(2), Symbols.SPY, 20, 20) { TickType = TickType.Trade });
|
||||
Assert.AreEqual(count, 0);
|
||||
|
||||
aggregator.Add(config, (s, e) => { count++; });
|
||||
|
||||
aggregator.Update(new Tick(reference.AddSeconds(3), Symbols.SPY, 200, 200) { TickType = TickType.Trade });
|
||||
Assert.AreEqual(count, 1);
|
||||
|
||||
aggregator.Remove(config);
|
||||
aggregator.Update(new Tick(reference.AddSeconds(4), Symbols.SPY, 20, 20) { TickType = TickType.Trade });
|
||||
Assert.AreEqual(count, 1);
|
||||
}
|
||||
|
||||
[TestCase(100, 1, typeof(TradeBar), Resolution.Minute)]
|
||||
[TestCase(120, 2, typeof(TradeBar), Resolution.Minute)]
|
||||
[TestCase(121, 2, typeof(TradeBar), Resolution.Minute)]
|
||||
[TestCase(100, 1, typeof(QuoteBar), Resolution.Minute)]
|
||||
[TestCase(120, 2, typeof(QuoteBar), Resolution.Minute)]
|
||||
[TestCase(121, 2, typeof(QuoteBar), Resolution.Minute)]
|
||||
[TestCase(100, 99, typeof(TradeBar), Resolution.Second)]
|
||||
[TestCase(121, 120, typeof(QuoteBar), Resolution.Second)]
|
||||
[TestCase(3599, 0, typeof(QuoteBar), Resolution.Hour)]
|
||||
[TestCase(3599, 0, typeof(TradeBar), Resolution.Hour)]
|
||||
[TestCase(3600, 1, typeof(QuoteBar), Resolution.Hour)]
|
||||
[TestCase(3600, 1, typeof(TradeBar), Resolution.Hour)]
|
||||
[TestCase(3601, 1, typeof(QuoteBar), Resolution.Hour)]
|
||||
[TestCase(3601, 1, typeof(TradeBar), Resolution.Hour)]
|
||||
public void CanHandleMultipleSubscriptions(int secondsToAdd, int expectedBars, Type dataType, Resolution resolution)
|
||||
{
|
||||
using var aggregator = GetDataAggregator();
|
||||
var reference = DateTime.Today;
|
||||
var symbols = new[] { Symbols.SPY, Symbols.AAPL, Symbols.USDJPY, Symbols.EURUSD };
|
||||
var enumerators = new Queue<IEnumerator<BaseData>>();
|
||||
|
||||
foreach (var symbol in symbols)
|
||||
{
|
||||
enumerators.Enqueue(aggregator.Add(GetSubscriptionDataConfig(dataType, symbol, resolution), (s, e) => { }));
|
||||
}
|
||||
|
||||
for (var i = 1; i <= secondsToAdd; i++)
|
||||
{
|
||||
foreach (var symbol in symbols)
|
||||
{
|
||||
aggregator.Update(new Tick(reference.AddSeconds(i), symbol, 20 + i, 20 + i) { TickType = dataType == typeof(TradeBar) ? TickType.Trade : TickType.Quote });
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var enumerator in enumerators)
|
||||
{
|
||||
for (int i = 0; i < expectedBars; i++)
|
||||
{
|
||||
enumerator.MoveNext();
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
}
|
||||
|
||||
enumerator.MoveNext();
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase(typeof(TradeBar), TickType.Trade, Resolution.Second)]
|
||||
[TestCase(typeof(QuoteBar), TickType.Quote, Resolution.Second)]
|
||||
[TestCase(typeof(Tick), TickType.Trade, Resolution.Tick)]
|
||||
[TestCase(typeof(Tick), TickType.Quote, Resolution.Tick)]
|
||||
public void CanHandleBars(Type type, TickType tickType, Resolution resolution)
|
||||
{
|
||||
using var aggregator = GetDataAggregator();
|
||||
var reference = DateTime.Today;
|
||||
var total = 0;
|
||||
var enumerator = aggregator.Add(GetSubscriptionDataConfig(type, Symbols.EURUSD, resolution, tickType), (s, e) => { });
|
||||
|
||||
for (int i = 0; i < 100; i++)
|
||||
{
|
||||
aggregator.Update(new Tick(reference.AddSeconds(i), Symbols.EURUSD, 20 + i, 20 + i) { TickType = tickType });
|
||||
}
|
||||
Thread.Sleep(250);
|
||||
|
||||
enumerator.MoveNext();
|
||||
while (enumerator.Current != null)
|
||||
{
|
||||
Assert.IsTrue(enumerator.Current.GetType() == type);
|
||||
var tick = enumerator.Current as Tick;
|
||||
if (tick != null)
|
||||
{
|
||||
Assert.IsTrue(tick.TickType == tickType);
|
||||
}
|
||||
total++;
|
||||
enumerator.MoveNext();
|
||||
}
|
||||
|
||||
if (resolution == Resolution.Second)
|
||||
{
|
||||
Assert.AreEqual(99, total);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.AreEqual(100, total);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SubscribeMultipleDataTypes()
|
||||
{
|
||||
var reference = DateTime.Today;
|
||||
var timeProvider = new ManualTimeProvider(reference);
|
||||
using var aggregator = GetDataAggregator(timeProvider);
|
||||
var symbol = Symbols.AAPL;
|
||||
|
||||
var configs = new[] {
|
||||
GetSubscriptionDataConfig<TradeBar>(symbol, Resolution.Minute),
|
||||
GetSubscriptionDataConfig<QuoteBar>(symbol, Resolution.Minute),
|
||||
GetSubscriptionDataConfig<Tick>(symbol, Resolution.Tick, TickType.Trade),
|
||||
GetSubscriptionDataConfig<Tick>(symbol, Resolution.Tick, TickType.Quote),
|
||||
GetSubscriptionDataConfig<Dividend>(symbol, Resolution.Tick),
|
||||
GetSubscriptionDataConfig<Split>(symbol, Resolution.Tick)
|
||||
};
|
||||
|
||||
var enumerators = new Queue<IEnumerator<BaseData>>();
|
||||
Array.ForEach(configs, (c) => enumerators.Enqueue(aggregator.Add(c, (s, e) => { })));
|
||||
|
||||
var expectedBars = new[] { 2, 2, 100, 100, 1, 1 };
|
||||
for (int i = 1; i <= 100; i++)
|
||||
{
|
||||
aggregator.Update(new Tick(reference.AddSeconds(i), symbol, 20 + i, 20 + i)
|
||||
{
|
||||
TickType = TickType.Trade
|
||||
});
|
||||
aggregator.Update(new Tick(reference.AddSeconds(i), symbol, 20 + i, 20 + i)
|
||||
{
|
||||
TickType = TickType.Quote
|
||||
});
|
||||
}
|
||||
|
||||
aggregator.Update(new Dividend(symbol, reference.AddSeconds(1), 0.47m, 108.60m));
|
||||
|
||||
aggregator.Update(new Split(symbol, reference.AddSeconds(1), 645.57m, 0.142857m, SplitType.SplitOccurred));
|
||||
|
||||
timeProvider.SetCurrentTime(reference.AddMinutes(2));
|
||||
|
||||
var dividendCount = 0;
|
||||
var splitCount = 0;
|
||||
var j = 0;
|
||||
foreach (var enumerator in enumerators)
|
||||
{
|
||||
for (int i = 0; i < expectedBars[j]; i++)
|
||||
{
|
||||
enumerator.MoveNext();
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
|
||||
if (enumerator.Current is Dividend)
|
||||
{
|
||||
dividendCount++;
|
||||
}
|
||||
if (enumerator.Current is Split)
|
||||
{
|
||||
splitCount++;
|
||||
}
|
||||
}
|
||||
|
||||
enumerator.MoveNext();
|
||||
Assert.IsNull(enumerator.Current);
|
||||
j++;
|
||||
}
|
||||
|
||||
Assert.AreEqual(1, dividendCount);
|
||||
Assert.AreEqual(1, splitCount);
|
||||
}
|
||||
|
||||
private IDataAggregator GetDataAggregator()
|
||||
{
|
||||
return GetDataAggregator(new ManualTimeProvider(DateTime.Today));
|
||||
}
|
||||
|
||||
private IDataAggregator GetDataAggregator(ITimeProvider timeProvider)
|
||||
{
|
||||
return new TestAggregationManager(timeProvider);
|
||||
}
|
||||
|
||||
private SubscriptionDataConfig GetSubscriptionDataConfig<T>(Symbol symbol, Resolution resolution, TickType? tickType = null)
|
||||
{
|
||||
return GetSubscriptionDataConfig(typeof(T), symbol, resolution, tickType);
|
||||
}
|
||||
|
||||
private SubscriptionDataConfig GetSubscriptionDataConfig(Type T, Symbol symbol, Resolution resolution, TickType? tickType = null)
|
||||
{
|
||||
return new SubscriptionDataConfig(
|
||||
T,
|
||||
symbol,
|
||||
resolution,
|
||||
TimeZones.Utc,
|
||||
TimeZones.Utc,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
tickType: tickType);
|
||||
}
|
||||
|
||||
private class TestAggregationManager : AggregationManager
|
||||
{
|
||||
public TestAggregationManager(ITimeProvider timeProvider)
|
||||
{
|
||||
TimeProvider = timeProvider;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using QuantConnect.Python;
|
||||
using QuantConnect.Algorithm;
|
||||
using QuantConnect.Securities;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Tests.Common.Securities;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
/// <summary>
|
||||
/// This type allows tests to easily create an algorithm that is mostly initialized in one line
|
||||
/// </summary>
|
||||
public class AlgorithmStub : QCAlgorithm
|
||||
{
|
||||
public List<SecurityChanges> SecurityChangesRecord { get; set; } = new List<SecurityChanges>();
|
||||
public DataManager DataManager { get; set; }
|
||||
public IDataFeed DataFeed { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Lanzy PandasConverter only if used
|
||||
/// </summary>
|
||||
public override PandasConverter PandasConverter
|
||||
{
|
||||
get
|
||||
{
|
||||
if(base.PandasConverter == null)
|
||||
{
|
||||
SetPandasConverter();
|
||||
}
|
||||
return base.PandasConverter;
|
||||
}
|
||||
}
|
||||
|
||||
public AlgorithmStub(bool createDataManager = true)
|
||||
{
|
||||
if (createDataManager)
|
||||
{
|
||||
var dataManagerStub = new DataManagerStub(this);
|
||||
DataManager = dataManagerStub;
|
||||
DataFeed = dataManagerStub.DataFeed;
|
||||
SubscriptionManager.SetDataManager(DataManager);
|
||||
}
|
||||
var orderProcessor = new FakeOrderProcessor();
|
||||
orderProcessor.TransactionManager = Transactions;
|
||||
Transactions.SetOrderProcessor(orderProcessor);
|
||||
}
|
||||
|
||||
public AlgorithmStub(IDataFeed dataFeed)
|
||||
{
|
||||
DataFeed = dataFeed;
|
||||
DataManager = new DataManagerStub(dataFeed, this);
|
||||
SubscriptionManager.SetDataManager(DataManager);
|
||||
Transactions.SetOrderProcessor(new FakeOrderProcessor());
|
||||
}
|
||||
|
||||
public void AddSecurities(Resolution resolution = Resolution.Second, List<string> equities = null, List<string> forex = null, List<string> crypto = null)
|
||||
{
|
||||
foreach (var ticker in equities ?? new List<string>())
|
||||
{
|
||||
AddSecurity(SecurityType.Equity, ticker, resolution);
|
||||
var symbol = SymbolCache.GetSymbol(ticker);
|
||||
Securities[symbol].Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
|
||||
}
|
||||
foreach (var ticker in forex ?? new List<string>())
|
||||
{
|
||||
AddSecurity(SecurityType.Forex, ticker, resolution);
|
||||
}
|
||||
foreach (var ticker in crypto ?? new List<string>())
|
||||
{
|
||||
AddSecurity(SecurityType.Crypto, ticker, resolution);
|
||||
var symbol = SymbolCache.GetSymbol(ticker);
|
||||
Securities[symbol].Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.Utc));
|
||||
}
|
||||
}
|
||||
|
||||
public void AddCryptoEntry(string ticker, string market)
|
||||
{
|
||||
var symbolProperties = SymbolPropertiesDatabase.GetSymbolProperties(market, null, SecurityType.Crypto, Currencies.USD);
|
||||
SymbolPropertiesDatabase.SetEntry(market, ticker, SecurityType.Crypto, symbolProperties);
|
||||
}
|
||||
|
||||
public override void OnSecuritiesChanged(SecurityChanges changes)
|
||||
{
|
||||
SecurityChangesRecord.Add(changes);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Auxiliary
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.Fixtures)]
|
||||
public class MapFileResolverTests
|
||||
{
|
||||
private readonly MapFileResolver _resolver = CreateMapFileResolver();
|
||||
|
||||
[Test]
|
||||
public void ChecksFirstDate()
|
||||
{
|
||||
var mapFileProvider = TestGlobals.MapFileProvider;
|
||||
var mapFileResolver = mapFileProvider.Get(AuxiliaryDataKey.EquityUsa);
|
||||
// QQQ started trading on 19990310
|
||||
var mapFile = mapFileResolver.ResolveMapFile("QQQ", new DateTime(1999, 3, 9));
|
||||
Assert.IsTrue(mapFile.IsNullOrEmpty());
|
||||
|
||||
var mapFile2 = mapFileResolver.ResolveMapFile("QQQ", new DateTime(2015, 3, 10));
|
||||
Assert.IsFalse(mapFile2.IsNullOrEmpty());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ResolvesCorrectlyReUsedTicker()
|
||||
{
|
||||
var mapFileProvider = TestGlobals.MapFileProvider;
|
||||
var mapFileResolver = mapFileProvider.Get(AuxiliaryDataKey.EquityUsa);
|
||||
|
||||
// FB.1 started trading on 19990929 and ended on 20030328
|
||||
var mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(1999, 9, 28));
|
||||
Assert.IsTrue(mapFile.IsNullOrEmpty());
|
||||
|
||||
mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(1999, 9, 29));
|
||||
Assert.IsFalse(mapFile.IsNullOrEmpty());
|
||||
|
||||
// FB started trading on 20120518
|
||||
mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(2012, 5, 17));
|
||||
Assert.IsTrue(mapFile.IsNullOrEmpty());
|
||||
|
||||
mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(2015, 5, 18));
|
||||
Assert.IsFalse(mapFile.IsNullOrEmpty());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void InitializationSpeedTest()
|
||||
{
|
||||
var mapFileProvider = TestGlobals.MapFileProvider;
|
||||
var sw = Stopwatch.StartNew();
|
||||
var mapFileresolver = mapFileProvider.Get(AuxiliaryDataKey.EquityUsa);
|
||||
sw.Stop();
|
||||
Log.Trace($"elapsed: {sw.Elapsed.TotalSeconds} seconds");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ResolvesStraightMapping()
|
||||
{
|
||||
var spyMapFile = _resolver.ResolveMapFile("SPY", new DateTime(2015, 08, 23));
|
||||
Assert.IsNotNull(spyMapFile);
|
||||
Assert.AreEqual("SPY", spyMapFile.GetMappedSymbol(new DateTime(2015, 08, 23)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MapFileReturnDefaultValueCorrectly()
|
||||
{
|
||||
var spyMapFile = _resolver.ResolveMapFile("PEPE", new DateTime(2015, 08, 23));
|
||||
Assert.IsNotNull(spyMapFile);
|
||||
Assert.AreEqual("Pepito", spyMapFile.GetMappedSymbol(new DateTime(2015, 08, 23), "Pepito"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ResolvesMapFilesOnNonSpecifiedDates()
|
||||
{
|
||||
var mapFile = _resolver.ResolveMapFile("GOOG", new DateTime(2014, 04, 01));
|
||||
Assert.AreEqual("GOOGL", mapFile.Permtick);
|
||||
|
||||
mapFile = _resolver.ResolveMapFile("GOOG", new DateTime(2014, 04, 03));
|
||||
Assert.AreEqual("GOOG", mapFile.Permtick);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ResolvesOldSymbolRemapped()
|
||||
{
|
||||
// on 2014.04.02 a symbol GOOG traded its last day, the following day it would trade under GOOGL
|
||||
var april2 = new DateTime(2014, 04, 02);
|
||||
var googMapFile = _resolver.ResolveMapFile("GOOG", april2);
|
||||
Assert.IsNotNull(googMapFile);
|
||||
Assert.AreEqual("GOOG", googMapFile.GetMappedSymbol(april2));
|
||||
Assert.AreEqual("GOOGL", googMapFile.GetMappedSymbol(april2.AddDays(1)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ResolvesExactMapping()
|
||||
{
|
||||
var oih1 = _resolver.ResolveMapFile("OIH.1", new DateTime(2011, 12, 20));
|
||||
Assert.IsNotNull(oih1);
|
||||
Assert.AreEqual("OIH", oih1.GetMappedSymbol(new DateTime(2010, 02, 06)));
|
||||
Assert.AreEqual("OIH", oih1.GetMappedSymbol(new DateTime(2010, 02, 07)));
|
||||
Assert.AreEqual("OIH", oih1.GetMappedSymbol(new DateTime(2011, 12, 20)));
|
||||
Assert.AreEqual(string.Empty, oih1.GetMappedSymbol(new DateTime(2011, 12, 21)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ResolvesRemappedSymbolWithBothMapFiles()
|
||||
{
|
||||
var date = new DateTime(2012, 06, 28);
|
||||
var mapFile = _resolver.ResolveMapFile("SPXL", date);
|
||||
Assert.IsNotNull(mapFile);
|
||||
Assert.AreEqual("BGU", mapFile.GetMappedSymbol(date));
|
||||
Assert.AreEqual("SPXL", mapFile.GetMappedSymbol(date.AddDays(1)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ResolvesRemappedAndDelistedSymbol()
|
||||
{
|
||||
var date = new DateTime(2018, 7, 23);
|
||||
var mapFile = _resolver.ResolveMapFile("TWX", date);
|
||||
Assert.IsNotNull(mapFile);
|
||||
Assert.AreEqual("AOL", mapFile.GetMappedSymbol(new DateTime(2000, 1, 1)));
|
||||
Assert.AreEqual("TWX", mapFile.GetMappedSymbol(new DateTime(2014, 1, 1)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ContinuousFuturesMappingMode()
|
||||
{
|
||||
var date = new DateTime(2018, 7, 23);
|
||||
var mapFile = _resolver.ResolveMapFile("NG", date);
|
||||
Assert.IsNotNull(mapFile);
|
||||
Assert.AreEqual("NG UNT495KXTOLD", mapFile.GetMappedSymbol(new DateTime(2010, 6, 15), dataMappingMode: DataMappingMode.LastTradingDay));
|
||||
Assert.AreEqual("NG UOMNNYK04BEP", mapFile.GetMappedSymbol(new DateTime(2010, 6, 15), dataMappingMode: DataMappingMode.FirstDayMonth));
|
||||
Assert.AreEqual("NG UMWMI2IDASAP", mapFile.GetMappedSymbol(new DateTime(2010, 6, 15), dataMappingMode: DataMappingMode.OpenInterest));
|
||||
}
|
||||
|
||||
private static MapFileResolver CreateMapFileResolver()
|
||||
{
|
||||
return new MapFileResolver(new List<MapFile>
|
||||
{
|
||||
// remapped
|
||||
new MapFile("goog", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(2014, 03, 27), "goocv"),
|
||||
new MapFileRow(new DateTime(2014, 04, 02), "goocv"),
|
||||
new MapFileRow(new DateTime(2050, 12, 31), "goog")
|
||||
}),
|
||||
new MapFile("googl", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(2004, 08, 19), "goog"),
|
||||
new MapFileRow(new DateTime(2014, 04, 02), "goog"),
|
||||
new MapFileRow(new DateTime(2050, 12, 31), "googl")
|
||||
}),
|
||||
// remapped (with both map files)
|
||||
new MapFile("bgu", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(2008, 11, 05), "bgu"),
|
||||
new MapFileRow(new DateTime(2012, 06, 28), "bgu")
|
||||
}),
|
||||
new MapFile("spxl", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(2008, 11, 05), "bgu"),
|
||||
new MapFileRow(new DateTime(2012, 06, 28), "bgu"),
|
||||
new MapFileRow(new DateTime(2050, 12, 31), "spxl")
|
||||
}),
|
||||
// straight mapping
|
||||
new MapFile("spy", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(1998, 01, 02), "spy"),
|
||||
new MapFileRow(new DateTime(2050, 12, 31), "spy")
|
||||
}),
|
||||
// new share class overtakes old share class same symbol
|
||||
new MapFile("oih.1", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(2010, 02, 07), "oih"),
|
||||
new MapFileRow(new DateTime(2011, 12, 20), "oih")
|
||||
}),
|
||||
new MapFile("oih", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(2011, 12, 21), "oih"),
|
||||
new MapFileRow(new DateTime(2050, 12, 31), "oih")
|
||||
}),
|
||||
// remapped + delisted
|
||||
new MapFile("twx.1", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(1998, 1, 2), "twx"),
|
||||
new MapFileRow(new DateTime(2001, 1, 11), "twx")
|
||||
}),
|
||||
new MapFile("twx", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(1998, 1, 2), "aol"),
|
||||
new MapFileRow(new DateTime(2003, 10, 15), "aol"),
|
||||
new MapFileRow(new DateTime(2018, 6, 15), "twx")
|
||||
}),
|
||||
new MapFile("ng", new List<MapFileRow>
|
||||
{
|
||||
new MapFileRow(new DateTime(2010, 6, 15), "ng unt495kxtold", Exchange.NYMEX, DataMappingMode.LastTradingDay),
|
||||
new MapFileRow(new DateTime(2010, 6, 15), "ng uomnnyk04bep", Exchange.NYMEX, DataMappingMode.FirstDayMonth),
|
||||
new MapFileRow(new DateTime(2010, 6, 15), "ng umwmi2idasap", Exchange.NYMEX, DataMappingMode.OpenInterest),
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class BacktestingFutureChainProviderTests
|
||||
{
|
||||
private ILogHandler _logHandler;
|
||||
private BacktestingFutureChainProvider _provider;
|
||||
|
||||
[OneTimeSetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
// Store initial Log Handler
|
||||
_logHandler = Log.LogHandler;
|
||||
_provider = new BacktestingFutureChainProvider();
|
||||
_provider.Initialize(new(TestGlobals.MapFileProvider, TestGlobals.HistoryProvider));
|
||||
}
|
||||
|
||||
[OneTimeTearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
// Restore intial Log Handler
|
||||
Log.LogHandler = _logHandler;
|
||||
}
|
||||
|
||||
[TestCase("20131011")]
|
||||
// saturday, will fetch previous tradable date instead
|
||||
[TestCase("20131012")]
|
||||
public void CorrectlyDeterminesContractList(string date)
|
||||
{
|
||||
var dateTime = Time.ParseDate(date);
|
||||
var symbol = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, dateTime.AddDays(10));
|
||||
var result = _provider.GetFutureContractList(symbol, dateTime);
|
||||
|
||||
Assert.IsNotEmpty(result);
|
||||
}
|
||||
|
||||
[TestCase("20201007", 2)]
|
||||
[TestCase("20131007", 5)]
|
||||
public void UsesMultipleResolutions(string strDate, int expectedCount)
|
||||
{
|
||||
// we don't have minute data for this date
|
||||
var date = Time.ParseDate(strDate);
|
||||
|
||||
var symbol = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, date);
|
||||
var futureChain = _provider.GetFutureContractList(symbol, date).ToList();
|
||||
|
||||
Assert.IsTrue(futureChain.All(x => x.ID.Date.Date >= date));
|
||||
Assert.IsTrue(futureChain.All(x => x.SecurityType == SecurityType.Future));
|
||||
Assert.IsTrue(futureChain.All(x => x.ID.Symbol == Futures.Indices.SP500EMini));
|
||||
Assert.AreEqual(expectedCount, futureChain.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.Fixtures)]
|
||||
public class BaseDataCollectionAggregatorReaderTests
|
||||
{
|
||||
[TestCase(Resolution.Daily, 1, true, true)]
|
||||
[TestCase( Resolution.Hour, 1, true, true)]
|
||||
[TestCase(Resolution.Daily, 5849, false, true)]
|
||||
[TestCase(Resolution.Hour, 40832, false, true)]
|
||||
[TestCase(Resolution.Daily, 1, true, false)]
|
||||
[TestCase(Resolution.Hour, 1, true, false)]
|
||||
[TestCase(Resolution.Daily, 5849, false, false)]
|
||||
[TestCase(Resolution.Hour, 40832, false, false)]
|
||||
public void AggregatesDataPerTime(Resolution resolution, int expectedCount, bool singleDate, bool isDataEphemeral)
|
||||
{
|
||||
var reader = Initialize(false, resolution, isDataEphemeral, out var dataSource);
|
||||
TestBaseDataCollection.SingleDate = singleDate;
|
||||
|
||||
var result = reader.Read(dataSource).ToList();
|
||||
|
||||
Assert.AreEqual(expectedCount, result.Count);
|
||||
Assert.IsTrue(result.All(data => data is TestBaseDataCollection));
|
||||
|
||||
if (expectedCount == 1)
|
||||
{
|
||||
var collection = result[0] as TestBaseDataCollection;
|
||||
Assert.IsNotNull(collection);
|
||||
Assert.GreaterOrEqual(collection.Data.Count, 5000);
|
||||
Assert.AreEqual(expectedCount, collection.Data.DistinctBy(data => data.Time).Count());
|
||||
Assert.IsTrue(collection.Data.All(x => x.Symbol == Symbols.AAPL));
|
||||
}
|
||||
}
|
||||
|
||||
private static ISubscriptionDataSourceReader Initialize(bool liveMode, Resolution resolution, bool isDataEphemeral, out SubscriptionDataSource source)
|
||||
{
|
||||
using var cache = new ZipDataCacheProvider(TestGlobals.DataProvider, isDataEphemeral: isDataEphemeral);
|
||||
var config = new SubscriptionDataConfig(typeof(TestBaseDataCollection),
|
||||
Symbols.SPY,
|
||||
resolution,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
false,
|
||||
false,
|
||||
false);
|
||||
var date = DateTime.MinValue;
|
||||
var path = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, resolution, TickType.Trade);
|
||||
source = new SubscriptionDataSource(path, SubscriptionTransportMedium.LocalFile);
|
||||
return new BaseDataCollectionAggregatorReader(cache, config, date, liveMode, null);
|
||||
}
|
||||
|
||||
private class TestBaseDataCollection : BaseDataCollection
|
||||
{
|
||||
public static volatile bool SingleDate;
|
||||
private static readonly TradeBar _factory = new TradeBar();
|
||||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var dataPoint = _factory.Reader(config, line, date, isLiveMode);
|
||||
if (SingleDate)
|
||||
{
|
||||
// single day
|
||||
dataPoint.Time = date;
|
||||
}
|
||||
// universe data collections can return data with a symbol different than the one in the configuration
|
||||
dataPoint.Symbol = Symbols.AAPL;
|
||||
return dataPoint;
|
||||
}
|
||||
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
return _factory.GetSource(config, date, isLiveMode);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.Fixtures)]
|
||||
public class BaseDataExchangeTests
|
||||
{
|
||||
// This is a default timeout for all tests to wait if something went wrong
|
||||
const int DefaultTimeout = 30000;
|
||||
private CancellationTokenSource _cancellationTokenSource;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_cancellationTokenSource = new CancellationTokenSource();
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_cancellationTokenSource.Cancel();
|
||||
_cancellationTokenSource.DisposeSafely();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EndsQueueConsumption()
|
||||
{
|
||||
var enqueable = new EnqueueableEnumerator<BaseData>();
|
||||
var exchange = new BaseDataExchange("test");
|
||||
|
||||
using var cancellationToken = new CancellationTokenSource();
|
||||
Task.Run(() =>
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
enqueable.Enqueue(new Tick {Symbol = Symbols.SPY, Time = DateTime.UtcNow});
|
||||
}
|
||||
});
|
||||
|
||||
BaseData last = null;
|
||||
using var lastUpdated = new AutoResetEvent(false);
|
||||
exchange.AddEnumerator(Symbols.SPY, enqueable, handleData: spy =>
|
||||
{
|
||||
last = spy;
|
||||
lastUpdated.Set();
|
||||
});
|
||||
|
||||
using var finishedRunning = new AutoResetEvent(false);
|
||||
Task.Run(() => { exchange.Start(); finishedRunning.Set(); } );
|
||||
|
||||
try
|
||||
{
|
||||
Assert.IsTrue(lastUpdated.WaitOne(DefaultTimeout));
|
||||
}
|
||||
finally
|
||||
{
|
||||
exchange.Stop();
|
||||
cancellationToken.Cancel();
|
||||
}
|
||||
|
||||
Assert.IsTrue(finishedRunning.WaitOne(DefaultTimeout));
|
||||
|
||||
var endTime = DateTime.UtcNow;
|
||||
Assert.IsNotNull(last);
|
||||
Assert.IsTrue(last.Time <= endTime);
|
||||
enqueable.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DefaultErrorHandlerDoesNotStopQueueConsumption()
|
||||
{
|
||||
var enqueable = new EnqueueableEnumerator<BaseData>();
|
||||
var exchange = new BaseDataExchange("test");
|
||||
|
||||
using var cancellationToken = new CancellationTokenSource();
|
||||
Task.Run(() =>
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
enqueable.Enqueue(new Tick { Symbol = Symbols.SPY, Time = DateTime.UtcNow });
|
||||
}
|
||||
});
|
||||
|
||||
var first = true;
|
||||
BaseData last = null;
|
||||
using var lastUpdated = new AutoResetEvent(false);
|
||||
exchange.AddEnumerator(Symbols.SPY, enqueable, handleData: spy =>
|
||||
{
|
||||
if (first)
|
||||
{
|
||||
first = false;
|
||||
throw new RegressionTestException("This exception should be swalloed by the exchange!");
|
||||
}
|
||||
last = spy;
|
||||
lastUpdated.Set();
|
||||
});
|
||||
|
||||
Task.Run(() => exchange.Start());
|
||||
|
||||
try
|
||||
{
|
||||
Assert.IsTrue(lastUpdated.WaitOne(DefaultTimeout));
|
||||
}
|
||||
finally
|
||||
{
|
||||
exchange.Stop();
|
||||
cancellationToken.Cancel();
|
||||
enqueable.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SetErrorHandlerExitsOnTrueReturn()
|
||||
{
|
||||
var enqueable = new EnqueueableEnumerator<BaseData>();
|
||||
var exchange = new BaseDataExchange("test");
|
||||
|
||||
using var cancellationToken = new CancellationTokenSource();
|
||||
Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
enqueable.Enqueue(new Tick { Symbol = Symbols.SPY, Time = DateTime.UtcNow });
|
||||
}
|
||||
});
|
||||
|
||||
var first = true;
|
||||
using var errorCaught = new AutoResetEvent(true);
|
||||
BaseData last = null;
|
||||
|
||||
exchange.SetErrorHandler(error =>
|
||||
{
|
||||
errorCaught.Set();
|
||||
return true;
|
||||
});
|
||||
|
||||
exchange.AddEnumerator(Symbols.SPY, enqueable, handleData: spy =>
|
||||
{
|
||||
if (first)
|
||||
{
|
||||
first = false;
|
||||
throw new RegressionTestException();
|
||||
}
|
||||
last = spy;
|
||||
});
|
||||
|
||||
Task.Run(() => exchange.Start());
|
||||
|
||||
try
|
||||
{
|
||||
Assert.IsTrue(errorCaught.WaitOne(DefaultTimeout));
|
||||
Assert.IsNull(last);
|
||||
}
|
||||
finally
|
||||
{
|
||||
exchange.Stop();
|
||||
enqueable.Dispose();
|
||||
cancellationToken.Cancel();
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RespectsShouldMoveNext()
|
||||
{
|
||||
var exchange = new BaseDataExchange("test");
|
||||
exchange.SetErrorHandler(exception => true);
|
||||
exchange.AddEnumerator(Symbol.Empty, new List<BaseData> {new Tick()}.GetEnumerator(), () => false);
|
||||
|
||||
using var isFaultedEvent = new ManualResetEvent(false);
|
||||
using var isCompletedEvent = new ManualResetEvent(false);
|
||||
Task.Run(() => exchange.Start()).ContinueWith(task =>
|
||||
{
|
||||
if (task.IsFaulted) isFaultedEvent.Set();
|
||||
isCompletedEvent.Set();
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
isCompletedEvent.WaitOne();
|
||||
Assert.IsFalse(isFaultedEvent.WaitOne(0));
|
||||
}
|
||||
finally
|
||||
{
|
||||
exchange.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FiresOnEnumeratorFinishedEvents()
|
||||
{
|
||||
var exchange = new BaseDataExchange("test");
|
||||
IEnumerator<BaseData> enumerator = new List<BaseData>().GetEnumerator();
|
||||
|
||||
using var isCompletedEvent = new ManualResetEvent(false);
|
||||
exchange.AddEnumerator(Symbol.Empty, enumerator, () => true, handler => isCompletedEvent.Set());
|
||||
Task.Run(() => exchange.Start());
|
||||
|
||||
isCompletedEvent.WaitOne();
|
||||
exchange.Stop();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RemovesBySymbol()
|
||||
{
|
||||
var exchange = new BaseDataExchange("test");
|
||||
var enumerator = new List<BaseData> {new Tick {Symbol = Symbols.SPY}}.GetEnumerator();
|
||||
exchange.AddEnumerator(Symbols.SPY, enumerator);
|
||||
var removed = exchange.RemoveEnumerator(Symbols.AAPL);
|
||||
Assert.IsNull(removed);
|
||||
removed = exchange.RemoveEnumerator(Symbols.SPY);
|
||||
Assert.AreEqual(Symbols.SPY, removed.Symbol);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Custom.Tiingo;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class CollectionSubscriptionDataSourceReaderTests
|
||||
{
|
||||
[Test]
|
||||
public void HandlesInitializationErrors()
|
||||
{
|
||||
var date = new DateTime(2018, 7, 7);
|
||||
var config = new SubscriptionDataConfig(typeof(TiingoPrice), Symbols.AAPL, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, true);
|
||||
var reader = new CollectionSubscriptionDataSourceReader(null, config, date, false, null);
|
||||
var source = new TiingoPrice().GetSource(config, date, false);
|
||||
|
||||
// should not throw with an empty or invalid Tiingo API token
|
||||
Assert.DoesNotThrow(() =>
|
||||
{
|
||||
var list = reader.Read(source).ToList();
|
||||
Assert.AreEqual(0, list.Count);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class CompositeTimeProviderTests
|
||||
{
|
||||
[Test]
|
||||
public void NoProviderGiven()
|
||||
{
|
||||
var compositeTimeProvider = new CompositeTimeProvider(Enumerable.Empty<ITimeProvider>());
|
||||
|
||||
var utcNow = DateTime.UtcNow;
|
||||
|
||||
Assert.LessOrEqual(utcNow, compositeTimeProvider.GetUtcNow());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SmallestTime()
|
||||
{
|
||||
var time = new DateTime(1999, 1, 1);
|
||||
|
||||
var timeProvider1 = new ManualTimeProvider();
|
||||
timeProvider1.SetCurrentTimeUtc(time.AddYears(1));
|
||||
var timeProvider2 = new ManualTimeProvider();
|
||||
timeProvider2.SetCurrentTimeUtc(time);
|
||||
var timeProvider3 = new ManualTimeProvider();
|
||||
timeProvider3.SetCurrentTimeUtc(time.AddYears(2));
|
||||
|
||||
var compositeTimeProvider = new CompositeTimeProvider(new[] { timeProvider1 , timeProvider2, timeProvider3 });
|
||||
|
||||
Assert.LessOrEqual(time, compositeTimeProvider.GetUtcNow());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,611 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Algorithm;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Transport;
|
||||
using QuantConnect.Lean.Engine.Results;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Lean.Engine.Storage;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.Fixtures)]
|
||||
public class CustomLiveDataFeedTests
|
||||
{
|
||||
private LiveSynchronizer _synchronizer;
|
||||
private IDataFeed _feed;
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_feed.Exit();
|
||||
_synchronizer.DisposeSafely();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmitsDailyCustomFutureDataOverWeekends()
|
||||
{
|
||||
using var api = new Api.Api();
|
||||
RemoteFileSubscriptionStreamReader.SetDownloadProvider(api);
|
||||
var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" };
|
||||
var startDate = new DateTime(2018, 4, 1);
|
||||
var endDate = new DateTime(2018, 4, 20);
|
||||
|
||||
// delete temp files
|
||||
foreach (var ticker in tickers)
|
||||
{
|
||||
var fileName = TestableCustomFuture.GetLocalFileName(ticker, "test");
|
||||
File.Delete(fileName);
|
||||
}
|
||||
|
||||
var algorithm = new QCAlgorithm();
|
||||
CreateDataFeed(algorithm.Settings);
|
||||
var dataManager = new DataManagerStub(algorithm, _feed);
|
||||
algorithm.SubscriptionManager.SetDataManager(dataManager);
|
||||
|
||||
var symbols = tickers.Select(ticker => algorithm.AddData<TestableCustomFuture>(ticker, Resolution.Daily).Symbol).ToList();
|
||||
|
||||
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
|
||||
timeProvider.SetCurrentTime(startDate);
|
||||
|
||||
var dataPointsEmitted = 0;
|
||||
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
|
||||
|
||||
using var cancellationTokenSource = new CancellationTokenSource();
|
||||
var lastFileWriteDate = DateTime.MinValue;
|
||||
|
||||
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
|
||||
var timerInterval = TimeSpan.FromMilliseconds(20);
|
||||
var timer = Ref.Create<Timer>(null);
|
||||
timer.Value = new Timer(state =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
|
||||
|
||||
if (currentTime.Date > endDate.Date)
|
||||
{
|
||||
Log.Trace($"Total data points emitted: {dataPointsEmitted.ToStringInvariant()}");
|
||||
|
||||
_feed.Exit();
|
||||
cancellationTokenSource.Cancel();
|
||||
return;
|
||||
}
|
||||
|
||||
if (currentTime.Date > lastFileWriteDate.Date)
|
||||
{
|
||||
foreach (var ticker in tickers)
|
||||
{
|
||||
var source = TestableCustomFuture.GetLocalFileName(ticker, "csv");
|
||||
|
||||
// write new local file including only rows up to current date
|
||||
var outputFileName = TestableCustomFuture.GetLocalFileName(ticker, "test");
|
||||
|
||||
var sb = new StringBuilder();
|
||||
{
|
||||
using (var reader = new StreamReader(source))
|
||||
{
|
||||
var firstLine = true;
|
||||
string line;
|
||||
while ((line = reader.ReadLine()) != null)
|
||||
{
|
||||
if (firstLine)
|
||||
{
|
||||
sb.AppendLine(line);
|
||||
firstLine = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
var csv = line.Split(',');
|
||||
var time = Parse.DateTimeExact(csv[0], "yyyy-MM-dd");
|
||||
if (time.Date >= currentTime.Date)
|
||||
break;
|
||||
|
||||
sb.AppendLine(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday)
|
||||
{
|
||||
var fileContent = sb.ToString();
|
||||
try
|
||||
{
|
||||
File.WriteAllText(outputFileName, fileContent);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
Log.Error("IOException: will sleep 200ms and retry once more");
|
||||
// lets sleep 200ms and retry once more, consumer could be reading the file
|
||||
// this exception happens in travis intermittently, GH issue 3273
|
||||
Thread.Sleep(200);
|
||||
File.WriteAllText(outputFileName, fileContent);
|
||||
}
|
||||
|
||||
Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}");
|
||||
}
|
||||
}
|
||||
|
||||
lastFileWriteDate = currentTime;
|
||||
}
|
||||
|
||||
// 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time
|
||||
timeProvider.Advance(TimeSpan.FromMinutes(20));
|
||||
|
||||
//Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}");
|
||||
|
||||
// restart the timer
|
||||
timer.Value.Change(timerInterval.Milliseconds, Timeout.Infinite);
|
||||
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Log.Error(exception);
|
||||
_feed.Exit();
|
||||
cancellationTokenSource.Cancel();
|
||||
}
|
||||
}, null, timerInterval.Milliseconds, Timeout.Infinite);
|
||||
|
||||
try
|
||||
{
|
||||
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
|
||||
{
|
||||
foreach (var dataPoint in timeSlice.Slice.Values)
|
||||
{
|
||||
Log.Trace($"Data point emitted at {timeSlice.Slice.Time.ToStringInvariant()}: " +
|
||||
$"{dataPoint.Symbol.Value} {dataPoint.Value.ToStringInvariant()} " +
|
||||
$"{dataPoint.EndTime.ToStringInvariant()}"
|
||||
);
|
||||
|
||||
dataPointsEmitted++;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Log.Trace($"Error: {exception}");
|
||||
}
|
||||
|
||||
timer.Value.Dispose();
|
||||
dataManager.RemoveAllSubscriptions();
|
||||
Assert.AreEqual(14 * tickers.Length, dataPointsEmitted);
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public void RemoteDataDoesNotIncreaseNumberOfSlices()
|
||||
{
|
||||
using var api = new Api.Api();
|
||||
RemoteFileSubscriptionStreamReader.SetDownloadProvider(api);
|
||||
|
||||
var startDate = new DateTime(2017, 4, 2);
|
||||
var endDate = new DateTime(2017, 4, 23);
|
||||
var algorithm = new QCAlgorithm();
|
||||
|
||||
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
|
||||
timeProvider.SetCurrentTime(startDate);
|
||||
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
|
||||
{
|
||||
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
|
||||
var tick = new Tick(time, Symbols.SPY, 1.3m, 1.2m, 1.3m)
|
||||
{
|
||||
TickType = TickType.Trade
|
||||
};
|
||||
var tick2 = new Tick(time, Symbols.AAPL, 1.3m, 1.2m, 1.3m)
|
||||
{
|
||||
TickType = TickType.Trade
|
||||
};
|
||||
return new[] { tick, tick2 };
|
||||
}, timeProvider, algorithm.Settings);
|
||||
CreateDataFeed(algorithm.Settings, dataQueueHandler);
|
||||
var dataManager = new DataManagerStub(algorithm, _feed);
|
||||
|
||||
algorithm.SubscriptionManager.SetDataManager(dataManager);
|
||||
var symbols = new List<Symbol>
|
||||
{
|
||||
algorithm.AddData<TestableRemoteCustomData>("FB", Resolution.Daily).Symbol,
|
||||
algorithm.AddData<TestableRemoteCustomData>("IBM", Resolution.Daily).Symbol,
|
||||
algorithm.AddEquity("SPY", Resolution.Daily).Symbol,
|
||||
algorithm.AddEquity("AAPL", Resolution.Daily).Symbol
|
||||
};
|
||||
|
||||
using var cancellationTokenSource = new CancellationTokenSource();
|
||||
|
||||
var dataPointsEmitted = 0;
|
||||
var slicesEmitted = 0;
|
||||
|
||||
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
|
||||
Thread.Sleep(5000); // Give remote sources a handicap, so the data is available in time
|
||||
|
||||
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
|
||||
var timerInterval = TimeSpan.FromMilliseconds(100);
|
||||
var timer = Ref.Create<Timer>(null);
|
||||
timer.Value = new Timer(state =>
|
||||
{
|
||||
// stop the timer to prevent reentrancy
|
||||
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
|
||||
|
||||
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
|
||||
|
||||
if (currentTime.Date > endDate.Date)
|
||||
{
|
||||
_feed.Exit();
|
||||
cancellationTokenSource.Cancel();
|
||||
return;
|
||||
}
|
||||
|
||||
timeProvider.Advance(TimeSpan.FromHours(3));
|
||||
|
||||
// restart the timer
|
||||
timer.Value.Change(timerInterval, timerInterval);
|
||||
|
||||
}, null, TimeSpan.FromSeconds(2), timerInterval);
|
||||
|
||||
try
|
||||
{
|
||||
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
|
||||
{
|
||||
if (timeSlice.Slice.HasData)
|
||||
{
|
||||
slicesEmitted++;
|
||||
dataPointsEmitted += timeSlice.Slice.Values.Count;
|
||||
|
||||
if (timeSlice.Time.ConvertFromUtc(TimeZones.NewYork).Hour == 0)
|
||||
{
|
||||
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[0]), $"Slice doesn't contain {symbols[0]}");
|
||||
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[1]), $"Slice doesn't contain {symbols[1]}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[2]), $"Slice doesn't contain {symbols[2]}");
|
||||
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[3]), $"Slice doesn't contain {symbols[3]}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Log.Trace($"Error: {exception}");
|
||||
}
|
||||
|
||||
timer.Value.Dispose();
|
||||
dataManager.RemoveAllSubscriptions();
|
||||
dataQueueHandler.DisposeSafely();
|
||||
// custom data arrives midnight, daily data 4pm
|
||||
Assert.AreEqual(14 * 2, slicesEmitted);
|
||||
Assert.AreEqual(14 * symbols.Count, dataPointsEmitted);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LiveDataFeedSourcesDataFromObjectStoreSort()
|
||||
{
|
||||
var dataPointsEmitted = 0;
|
||||
var slicesEmitted = 0;
|
||||
using var cancellationTokenSource = new CancellationTokenSource();
|
||||
|
||||
var mockCustomData = new string[]
|
||||
{
|
||||
"2017-04-28,173.82",
|
||||
"2017-04-27,173.52",
|
||||
"2017-04-26,174.73",
|
||||
"2017-04-25,173.47",
|
||||
"2017-04-24,172.08",
|
||||
"2017-04-21,172.53",
|
||||
"2017-04-20,170.65",
|
||||
"2017-04-19,171.04",
|
||||
"2017-04-18,169.92",
|
||||
"2017-04-17,169.75",
|
||||
"2017-04-13,170.79",
|
||||
"2017-04-12,161.76",
|
||||
"2017-04-11,161.32",
|
||||
"2017-04-10,162.05",
|
||||
"2017-04-07,161.29",
|
||||
"2017-04-06,161.78",
|
||||
"2017-04-05,160.53",
|
||||
"2017-04-04,160.29",
|
||||
"2017-04-03,160.53"
|
||||
};
|
||||
|
||||
var objectText = string.Join("\n", mockCustomData);
|
||||
|
||||
var (dataManager, timer, _) = RunLiveDataFeedWithObjectStore(
|
||||
new DateTime(2017, 4, 2),
|
||||
new DateTime(2017, 4, 28),
|
||||
cancellationTokenSource, "CustomData/CustomIBM", objectText,
|
||||
(algorithm) => algorithm.AddData<CustomDataSort>("IBM", Resolution.Daily).Symbol);
|
||||
|
||||
var baseData = new List<BaseData>();
|
||||
|
||||
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
|
||||
{
|
||||
if (timeSlice.Slice.HasData)
|
||||
{
|
||||
foreach (var customData in timeSlice.CustomData)
|
||||
{
|
||||
foreach (var data in customData.Data)
|
||||
{
|
||||
baseData.Add(data);
|
||||
}
|
||||
}
|
||||
|
||||
slicesEmitted++;
|
||||
dataPointsEmitted += timeSlice.Slice.Values.Count;
|
||||
}
|
||||
}
|
||||
|
||||
timer.Dispose();
|
||||
dataManager.RemoveAllSubscriptions();
|
||||
Assert.AreEqual(mockCustomData.Length, slicesEmitted);
|
||||
Assert.AreEqual(slicesEmitted, dataPointsEmitted);
|
||||
|
||||
for (int i = 0; i < baseData.Count - 1; i++)
|
||||
{
|
||||
if (baseData[i].EndTime > baseData[i + 1].EndTime)
|
||||
{
|
||||
Assert.Fail($"Order failure: {baseData[i].EndTime} > {baseData[i + 1].EndTime} at index {i}.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LiveDataFeedSourcesDataFromObjectStore()
|
||||
{
|
||||
using var cancellationTokenSource = new CancellationTokenSource();
|
||||
|
||||
var dataPointsEmitted = 0;
|
||||
var slicesEmitted = 0;
|
||||
|
||||
var mockCustomData = new string[]
|
||||
{
|
||||
"2017-04-03,173.82",
|
||||
"2017-04-04,173.52",
|
||||
"2017-04-05,174.72",
|
||||
"2017-04-06,173.47",
|
||||
"2017-04-07,172.08",
|
||||
"2017-04-10,172.53",
|
||||
"2017-04-11,170.65",
|
||||
"2017-04-12,171.04",
|
||||
"2017-04-13,169.92",
|
||||
"2017-04-17,169.75",
|
||||
"2017-04-18,170.79",
|
||||
"2017-04-19,161.76",
|
||||
"2017-04-20,161.32",
|
||||
"2017-04-21,162.05",
|
||||
"2017-04-24,161.29",
|
||||
"2017-04-25,161.78",
|
||||
"2017-04-26,160.53",
|
||||
"2017-04-27,160.29",
|
||||
"2017-04-28,160.5"
|
||||
};
|
||||
|
||||
var objectText = string.Join("\n", mockCustomData);
|
||||
|
||||
var (dataManager, timer, symbol) = RunLiveDataFeedWithObjectStore(new DateTime(2017, 4, 2), new DateTime(2017, 4, 23), cancellationTokenSource, "CustomData/CustomIBM", objectText,
|
||||
algorithm => algorithm.AddData<TestableObjectStoreCustomData>("IBM", Resolution.Daily).Symbol);
|
||||
|
||||
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
|
||||
{
|
||||
if (timeSlice.Slice.HasData)
|
||||
{
|
||||
slicesEmitted++;
|
||||
dataPointsEmitted += timeSlice.Slice.Values.Count;
|
||||
Assert.AreEqual(symbol, timeSlice.Slice.Values.Single().Symbol, $"Slice doesn't contain {symbol}");
|
||||
}
|
||||
}
|
||||
|
||||
timer.Dispose();
|
||||
dataManager.RemoveAllSubscriptions();
|
||||
Assert.AreEqual(14, slicesEmitted);
|
||||
Assert.AreEqual(slicesEmitted, dataPointsEmitted);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the live data feed with an object store, simulating real-time updates from stored data.
|
||||
/// This method initializes the data feed, sets up a manual time provider, and stores data in the object store.
|
||||
/// </summary>
|
||||
/// <param name="startDate">The start date for the live data feed.</param>
|
||||
/// <param name="endDate">The end date for the live data feed.</param>
|
||||
/// <param name="cancellationTokenSource">A cancellation token to stop the live feed once the end date is reached.</param>
|
||||
/// <param name="objectPath">The object store path where data is saved.</param>
|
||||
/// <param name="objectText">The content (data) to be saved in the object store at the specified path.</param>
|
||||
/// <param name="AddData">A function that adds data to the algorithm and returns a symbol representing the data.</param>
|
||||
/// <returns>
|
||||
/// A tuple containing:
|
||||
/// <list type="bullet">
|
||||
/// <item><description><see cref="DataManagerStub"/> - The data manager handling data subscriptions for the algorithm.</description></item>
|
||||
/// <item><description><see cref="Timer"/> - The timer used to advance time and simulate live updates.</description></item>
|
||||
/// <item><description><see cref="Symbol"/> - The symbol of the added data in the algorithm.</description></item>
|
||||
/// </list>
|
||||
/// </returns>
|
||||
private (DataManagerStub, Timer, Symbol) RunLiveDataFeedWithObjectStore(DateTime startDate, DateTime endDate, CancellationTokenSource cancellationTokenSource,
|
||||
string objectPath, string objectText, Func<QCAlgorithm, Symbol> AddData)
|
||||
{
|
||||
using var api = new Api.Api();
|
||||
RemoteFileSubscriptionStreamReader.SetDownloadProvider(api);
|
||||
|
||||
var algorithm = new QCAlgorithm();
|
||||
|
||||
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
|
||||
timeProvider.SetCurrentTime(startDate);
|
||||
|
||||
CreateDataFeed(algorithm.Settings);
|
||||
var dataManager = new DataManagerStub(algorithm, _feed);
|
||||
algorithm.SubscriptionManager.SetDataManager(dataManager);
|
||||
|
||||
using var store = new LocalObjectStore();
|
||||
store.Initialize(0, 0, "", new Controls(), AlgorithmMode.Backtesting);
|
||||
algorithm.SetObjectStore(store);
|
||||
algorithm.ObjectStore.Save(objectPath, objectText);
|
||||
|
||||
var symbol = AddData(algorithm);
|
||||
|
||||
RunLiveDataFeed(algorithm, startDate, new[] { symbol }, timeProvider, dataManager);
|
||||
|
||||
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
|
||||
var timerInterval = TimeSpan.FromMilliseconds(10);
|
||||
var timer = Ref.Create<Timer>(null);
|
||||
timer.Value = new Timer(state =>
|
||||
{
|
||||
// stop the timer to prevent reentrancy
|
||||
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
|
||||
|
||||
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
|
||||
|
||||
if (currentTime.Date > endDate.Date)
|
||||
{
|
||||
_feed.Exit();
|
||||
cancellationTokenSource.Cancel();
|
||||
return;
|
||||
}
|
||||
|
||||
timeProvider.Advance(TimeSpan.FromHours(3));
|
||||
|
||||
// restart the timer
|
||||
timer.Value.Change(timerInterval, timerInterval);
|
||||
|
||||
}, null, TimeSpan.FromMilliseconds(500), timerInterval);
|
||||
|
||||
return (dataManager, timer.Value, symbol);
|
||||
}
|
||||
|
||||
private void CreateDataFeed(IAlgorithmSettings settings,
|
||||
FuncDataQueueHandler funcDataQueueHandler = null)
|
||||
{
|
||||
_feed = new TestableLiveTradingDataFeed(settings, funcDataQueueHandler ?? new FuncDataQueueHandler(x => Enumerable.Empty<BaseData>(), RealTimeProvider.Instance, new AlgorithmSettings()));
|
||||
}
|
||||
|
||||
private void RunLiveDataFeed(
|
||||
IAlgorithm algorithm,
|
||||
DateTime startDate,
|
||||
IEnumerable<Symbol> symbols,
|
||||
ITimeProvider timeProvider,
|
||||
DataManager dataManager)
|
||||
{
|
||||
_synchronizer = new TestableLiveSynchronizer(timeProvider);
|
||||
_synchronizer.Initialize(algorithm, dataManager, new());
|
||||
|
||||
_feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
|
||||
TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, dataManager, _synchronizer, new DataChannelProvider());
|
||||
|
||||
foreach (var symbol in symbols)
|
||||
{
|
||||
var config = algorithm.Securities[symbol].SubscriptionDataConfig;
|
||||
var request = new SubscriptionRequest(false, null, algorithm.Securities[symbol], config, startDate, Time.EndOfTime);
|
||||
dataManager.AddSubscription(request);
|
||||
}
|
||||
}
|
||||
|
||||
private static int _countFilesWritten;
|
||||
|
||||
public class TestableCustomFuture : BaseData
|
||||
{
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
// use local file instead of remote file
|
||||
var source = GetLocalFileName(config.Symbol.Value, "test");
|
||||
|
||||
return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile);
|
||||
}
|
||||
|
||||
public static string GetLocalFileName(string ticker, string fileExtension)
|
||||
{
|
||||
return $"./TestData/custom_future_{ticker.Replace("/", "_").ToLowerInvariant()}.{fileExtension}";
|
||||
}
|
||||
|
||||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var csv = line.Split(',');
|
||||
|
||||
var data = new TestableCustomFuture
|
||||
{
|
||||
Symbol = config.Symbol,
|
||||
Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture),
|
||||
Value = csv[6].ToDecimal()
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
public class TestableRemoteCustomData : BaseData
|
||||
{
|
||||
public override DateTime EndTime
|
||||
{
|
||||
get { return Time + Period; }
|
||||
set { Time = value - Period; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a time span of one day
|
||||
/// </summary>
|
||||
public TimeSpan Period
|
||||
{
|
||||
get { return QuantConnect.Time.OneDay; }
|
||||
}
|
||||
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var source = $"https://www.dl.dropboxusercontent.com/s/1w6x1kfrlvx3d2v/CustomIBM.csv?dl=0";
|
||||
return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile);
|
||||
}
|
||||
|
||||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var csv = line.Split(',');
|
||||
|
||||
var data = new TestableRemoteCustomData
|
||||
{
|
||||
Symbol = config.Symbol,
|
||||
Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture),
|
||||
Value = csv[1].ToDecimal()
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
public class TestableObjectStoreCustomData : TestableRemoteCustomData
|
||||
{
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
return new SubscriptionDataSource("CustomData/CustomIBM", SubscriptionTransportMedium.ObjectStore, FileFormat.Csv);
|
||||
}
|
||||
}
|
||||
|
||||
private class CustomDataSort : TestableRemoteCustomData
|
||||
{
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
return new SubscriptionDataSource("CustomData/CustomIBM", SubscriptionTransportMedium.ObjectStore, FileFormat.Csv) { Sort = true };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using QuantConnect.Data;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
/// <summary>
|
||||
/// Custom data type that uses a remote file download
|
||||
/// </summary>
|
||||
public class CustomMockedFileBaseData : BaseData
|
||||
{
|
||||
private int _incrementsToAdd;
|
||||
public static DateTime StartDate;
|
||||
|
||||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var start = StartDate.AddTicks(config.Increment.Ticks * _incrementsToAdd++).ConvertFromUtc(config.DataTimeZone);
|
||||
return new CustomMockedFileBaseData
|
||||
{
|
||||
Symbol = config.Symbol,
|
||||
Time = start,
|
||||
EndTime = start.Add(config.Increment),
|
||||
Value = 10
|
||||
};
|
||||
}
|
||||
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
// using an already existing file, the file it self and its contents don't really matter. The reader will mock the values
|
||||
return new SubscriptionDataSource("./TestData/spy_with_ichimoku.csv", SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.DataDownloader;
|
||||
using System;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.DataDownloader
|
||||
{
|
||||
[TestFixture]
|
||||
public class CanonicalDataDownloaderDecoratorTests
|
||||
{
|
||||
[TestCase("2026/03/20", "2025/03/01", "2025/12/31", "2025/03/01", "2025/12/31")]
|
||||
[TestCase("2026/03/20", "2025/03/01", "2026/03/25", "2025/03/01", "2026/03/21")]
|
||||
[TestCase("2026/03/20", "2020/01/01", "2026/03/25", "2024/03/20", "2026/03/21")]
|
||||
[TestCase("2022/02/28", "2018/01/01", "2020/12/31", "2020/02/28", "2020/12/31")]
|
||||
[TestCase("2022/02/28", "2018/01/01", "2019/01/01", default, default)]
|
||||
[TestCase("2022/02/28", "2022/01/01", "2019/01/01", default, default)]
|
||||
[TestCase("2022/02/28", "2022/03/01", "2022/03/02", default, default)]
|
||||
[TestCase("2022/02/28", "2022/02/01 9:30", "2022/02/10 16:00", "2022/02/01 9:30", "2022/02/10 16:00")]
|
||||
[TestCase("2022/02/28", "2022/02/01 9:30", "2022/02/01 9:30", "2022/02/01 9:30", "2022/02/01 9:30")]
|
||||
[TestCase("2022/02/28", "2018/02/01 9:30", "2022/03/01 9:30", "2020/02/28", "2022/03/01")]
|
||||
[TestCase("2022/02/28", "2018/02/01 9:30", "2018/03/01 9:30", default, default)]
|
||||
[TestCase("2022/02/28", "1997/12/31", "2026/02/20", "2020/02/28", "2022/03/01")]
|
||||
public void ShouldReceiveAdjustedDateFutureContract(DateTime expiry, DateTime startUtc, DateTime endUtc, DateTime expectedStart, DateTime expectedEnd)
|
||||
{
|
||||
var future = Symbol.CreateFuture(Securities.Futures.Indices.SP500EMini, Market.CME, expiry);
|
||||
|
||||
CanonicalDataDownloaderDecorator.TryAdjustDateRangeForContract(future, startUtc, endUtc, out var start, out var end);
|
||||
|
||||
Assert.AreEqual(expectedStart, start);
|
||||
Assert.AreEqual(expectedEnd, end);
|
||||
}
|
||||
|
||||
[TestCase("2026/02/20", "2025/03/01", "2025/12/31", "2025/03/01", "2025/12/31")]
|
||||
[TestCase("2026/02/20", "2025/02/18", "2026/03/25", "2025/02/20", "2026/02/21")]
|
||||
[TestCase("2026/02/20", "2020/01/01", "2026/03/25", "2025/02/20", "2026/02/21")]
|
||||
[TestCase("2026/02/20", "1997/12/31", "2026/02/20", "2025/02/20", "2026/02/21")]
|
||||
[TestCase("2020/02/20", "2020/01/01", "2021/03/25", "2020/01/01", "2020/02/21")]
|
||||
[TestCase("2020/02/20", "2019/01/01", "2021/03/25", "2019/02/20", "2020/02/21")]
|
||||
[TestCase("2020/02/20", "2018/01/01", "2020/02/25", "2019/02/20", "2020/02/21")]
|
||||
[TestCase("2020/02/20", "2019/08/10", "2020/02/20", "2019/08/10", "2020/02/21")]
|
||||
[TestCase("2020/02/20", "2018/01/01", "2018/02/01", default, default)]
|
||||
[TestCase("2020/02/20", "2021/01/01", "2022/02/01", default, default)]
|
||||
[TestCase("2020/02/20", "2020/01/01 9:30", "2020/02/20 16:00", "2020/01/01 9:30", "2020/02/21")]
|
||||
[TestCase("2020/02/20", "2019/01/01 9:30", "2020/02/20 16:00", "2019/02/20", "2020/02/21")]
|
||||
[TestCase("2020/02/20", "2019/01/01 9:30", "2019/02/01 9:30", default, default)]
|
||||
[TestCase("2020/02/20", "2019/02/20 9:30", "2020/02/20", "2019/02/20 9:30", "2020/02/21")]
|
||||
[TestCase("2020/02/20", "2020/02/20 9:30", "2020/02/21", default, default)]
|
||||
public void ShouldReceiveAdjustedDateOptionContract(DateTime expiry, DateTime startUtc, DateTime endUtc, DateTime expectedStart, DateTime expectedEnd)
|
||||
{
|
||||
var aapl = Symbol.Create("AAPL", SecurityType.Equity, Market.USA);
|
||||
var option = Symbol.CreateOption(aapl, aapl.ID.Market, aapl.SecurityType.DefaultOptionStyle(), OptionRight.Call, 260m, expiry);
|
||||
|
||||
CanonicalDataDownloaderDecorator.TryAdjustDateRangeForContract(option, startUtc, endUtc, out var start, out var end);
|
||||
|
||||
Assert.AreEqual(expectedStart, start);
|
||||
Assert.AreEqual(expectedEnd, end);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ShouldNotAdjustNonOptionOrFutureContract()
|
||||
{
|
||||
var aapl = Symbol.Create("AAPL", SecurityType.Equity, Market.USA);
|
||||
var res = CanonicalDataDownloaderDecorator.TryAdjustDateRangeForContract(aapl, new DateTime(2025, 03, 01), new DateTime(2025, 12, 31), out var start, out var end);
|
||||
Assert.IsFalse(res);
|
||||
Assert.AreEqual(default(DateTime), start);
|
||||
Assert.AreEqual(default(DateTime), end);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Data.Market;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.DataDownloader;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.DataDownloader
|
||||
{
|
||||
[TestFixture]
|
||||
public class DataDownloaderSelectorTests
|
||||
{
|
||||
private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>("LocalDiskMapFileProvider");
|
||||
|
||||
[Test]
|
||||
public void GetDataDownloaderReturnsBaseDownloaderForNonLeanType()
|
||||
{
|
||||
var baseDownloader = new TestDataDownloader();
|
||||
using var selector = new DataDownloaderSelector(baseDownloader, _mapFileProvider, new TestDataProvider());
|
||||
|
||||
var actualBaseDownloader = selector.GetDataDownloader(typeof(object));
|
||||
|
||||
Assert.AreSame(baseDownloader, actualBaseDownloader);
|
||||
|
||||
var actualCanonicalDownloader = selector.GetDataDownloader(typeof(TradeBar));
|
||||
|
||||
Assert.AreEqual(typeof(CanonicalDataDownloaderDecorator), actualCanonicalDownloader.GetType());
|
||||
}
|
||||
|
||||
private class TestDataDownloader : IDataDownloader
|
||||
{
|
||||
public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
|
||||
private class TestDataProvider : IDataProvider
|
||||
{
|
||||
public event EventHandler<DataProviderNewDataRequestEventArgs> NewDataRequest;
|
||||
|
||||
public Stream Fetch(string key)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using QuantConnect.Algorithm;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Securities;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
internal class DataManagerStub : DataManager
|
||||
{
|
||||
public ISecurityService SecurityService { get; }
|
||||
public IAlgorithm Algorithm { get; }
|
||||
public IDataFeed DataFeed { get; }
|
||||
|
||||
public DataManagerStub()
|
||||
: this(new QCAlgorithm())
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public DataManagerStub(ITimeKeeper timeKeeper)
|
||||
: this(new QCAlgorithm(), timeKeeper)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public DataManagerStub(IAlgorithm algorithm, IDataFeed dataFeed, bool liveMode = false)
|
||||
: this(dataFeed, algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork), liveMode)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public DataManagerStub(IAlgorithm algorithm)
|
||||
: this(new NullDataFeed(), algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork))
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public DataManagerStub(IDataFeed dataFeed, IAlgorithm algorithm)
|
||||
: this(dataFeed, algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork))
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public DataManagerStub(IAlgorithm algorithm, ITimeKeeper timeKeeper)
|
||||
: this(new NullDataFeed(), algorithm, timeKeeper)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public DataManagerStub(IDataFeed dataFeed, IAlgorithm algorithm, ITimeKeeper timeKeeper, bool liveMode = false)
|
||||
: this(dataFeed, algorithm, timeKeeper, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder(), liveMode)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public DataManagerStub(IDataFeed dataFeed, IAlgorithm algorithm, ITimeKeeper timeKeeper, MarketHoursDatabase marketHoursDatabase, SymbolPropertiesDatabase symbolPropertiesDatabase, bool liveMode = false)
|
||||
: this(dataFeed, algorithm, timeKeeper, marketHoursDatabase,
|
||||
new SecurityService(algorithm.Portfolio.CashBook,
|
||||
marketHoursDatabase,
|
||||
symbolPropertiesDatabase,
|
||||
algorithm,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCacheProvider(algorithm.Portfolio),
|
||||
algorithm: algorithm),
|
||||
liveMode)
|
||||
{
|
||||
}
|
||||
|
||||
public DataManagerStub(IDataFeed dataFeed, IAlgorithm algorithm, ITimeKeeper timeKeeper, MarketHoursDatabase marketHoursDatabase, SecurityService securityService, bool liveMode = false)
|
||||
: this(dataFeed,
|
||||
algorithm,
|
||||
timeKeeper,
|
||||
marketHoursDatabase,
|
||||
securityService,
|
||||
new DataPermissionManager(),
|
||||
liveMode)
|
||||
{
|
||||
}
|
||||
|
||||
public DataManagerStub(IDataFeed dataFeed, IAlgorithm algorithm, ITimeKeeper timeKeeper, MarketHoursDatabase marketHoursDatabase, SecurityService securityService, DataPermissionManager dataPermissionManager, bool liveMode = false)
|
||||
: base(dataFeed,
|
||||
new UniverseSelection(algorithm, securityService, dataPermissionManager, TestGlobals.DataProvider),
|
||||
algorithm,
|
||||
timeKeeper,
|
||||
marketHoursDatabase,
|
||||
liveMode,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
dataPermissionManager)
|
||||
{
|
||||
SecurityService = securityService;
|
||||
algorithm.Securities.SetSecurityService(securityService);
|
||||
Algorithm = algorithm;
|
||||
DataFeed = dataFeed;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using NodaTime;
|
||||
using NUnit.Framework;
|
||||
using NUnit.Framework.Internal;
|
||||
using QuantConnect.Algorithm;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using QuantConnect.Lean.Engine.Results;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Securities.Equity;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class DataManagerTests
|
||||
{
|
||||
private QCAlgorithm _algorithm;
|
||||
private SecurityService _securityService;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_algorithm = new AlgorithmStub();
|
||||
_securityService = new SecurityService(_algorithm.Portfolio.CashBook,
|
||||
MarketHoursDatabase.AlwaysOpen,
|
||||
SymbolPropertiesDatabase.FromDataFolder(),
|
||||
_algorithm,
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
new SecurityCacheProvider(_algorithm.Portfolio),
|
||||
algorithm: _algorithm);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReturnsExistingConfig()
|
||||
{
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
var dataManager = new DataManager(new NullDataFeed(),
|
||||
new UniverseSelection(_algorithm,
|
||||
_securityService,
|
||||
dataPermissionManager,
|
||||
TestGlobals.DataProvider),
|
||||
_algorithm,
|
||||
_algorithm.TimeKeeper,
|
||||
MarketHoursDatabase.AlwaysOpen,
|
||||
false,
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
dataPermissionManager);
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
false,
|
||||
false,
|
||||
false);
|
||||
|
||||
var sameConfig = dataManager.SubscriptionManagerGetOrAdd(config);
|
||||
Assert.IsTrue(ReferenceEquals(sameConfig, config));
|
||||
Assert.AreEqual(1, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
|
||||
|
||||
var otherInstance = new SubscriptionDataConfig(config.Type,
|
||||
config.Symbol,
|
||||
config.Resolution,
|
||||
config.DataTimeZone,
|
||||
config.ExchangeTimeZone,
|
||||
config.FillDataForward,
|
||||
config.ExtendedMarketHours,
|
||||
config.IsInternalFeed);
|
||||
|
||||
sameConfig = dataManager.SubscriptionManagerGetOrAdd(otherInstance);
|
||||
Assert.IsTrue(ReferenceEquals(sameConfig, config));
|
||||
Assert.AreEqual(1, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
|
||||
|
||||
dataManager.RemoveAllSubscriptions();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RemovesExistingConfig()
|
||||
{
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
var dataFeed = new TestDataFeed();
|
||||
var dataManager = new DataManager(dataFeed,
|
||||
new UniverseSelection(_algorithm,
|
||||
_securityService,
|
||||
dataPermissionManager,
|
||||
TestGlobals.DataProvider),
|
||||
_algorithm,
|
||||
_algorithm.TimeKeeper,
|
||||
MarketHoursDatabase.AlwaysOpen,
|
||||
false,
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
dataPermissionManager);
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
false,
|
||||
false,
|
||||
false);
|
||||
|
||||
Assert.IsTrue(ReferenceEquals(dataManager.SubscriptionManagerGetOrAdd(config), config));
|
||||
Assert.AreEqual(1, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
|
||||
|
||||
// we didn't add any subscription yet
|
||||
Assert.IsFalse(dataManager.RemoveSubscription(config));
|
||||
|
||||
var request = new SubscriptionRequest(false,
|
||||
null,
|
||||
new Security(Symbols.SPY,
|
||||
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
|
||||
new Cash(Currencies.USD, 1, 1),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
new IdentityCurrencyConverter(Currencies.USD),
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
new SecurityCache()),
|
||||
config,
|
||||
new DateTime(2019, 1, 1),
|
||||
new DateTime(2019, 1, 1));
|
||||
|
||||
using var enquableEnumerator = new EnqueueableEnumerator<SubscriptionData>();
|
||||
dataFeed.Subscription = new Subscription(request,
|
||||
enquableEnumerator,
|
||||
null);
|
||||
|
||||
Assert.IsTrue(dataManager.AddSubscription(request));
|
||||
Assert.IsTrue(dataManager.RemoveSubscription(config));
|
||||
Assert.AreEqual(0, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
|
||||
|
||||
dataManager.RemoveAllSubscriptions();
|
||||
}
|
||||
|
||||
[Test]
|
||||
// reproduces GH issue 3877
|
||||
public void ConfigurationForAddedSubscriptionIsAlwaysPresent()
|
||||
{
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
var dataFeed = new TestDataFeed();
|
||||
var dataManager = new DataManager(dataFeed,
|
||||
new UniverseSelection(_algorithm,
|
||||
_securityService,
|
||||
dataPermissionManager,
|
||||
TestGlobals.DataProvider),
|
||||
_algorithm,
|
||||
_algorithm.TimeKeeper,
|
||||
MarketHoursDatabase.AlwaysOpen,
|
||||
false,
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
dataPermissionManager);
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
false,
|
||||
false,
|
||||
false);
|
||||
|
||||
// Universe A: adds the config
|
||||
dataManager.SubscriptionManagerGetOrAdd(config);
|
||||
|
||||
var request = new SubscriptionRequest(false,
|
||||
null,
|
||||
new Security(Symbols.SPY,
|
||||
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
|
||||
new Cash(Currencies.USD, 1, 1),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
new IdentityCurrencyConverter(Currencies.USD),
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
new SecurityCache()),
|
||||
config,
|
||||
new DateTime(2019, 1, 1),
|
||||
new DateTime(2019, 1, 1));
|
||||
|
||||
using var enquableEnumerator = new EnqueueableEnumerator<SubscriptionData>();
|
||||
dataFeed.Subscription = new Subscription(request,
|
||||
enquableEnumerator,
|
||||
null);
|
||||
|
||||
// Universe A: adds the subscription
|
||||
dataManager.AddSubscription(request);
|
||||
|
||||
// Universe B: adds the config
|
||||
dataManager.SubscriptionManagerGetOrAdd(config);
|
||||
|
||||
// Universe A: removes the subscription
|
||||
Assert.IsTrue(dataManager.RemoveSubscription(config));
|
||||
Assert.AreEqual(0, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
|
||||
|
||||
// Universe B: adds the subscription
|
||||
Assert.IsTrue(dataManager.AddSubscription(request));
|
||||
|
||||
// the config should be present
|
||||
Assert.AreEqual(1, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
|
||||
|
||||
dataManager.RemoveAllSubscriptions();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ScaledRawNormalizationModeIsNotAllowed()
|
||||
{
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
var dataFeed = new TestDataFeed();
|
||||
var dataManager = new DataManager(dataFeed,
|
||||
new UniverseSelection(_algorithm,
|
||||
_securityService,
|
||||
dataPermissionManager,
|
||||
TestGlobals.DataProvider),
|
||||
_algorithm,
|
||||
_algorithm.TimeKeeper,
|
||||
MarketHoursDatabase.AlwaysOpen,
|
||||
false,
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
dataPermissionManager);
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
dataNormalizationMode: DataNormalizationMode.ScaledRaw);
|
||||
|
||||
using var universe = new TestUniverse(
|
||||
config,
|
||||
new UniverseSettings(Resolution.Daily, 1, false, false, TimeSpan.FromDays(365)));
|
||||
var security = new Equity(
|
||||
config.Symbol,
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
|
||||
new Cash(Currencies.USD, 1, 1),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
new IdentityCurrencyConverter(Currencies.USD),
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
new SecurityCache());
|
||||
|
||||
var subscriptionRequest = new SubscriptionRequest(
|
||||
false,
|
||||
universe,
|
||||
security,
|
||||
config,
|
||||
new DateTime(2014, 10, 10),
|
||||
new DateTime(2015, 10, 10));
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(() =>
|
||||
{
|
||||
dataManager.AddSubscription(subscriptionRequest);
|
||||
});
|
||||
Assert.That(exception.Message, Does.Contain(nameof(DataNormalizationMode.ScaledRaw)));
|
||||
}
|
||||
|
||||
private class TestDataFeed : IDataFeed
|
||||
{
|
||||
public Subscription Subscription { get; set; }
|
||||
|
||||
public bool IsActive { get; }
|
||||
|
||||
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler,
|
||||
IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider,
|
||||
IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider,
|
||||
IDataChannelProvider channelProvider)
|
||||
{
|
||||
}
|
||||
|
||||
public Subscription CreateSubscription(SubscriptionRequest request)
|
||||
{
|
||||
return Subscription;
|
||||
}
|
||||
|
||||
public void RemoveSubscription(Subscription subscription)
|
||||
{
|
||||
}
|
||||
|
||||
public void Exit()
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private class TestUniverse : Universe
|
||||
{
|
||||
public TestUniverse(SubscriptionDataConfig config, UniverseSettings universeSettings)
|
||||
: base(config)
|
||||
{
|
||||
UniverseSettings = universeSettings;
|
||||
}
|
||||
public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Algorithm.CSharp;
|
||||
using QuantConnect.Configuration;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Packets;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class DataPermissionManagerTests
|
||||
{
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
Config.Set("data-permission-manager", "TestDataPermissionManager");
|
||||
TestInvalidConfigurationAlgorithm.Count = 0;
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
Config.Reset();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void InvalidConfigurationAddSecurity()
|
||||
{
|
||||
var parameter = new RegressionTests.AlgorithmStatisticsTestParameters(nameof(BasicTemplateDailyAlgorithm),
|
||||
new Dictionary<string, string>(),
|
||||
Language.CSharp,
|
||||
// will throw on initialization
|
||||
AlgorithmStatus.Running);
|
||||
|
||||
var result = AlgorithmRunner.RunLocalBacktest(parameter.Algorithm,
|
||||
parameter.Statistics,
|
||||
parameter.Language,
|
||||
parameter.ExpectedFinalStatus);
|
||||
|
||||
// algorithm was never set
|
||||
Assert.IsEmpty(result.AlgorithmManager.AlgorithmId);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void InvalidConfigurationHistoryRequest()
|
||||
{
|
||||
var parameter = new RegressionTests.AlgorithmStatisticsTestParameters(nameof(TestInvalidConfigurationAlgorithm),
|
||||
new Dictionary<string, string>(),
|
||||
Language.CSharp,
|
||||
// will throw on initialization
|
||||
AlgorithmStatus.Running);
|
||||
|
||||
var result = AlgorithmRunner.RunLocalBacktest(parameter.Algorithm,
|
||||
parameter.Statistics,
|
||||
parameter.Language,
|
||||
parameter.ExpectedFinalStatus,
|
||||
setupHandler: "TestInvalidConfigurationSetupHandler");
|
||||
|
||||
// algorithm was never set
|
||||
Assert.IsEmpty(result.AlgorithmManager.AlgorithmId);
|
||||
// let's assert initialize was called by the history call failed
|
||||
Assert.AreEqual(1, TestInvalidConfigurationAlgorithm.Count);
|
||||
}
|
||||
|
||||
public class TestDataPermissionManager : DataPermissionManager
|
||||
{
|
||||
public override void AssertConfiguration(SubscriptionDataConfig subscriptionDataConfig, DateTime startTimeLocal, DateTime endTimeLocal)
|
||||
{
|
||||
throw new InvalidOperationException("Invalid configuration");
|
||||
}
|
||||
}
|
||||
|
||||
public class TestInvalidConfigurationAlgorithm : BasicTemplateDailyAlgorithm
|
||||
{
|
||||
public static int Count;
|
||||
public override void Initialize()
|
||||
{
|
||||
Count++;
|
||||
#pragma warning disable CS0618
|
||||
History("SPY", 1, Resolution.Tick).ToList();
|
||||
#pragma warning restore CS0618
|
||||
Count++;
|
||||
}
|
||||
}
|
||||
|
||||
public class TestInvalidConfigurationSetupHandler : AlgorithmRunner.RegressionSetupHandlerWrapper
|
||||
{
|
||||
public override IAlgorithm CreateAlgorithmInstance(AlgorithmNodePacket algorithmNodePacket, string assemblyPath)
|
||||
{
|
||||
Algorithm = new TestInvalidConfigurationAlgorithm();
|
||||
return Algorithm;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Queues;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class DataQueueHandlerManagerTests
|
||||
{
|
||||
[TestCase("BacktestingBrokerage")]
|
||||
public void GetFactoryFromDataQueueHandler(string dataQueueHandler)
|
||||
{
|
||||
var factory = JobQueue.GetFactoryFromDataQueueHandler(dataQueueHandler);
|
||||
Assert.NotNull(factory);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SetJob()
|
||||
{
|
||||
//Array IDQH
|
||||
var dataHandlers = Newtonsoft.Json.JsonConvert.SerializeObject(new[] { "FakeDataQueue" });
|
||||
var jobWithArrayIDQH = new LiveNodePacket
|
||||
{
|
||||
DataQueueHandler = dataHandlers
|
||||
};
|
||||
var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
compositeDataQueueHandler.SetJob(jobWithArrayIDQH);
|
||||
compositeDataQueueHandler.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SubscribeReturnsNull()
|
||||
{
|
||||
var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
var enumerator = compositeDataQueueHandler.Subscribe(GetConfig(), (_, _) => {});
|
||||
Assert.Null(enumerator);
|
||||
compositeDataQueueHandler.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SubscribeReturnsNotNull()
|
||||
{
|
||||
var dataHandlers = Newtonsoft.Json.JsonConvert.SerializeObject(new[] { "FakeDataQueue" });
|
||||
var job = new LiveNodePacket
|
||||
{
|
||||
DataQueueHandler = dataHandlers
|
||||
};
|
||||
var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
compositeDataQueueHandler.SetJob(job);
|
||||
var enumerator = compositeDataQueueHandler.Subscribe(GetConfig(), (_, _) => {});
|
||||
Assert.NotNull(enumerator);
|
||||
compositeDataQueueHandler.Dispose();
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Unsubscribe()
|
||||
{
|
||||
var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
compositeDataQueueHandler.Unsubscribe(GetConfig());
|
||||
compositeDataQueueHandler.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void IsNotUniverseProvider()
|
||||
{
|
||||
var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
Assert.IsFalse(compositeDataQueueHandler.HasUniverseProvider);
|
||||
Assert.Throws<NotSupportedException>(() => compositeDataQueueHandler.LookupSymbols(Symbols.ES_Future_Chain, false));
|
||||
Assert.Throws<NotSupportedException>(() => compositeDataQueueHandler.CanPerformSelection());
|
||||
compositeDataQueueHandler.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DoubleSubscribe()
|
||||
{
|
||||
var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
compositeDataQueueHandler.SetJob(new LiveNodePacket { DataQueueHandler = "[ \"TestDataHandler\" ]" });
|
||||
|
||||
var dataConfig = GetConfig();
|
||||
var enumerator = compositeDataQueueHandler.Subscribe(dataConfig, (_, _) => {});
|
||||
|
||||
Assert.DoesNotThrow(() => compositeDataQueueHandler.Subscribe(dataConfig, (_, _) => { }));
|
||||
compositeDataQueueHandler.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SingleSubscribe()
|
||||
{
|
||||
TestDataHandler.UnsubscribeCounter = 0;
|
||||
var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
compositeDataQueueHandler.SetJob(new LiveNodePacket { DataQueueHandler = "[ \"TestDataHandler\" ]" });
|
||||
|
||||
var dataConfig = GetConfig();
|
||||
var enumerator = compositeDataQueueHandler.Subscribe(dataConfig, (_, _) => {});
|
||||
|
||||
compositeDataQueueHandler.Unsubscribe(dataConfig);
|
||||
compositeDataQueueHandler.Unsubscribe(dataConfig);
|
||||
compositeDataQueueHandler.Unsubscribe(dataConfig);
|
||||
|
||||
Assert.AreEqual(1, TestDataHandler.UnsubscribeCounter);
|
||||
|
||||
compositeDataQueueHandler.Dispose();
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public void MappedConfig(bool canonicalUnsubscribeFirst)
|
||||
{
|
||||
TestDataHandler.UnsubscribeCounter = 0;
|
||||
TestDataHandler.SubscribeCounter = 0;
|
||||
var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
compositeDataQueueHandler.SetJob(new LiveNodePacket { DataQueueHandler = "[ \"TestDataHandler\" ]" });
|
||||
|
||||
var canonicalSymbol = Symbols.ES_Future_Chain.UpdateMappedSymbol(Symbols.Future_ESZ18_Dec2018.ID.ToString());
|
||||
var canonicalConfig = GetConfig(canonicalSymbol);
|
||||
var contractConfig = GetConfig(Symbols.Future_ESZ18_Dec2018);
|
||||
|
||||
var enumerator = new LiveSubscriptionEnumerator(canonicalConfig, compositeDataQueueHandler, (_, _) => {}, (_) => false);
|
||||
var enumerator2 = new LiveSubscriptionEnumerator(contractConfig, compositeDataQueueHandler, (_, _) => {}, (_) => false);
|
||||
|
||||
var firstUnsubscribe = canonicalUnsubscribeFirst ? canonicalConfig : contractConfig;
|
||||
var secondUnsubscribe = canonicalUnsubscribeFirst ? contractConfig : canonicalConfig;
|
||||
|
||||
Assert.AreEqual(2, TestDataHandler.SubscribeCounter);
|
||||
|
||||
compositeDataQueueHandler.UnsubscribeWithMapping(firstUnsubscribe);
|
||||
Assert.AreEqual(1, TestDataHandler.UnsubscribeCounter);
|
||||
|
||||
compositeDataQueueHandler.UnsubscribeWithMapping(secondUnsubscribe);
|
||||
Assert.AreEqual(2, TestDataHandler.UnsubscribeCounter);
|
||||
|
||||
enumerator.Dispose();
|
||||
enumerator2.Dispose();
|
||||
compositeDataQueueHandler.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void HandleExplodingDataQueueHandler()
|
||||
{
|
||||
using var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
// first exploding
|
||||
compositeDataQueueHandler.SetJob(new LiveNodePacket { DataQueueHandler = "[ \"ExplodingDataHandler\", \"TestDataHandler\" ]" });
|
||||
IEnumerator<BaseData> enumerator = null;
|
||||
Assert.DoesNotThrow(() =>
|
||||
{
|
||||
enumerator = compositeDataQueueHandler.Subscribe(GetConfig(), (_, _) => { });
|
||||
});
|
||||
Assert.IsNotNull(enumerator);
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ExplodingDataQueueHandler()
|
||||
{
|
||||
using var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
compositeDataQueueHandler.SetJob(new LiveNodePacket { DataQueueHandler = "[ \"ExplodingDataHandler\" ]" });
|
||||
Assert.Throws<Exception>(() =>
|
||||
{
|
||||
using var enumerator = compositeDataQueueHandler.Subscribe(GetConfig(), (_, _) => { });
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void HandlesCustomData()
|
||||
{
|
||||
var customSymbol = Symbol.CreateBase(typeof(AlgorithmSettings), Symbols.SPY);
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar), customSymbol, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork,
|
||||
false, false, false, false, TickType.Trade, false);
|
||||
var dataHandlers = Newtonsoft.Json.JsonConvert.SerializeObject(new[] { "FakeDataQueue" });
|
||||
var job = new LiveNodePacket
|
||||
{
|
||||
DataQueueHandler = dataHandlers
|
||||
};
|
||||
var compositeDataQueueHandler = new DataQueueHandlerManager(new AlgorithmSettings());
|
||||
compositeDataQueueHandler.SetJob(job);
|
||||
var enumerator = compositeDataQueueHandler.Subscribe(config, (_, _) => { });
|
||||
Assert.NotNull(enumerator);
|
||||
compositeDataQueueHandler.Dispose();
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
private static SubscriptionDataConfig GetConfig(Symbol symbol = null)
|
||||
{
|
||||
return new SubscriptionDataConfig(typeof(TradeBar), symbol ?? Symbols.SPY, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork,
|
||||
false, false, false, false, TickType.Trade, false);
|
||||
}
|
||||
|
||||
|
||||
private class TestDataHandler : IDataQueueHandler
|
||||
{
|
||||
public static int SubscribeCounter { get; set; }
|
||||
|
||||
public static int UnsubscribeCounter { get; set; }
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
public virtual IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
|
||||
{
|
||||
SubscribeCounter++;
|
||||
return Enumerable.Empty<BaseData>().GetEnumerator();
|
||||
}
|
||||
|
||||
public void Unsubscribe(SubscriptionDataConfig dataConfig)
|
||||
{
|
||||
UnsubscribeCounter++;
|
||||
}
|
||||
|
||||
public void SetJob(LiveNodePacket job)
|
||||
{
|
||||
}
|
||||
|
||||
public bool IsConnected { get; }
|
||||
}
|
||||
|
||||
private class ExplodingDataHandler : TestDataHandler
|
||||
{
|
||||
public override IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
|
||||
{
|
||||
throw new Exception("ExplodingDataHandler exception!");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NodaTime;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class DateChangeTimeKeeperTests
|
||||
{
|
||||
private static DateTime _start = new DateTime(2024, 10, 01);
|
||||
private static DateTime _end = new DateTime(2024, 10, 11);
|
||||
|
||||
private static TestCaseData[] TimeZonesTestCases => new TestCaseData[]
|
||||
{
|
||||
new(TimeZones.Utc, DateTimeZone.ForOffset(Offset.FromHours(-1))),
|
||||
new(TimeZones.Utc, DateTimeZone.ForOffset(Offset.FromHours(-2))),
|
||||
new(TimeZones.Utc, DateTimeZone.ForOffset(Offset.FromHours(-3))),
|
||||
new(TimeZones.Utc, DateTimeZone.ForOffset(Offset.FromHours(-4))),
|
||||
new(TimeZones.Utc, DateTimeZone.ForOffset(Offset.FromHours(1))),
|
||||
new(TimeZones.Utc, DateTimeZone.ForOffset(Offset.FromHours(2))),
|
||||
new(TimeZones.Utc, DateTimeZone.ForOffset(Offset.FromHours(3))),
|
||||
new(TimeZones.Utc, DateTimeZone.ForOffset(Offset.FromHours(4))),
|
||||
new(TimeZones.Utc, TimeZones.Utc),
|
||||
new(TimeZones.NewYork, TimeZones.NewYork),
|
||||
new(TimeZones.HongKong, TimeZones.HongKong),
|
||||
};
|
||||
|
||||
[TestCaseSource(nameof(TimeZonesTestCases))]
|
||||
public void EmitsFirstExchangeDateEvent(DateTimeZone dataTimeZone, DateTimeZone exchangeTimeZone)
|
||||
{
|
||||
using var timeKeeper = GetTimeKeeper(dataTimeZone, exchangeTimeZone, true, null, out var tradableDates, out var config, out var exchangeHours);
|
||||
|
||||
var emittedExchangeDates = new List<DateTime>();
|
||||
void HandleNewTradableDate(object sender, DateTime date)
|
||||
{
|
||||
emittedExchangeDates.Add(date);
|
||||
}
|
||||
|
||||
timeKeeper.NewExchangeDate += HandleNewTradableDate;
|
||||
|
||||
var firstDataDate = tradableDates[0];
|
||||
var firstDataDateInExchangeTimeZone = firstDataDate.ConvertTo(dataTimeZone, exchangeTimeZone);
|
||||
var firstExchangeDateIsBeforeFirstDataDate = firstDataDateInExchangeTimeZone < firstDataDate;
|
||||
|
||||
try
|
||||
{
|
||||
Assert.IsTrue(timeKeeper.TryAdvanceUntilNextDataDate());
|
||||
Assert.AreEqual(1, emittedExchangeDates.Count);
|
||||
|
||||
if (firstExchangeDateIsBeforeFirstDataDate)
|
||||
{
|
||||
// The first exchange date is the day before the first data date when the exchange is behind the data
|
||||
var expectedFirstExchangeDate = firstDataDate.AddDays(-1);
|
||||
Assert.AreEqual(expectedFirstExchangeDate, emittedExchangeDates[0]);
|
||||
Assert.AreEqual(expectedFirstExchangeDate.ConvertTo(exchangeTimeZone, dataTimeZone), timeKeeper.DataTime);
|
||||
}
|
||||
else
|
||||
{
|
||||
// If exchange is ahead of the data, even though technically at the first data date the exchange date hasn't changed,
|
||||
// we emit the first one so that first daily actions are performed (mappings, delistings, etc).
|
||||
Assert.AreEqual(firstDataDate, emittedExchangeDates[0]);
|
||||
Assert.AreEqual(firstDataDate, timeKeeper.DataTime);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
timeKeeper.NewExchangeDate -= HandleNewTradableDate;
|
||||
}
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(TimeZonesTestCases))]
|
||||
public void ExchangeDatesAreEmittedByAdvancingToNextDataDate(DateTimeZone dataTimeZone, DateTimeZone exchangeTimeZone)
|
||||
{
|
||||
using var timeKeeper = GetTimeKeeper(dataTimeZone, exchangeTimeZone, false, null, out var tradableDates, out var config, out var exchangeHours);
|
||||
|
||||
var exchangeDateEmitted = false;
|
||||
var emittedExchangeDates = new List<DateTime>();
|
||||
void HandleNewTradableDate(object sender, DateTime date)
|
||||
{
|
||||
emittedExchangeDates.Add(date);
|
||||
exchangeDateEmitted = true;
|
||||
}
|
||||
|
||||
timeKeeper.NewExchangeDate += HandleNewTradableDate;
|
||||
|
||||
var expectedExchangeDates = new List<DateTime>()
|
||||
{
|
||||
new(2024, 10, 1),
|
||||
new(2024, 10, 2),
|
||||
new(2024, 10, 3),
|
||||
new(2024, 10, 4),
|
||||
new(2024, 10, 7),
|
||||
new(2024, 10, 8),
|
||||
new(2024, 10, 9),
|
||||
new(2024, 10, 10),
|
||||
new(2024, 10, 11),
|
||||
};
|
||||
|
||||
var firstDataDate = tradableDates[0];
|
||||
var firstDataDateInExchangeTimeZone = firstDataDate.ConvertTo(dataTimeZone, exchangeTimeZone);
|
||||
var exchangeIsBehindData = firstDataDateInExchangeTimeZone < firstDataDate;
|
||||
if (exchangeIsBehindData)
|
||||
{
|
||||
expectedExchangeDates.Insert(0, new(2024, 9, 30));
|
||||
expectedExchangeDates.RemoveAt(expectedExchangeDates.Count - 1);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Flush first date:
|
||||
Assert.IsTrue(timeKeeper.TryAdvanceUntilNextDataDate());
|
||||
Assert.IsTrue(exchangeDateEmitted);
|
||||
|
||||
for (var i = 1; i < tradableDates.Count; i++)
|
||||
{
|
||||
exchangeDateEmitted = false;
|
||||
Assert.IsTrue(timeKeeper.TryAdvanceUntilNextDataDate());
|
||||
|
||||
if (timeKeeper.DataTime == timeKeeper.ExchangeTime)
|
||||
{
|
||||
Assert.AreEqual(tradableDates[i], timeKeeper.DataTime);
|
||||
Assert.AreEqual(tradableDates[i], timeKeeper.ExchangeTime);
|
||||
Assert.IsTrue(exchangeDateEmitted);
|
||||
Assert.AreEqual(tradableDates[i], emittedExchangeDates[^1]);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (exchangeIsBehindData)
|
||||
{
|
||||
Assert.IsFalse(exchangeDateEmitted);
|
||||
Assert.AreEqual(tradableDates[i - 1], timeKeeper.DataTime);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.IsTrue(exchangeDateEmitted);
|
||||
Assert.AreEqual(emittedExchangeDates[^1], timeKeeper.ExchangeTime);
|
||||
}
|
||||
|
||||
// Move again to the next data date or next exchange date
|
||||
exchangeDateEmitted = false;
|
||||
Assert.IsTrue(timeKeeper.TryAdvanceUntilNextDataDate());
|
||||
|
||||
if (exchangeIsBehindData)
|
||||
{
|
||||
Assert.IsTrue(exchangeDateEmitted);
|
||||
Assert.AreEqual(emittedExchangeDates[^1], timeKeeper.ExchangeTime);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.IsFalse(exchangeDateEmitted);
|
||||
Assert.AreEqual(tradableDates[i], timeKeeper.DataTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CollectionAssert.AreEqual(expectedExchangeDates, emittedExchangeDates);
|
||||
}
|
||||
finally
|
||||
{
|
||||
timeKeeper.NewExchangeDate -= HandleNewTradableDate;
|
||||
}
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(TimeZonesTestCases))]
|
||||
public void ExchangeDatesAreEmittedByAdvancingToNextGivenExchangeTime(DateTimeZone dataTimeZone, DateTimeZone exchangeTimeZone)
|
||||
{
|
||||
using var timeKeeper = GetTimeKeeper(dataTimeZone, exchangeTimeZone, false, null, out var tradableDates, out var config, out var exchangeHours);
|
||||
|
||||
var exchangeDateEmitted = false;
|
||||
var emittedExchangeDates = new List<DateTime>();
|
||||
void HandleNewTradableDate(object sender, DateTime date)
|
||||
{
|
||||
emittedExchangeDates.Add(date);
|
||||
exchangeDateEmitted = true;
|
||||
}
|
||||
|
||||
timeKeeper.NewExchangeDate += HandleNewTradableDate;
|
||||
|
||||
var expectedExchangeDates = new List<DateTime>()
|
||||
{
|
||||
new(2024, 10, 1),
|
||||
new(2024, 10, 2),
|
||||
new(2024, 10, 3),
|
||||
new(2024, 10, 4),
|
||||
new(2024, 10, 7),
|
||||
new(2024, 10, 8),
|
||||
new(2024, 10, 9),
|
||||
new(2024, 10, 10),
|
||||
new(2024, 10, 11),
|
||||
};
|
||||
|
||||
var firstDataDate = tradableDates[0];
|
||||
var firstDataDateInExchangeTimeZone = firstDataDate.ConvertTo(dataTimeZone, exchangeTimeZone);
|
||||
var exchangeIsBehindData = firstDataDateInExchangeTimeZone < firstDataDate;
|
||||
if (exchangeIsBehindData)
|
||||
{
|
||||
expectedExchangeDates.Insert(0, new(2024, 9, 30));
|
||||
expectedExchangeDates.RemoveAt(expectedExchangeDates.Count - 1);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Flush first date:
|
||||
Assert.IsTrue(timeKeeper.TryAdvanceUntilNextDataDate());
|
||||
Assert.IsTrue(exchangeDateEmitted);
|
||||
|
||||
// Using multiple step sizes to simulate data coming in with gaps
|
||||
var steps = new Queue<TimeSpan>();
|
||||
steps.Enqueue(TimeSpan.FromMinutes(1));
|
||||
steps.Enqueue(TimeSpan.FromMinutes(5));
|
||||
steps.Enqueue(TimeSpan.FromMinutes(30));
|
||||
steps.Enqueue(TimeSpan.FromHours(1));
|
||||
|
||||
while (emittedExchangeDates.Count != expectedExchangeDates.Count)
|
||||
{
|
||||
exchangeDateEmitted = false;
|
||||
var currentExchangeTime = timeKeeper.ExchangeTime;
|
||||
var step = steps.Dequeue();
|
||||
steps.Enqueue(step);
|
||||
var nextExchangeTime = currentExchangeTime + step;
|
||||
timeKeeper.AdvanceTowardsExchangeTime(nextExchangeTime);
|
||||
|
||||
if (nextExchangeTime.Date != currentExchangeTime.Date &&
|
||||
exchangeHours.IsDateOpen(nextExchangeTime.Date, config.ExtendedMarketHours))
|
||||
{
|
||||
Assert.IsTrue(exchangeDateEmitted);
|
||||
Assert.AreEqual(emittedExchangeDates[^1], nextExchangeTime.Date);
|
||||
Assert.AreEqual(timeKeeper.ExchangeTime, nextExchangeTime.Date);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.IsFalse(exchangeDateEmitted);
|
||||
Assert.AreEqual(timeKeeper.ExchangeTime, nextExchangeTime);
|
||||
}
|
||||
}
|
||||
|
||||
CollectionAssert.AreEqual(expectedExchangeDates, emittedExchangeDates);
|
||||
}
|
||||
finally
|
||||
{
|
||||
timeKeeper.NewExchangeDate -= HandleNewTradableDate;
|
||||
}
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(TimeZonesTestCases))]
|
||||
public void DoesNotAdvancePastDelistingDateWhenAdvancingToNextDataDate(DateTimeZone dataTimeZone, DateTimeZone exchangeTimeZone)
|
||||
{
|
||||
var delistingDate = _start.AddDays(7);
|
||||
using var timeKeeper = GetTimeKeeper(dataTimeZone, exchangeTimeZone, false, delistingDate, out var _, out var _, out var _);
|
||||
|
||||
while (timeKeeper.TryAdvanceUntilNextDataDate())
|
||||
{
|
||||
Assert.LessOrEqual(timeKeeper.ExchangeTime.Date, delistingDate.AddDays(1));
|
||||
}
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(TimeZonesTestCases))]
|
||||
public void ThrowsIfNotInitialized(DateTimeZone dataTimeZone, DateTimeZone exchangeTimeZone)
|
||||
{
|
||||
using var timeKeeper = GetTimeKeeper(dataTimeZone, exchangeTimeZone, false, null, out var _, out var _, out var _);
|
||||
|
||||
Assert.Throws<InvalidOperationException>(() => timeKeeper.AdvanceTowardsExchangeTime(timeKeeper.ExchangeTime.AddHours(1)));
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(TimeZonesTestCases))]
|
||||
public void FinishesRightAwayIfThereAreNoTradableDates(DateTimeZone dataTimeZone, DateTimeZone exchangeTimeZone)
|
||||
{
|
||||
using var timeKeeper = GetNoDatesTimeKeeper(dataTimeZone, exchangeTimeZone);
|
||||
|
||||
Assert.IsFalse(timeKeeper.TryAdvanceUntilNextDataDate());
|
||||
}
|
||||
|
||||
private static DateChangeTimeKeeper GetTimeKeeper(DateTimeZone dataTimeZone,
|
||||
DateTimeZone exchangeTimeZone,
|
||||
bool exchangeAlwaysOpen,
|
||||
DateTime? delistingDate,
|
||||
out List<DateTime> tradableDates,
|
||||
out SubscriptionDataConfig config,
|
||||
out SecurityExchangeHours exchangeHours)
|
||||
{
|
||||
var symbol = Symbols.SPY;
|
||||
exchangeHours = GetMarketHours(symbol, exchangeTimeZone, exchangeAlwaysOpen);
|
||||
config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
symbol,
|
||||
Resolution.Minute,
|
||||
dataTimeZone,
|
||||
exchangeTimeZone,
|
||||
false,
|
||||
true,
|
||||
false);
|
||||
|
||||
tradableDates = Time.EachTradeableDayInTimeZone(exchangeHours,
|
||||
_start,
|
||||
_end,
|
||||
config.DataTimeZone,
|
||||
config.ExtendedMarketHours).ToList();
|
||||
return new DateChangeTimeKeeper(tradableDates, config, exchangeHours, delistingDate ?? symbol.GetDelistingDate());
|
||||
}
|
||||
|
||||
private static DateChangeTimeKeeper GetNoDatesTimeKeeper(DateTimeZone dataTimeZone, DateTimeZone exchangeTimeZone)
|
||||
{
|
||||
var symbol = Symbols.SPY;
|
||||
var exchangeHours = GetMarketHours(symbol, exchangeTimeZone, true);
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
symbol,
|
||||
Resolution.Minute,
|
||||
dataTimeZone,
|
||||
exchangeTimeZone,
|
||||
false,
|
||||
true,
|
||||
false);
|
||||
|
||||
var tradableDates = Enumerable.Empty<DateTime>().ToList();
|
||||
return new DateChangeTimeKeeper(tradableDates, config, exchangeHours, symbol.GetDelistingDate());
|
||||
}
|
||||
|
||||
private static SecurityExchangeHours GetMarketHours(Symbol symbol, DateTimeZone exchangeTimeZone, bool alwaysOpen)
|
||||
{
|
||||
SecurityExchangeHours exchangeHours;
|
||||
if (alwaysOpen)
|
||||
{
|
||||
exchangeHours = SecurityExchangeHours.AlwaysOpen(exchangeTimeZone);
|
||||
}
|
||||
else
|
||||
{
|
||||
var databaseExchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
|
||||
exchangeHours = new SecurityExchangeHours(exchangeTimeZone,
|
||||
databaseExchangeHours.Holidays,
|
||||
databaseExchangeHours.MarketHours.ToDictionary(),
|
||||
databaseExchangeHours.EarlyCloses,
|
||||
databaseExchangeHours.LateOpens);
|
||||
}
|
||||
|
||||
return exchangeHours;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using System.Threading;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Logging;
|
||||
using System.Threading.Tasks;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Data.Market;
|
||||
using System.Collections.Generic;
|
||||
using DataFeed = QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Data.Custom.IconicTypes;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class DownloaderDataProviderTests
|
||||
{
|
||||
[Test]
|
||||
public void ConcurrentDownloadsSameFile()
|
||||
{
|
||||
Log.DebuggingEnabled = true;
|
||||
var downloader = new DataDownloaderTest();
|
||||
using var dataProvider = new DataFeed.DownloaderDataProvider(downloader);
|
||||
|
||||
var date = new DateTime(2000, 3, 17);
|
||||
var dataSymbol = Symbol.Create("TEST", SecurityType.Equity, Market.USA);
|
||||
var actualPath = LeanData.GenerateZipFilePath(Globals.DataFolder, dataSymbol, date, Resolution.Daily, TickType.Trade);
|
||||
|
||||
// the symbol of the data is the same
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
downloader.Data.Add(new TradeBar(date.AddDays(i), dataSymbol, i, i, i, i, i));
|
||||
}
|
||||
File.Delete(actualPath);
|
||||
|
||||
var failures = 0L;
|
||||
using var cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10));
|
||||
|
||||
// here we request symbols which is different than the data the downloader will return & we need to store, simulating mapping behavior
|
||||
// the data will always use 'dataSymbol' and the same output path
|
||||
var endedCount = 0;
|
||||
var taskCount = 5;
|
||||
for (var i = 0; i < taskCount; i++)
|
||||
{
|
||||
var myLockId = i;
|
||||
var task = new Task(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var count = 0;
|
||||
while (count++ < 10)
|
||||
{
|
||||
var requestSymbol = Symbol.Create($"TEST{count + Math.Pow(10, myLockId)}", SecurityType.Equity, Market.USA);
|
||||
var path = LeanData.GenerateZipFilePath(Globals.DataFolder, requestSymbol, date, Resolution.Daily, TickType.Trade);
|
||||
// we will get null back because the data is stored to another path, the 'dataSymbol' path which is read bellow
|
||||
Assert.IsNull(dataProvider.Fetch(path));
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Interlocked.Increment(ref failures);
|
||||
Log.Error(exception);
|
||||
cancellationToken.Cancel();
|
||||
}
|
||||
|
||||
if(Interlocked.Increment(ref endedCount) == taskCount)
|
||||
{
|
||||
// the end
|
||||
cancellationToken.Cancel();
|
||||
}
|
||||
}, cancellationToken.Token);
|
||||
task.Start();
|
||||
}
|
||||
|
||||
cancellationToken.Token.WaitHandle.WaitOne();
|
||||
|
||||
Assert.AreEqual(0, Interlocked.Read(ref failures));
|
||||
lock (downloader.DataDownloaderGetParameters)
|
||||
{
|
||||
Assert.AreEqual(downloader.DataDownloaderGetParameters.Count, 50);
|
||||
}
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(actualPath).Single();
|
||||
|
||||
// the data was merged
|
||||
Assert.AreEqual(66, data.Value.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CustomDataRequest()
|
||||
{
|
||||
var downloader = new DataDownloaderTest();
|
||||
using var dataProvider = new DataFeed.DownloaderDataProvider(downloader);
|
||||
|
||||
var customData = Symbol.CreateBase(typeof(LinkedData), Symbols.SPY, Market.USA);
|
||||
var date = new DateTime(2023, 3, 17);
|
||||
var path = LeanData.GenerateZipFilePath(Globals.DataFolder + "fake", customData, date, Resolution.Minute, TickType.Trade);
|
||||
Assert.IsNull(dataProvider.Fetch(path));
|
||||
|
||||
Assert.AreEqual(0, downloader.DataDownloaderGetParameters.Count);
|
||||
}
|
||||
|
||||
[TestCase(Resolution.Daily, SecurityType.Equity)]
|
||||
[TestCase(Resolution.Hour, SecurityType.Equity)]
|
||||
[TestCase(Resolution.Minute, SecurityType.Equity)]
|
||||
[TestCase(Resolution.Second, SecurityType.Equity)]
|
||||
[TestCase(Resolution.Tick, SecurityType.Equity)]
|
||||
[TestCase(Resolution.Daily, SecurityType.Option)]
|
||||
[TestCase(Resolution.Hour, SecurityType.Option)]
|
||||
[TestCase(Resolution.Minute, SecurityType.Option)]
|
||||
[TestCase(Resolution.Second, SecurityType.Option)]
|
||||
[TestCase(Resolution.Tick, SecurityType.Option)]
|
||||
[TestCase(Resolution.Daily, SecurityType.IndexOption)]
|
||||
[TestCase(Resolution.Hour, SecurityType.IndexOption)]
|
||||
[TestCase(Resolution.Minute, SecurityType.IndexOption)]
|
||||
[TestCase(Resolution.Second, SecurityType.IndexOption)]
|
||||
[TestCase(Resolution.Tick, SecurityType.IndexOption)]
|
||||
[TestCase(Resolution.Daily, SecurityType.Crypto)]
|
||||
[TestCase(Resolution.Hour, SecurityType.Crypto)]
|
||||
[TestCase(Resolution.Minute, SecurityType.Crypto)]
|
||||
[TestCase(Resolution.Second, SecurityType.Crypto)]
|
||||
[TestCase(Resolution.Tick, SecurityType.Crypto)]
|
||||
[TestCase(Resolution.Daily, SecurityType.Future)]
|
||||
[TestCase(Resolution.Hour, SecurityType.Future)]
|
||||
[TestCase(Resolution.Minute, SecurityType.Future)]
|
||||
[TestCase(Resolution.Second, SecurityType.Future)]
|
||||
[TestCase(Resolution.Tick, SecurityType.Future)]
|
||||
[TestCase(Resolution.Daily, SecurityType.FutureOption)]
|
||||
[TestCase(Resolution.Hour, SecurityType.FutureOption)]
|
||||
[TestCase(Resolution.Minute, SecurityType.FutureOption)]
|
||||
[TestCase(Resolution.Second, SecurityType.FutureOption)]
|
||||
[TestCase(Resolution.Tick, SecurityType.FutureOption)]
|
||||
public void CorrectArguments(Resolution resolution, SecurityType securityType)
|
||||
{
|
||||
var downloader = new DataDownloaderTest();
|
||||
using var dataProvider = new DataFeed.DownloaderDataProvider(downloader);
|
||||
|
||||
Symbol symbol = null;
|
||||
Symbol expectedSymbol = null;
|
||||
var expectedLowResolutionStart = new DateTime(1998, 1, 2);
|
||||
var expectedLowResolutionEndUtc = DateTime.UtcNow.Date.AddDays(-1);
|
||||
if (securityType == SecurityType.Equity)
|
||||
{
|
||||
symbol = expectedSymbol = Symbols.AAPL;
|
||||
}
|
||||
else if (securityType == SecurityType.Option)
|
||||
{
|
||||
symbol = Symbols.SPY_C_192_Feb19_2016;
|
||||
expectedSymbol = symbol.Canonical;
|
||||
}
|
||||
else if (securityType == SecurityType.IndexOption)
|
||||
{
|
||||
symbol = Symbol.CreateOption(Symbols.SPX, Symbols.SPX.ID.Market, OptionStyle.European, OptionRight.Call, 38000m, new DateTime(2021, 01, 15));
|
||||
expectedSymbol = symbol.Canonical;
|
||||
}
|
||||
else if (securityType == SecurityType.Crypto)
|
||||
{
|
||||
expectedLowResolutionStart = new DateTime(2009, 1, 1);
|
||||
symbol = expectedSymbol = Symbols.BTCUSD;
|
||||
}
|
||||
else if (securityType == SecurityType.Future)
|
||||
{
|
||||
symbol = Symbols.Future_CLF19_Jan2019;
|
||||
expectedSymbol = symbol.Canonical;
|
||||
}
|
||||
else if (securityType == SecurityType.FutureOption)
|
||||
{
|
||||
symbol = Symbols.CreateFutureOptionSymbol(Symbols.Future_CLF19_Jan2019, OptionRight.Call, 10, new DateTime(2019, 1, 21));
|
||||
expectedSymbol = symbol.Canonical;
|
||||
}
|
||||
|
||||
var date = new DateTime(2023, 3, 17);
|
||||
var timezone = MarketHoursDatabase.FromDataFolder().GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
|
||||
|
||||
// For options and index option in hour or daily resolution, the whole year is downloaded
|
||||
if (resolution > Resolution.Minute && (securityType == SecurityType.Option || securityType == SecurityType.IndexOption))
|
||||
{
|
||||
var expectedStartUtc = new DateTime(date.Year, 1, 1);
|
||||
expectedLowResolutionStart = expectedStartUtc.ConvertFromUtc(timezone);
|
||||
expectedLowResolutionEndUtc = expectedStartUtc.AddYears(1);
|
||||
}
|
||||
|
||||
var path = LeanData.GenerateZipFilePath(Globals.DataFolder + "fake", symbol, date, resolution, TickType.Trade);
|
||||
Assert.IsNull(dataProvider.Fetch(path));
|
||||
|
||||
var arguments = downloader.DataDownloaderGetParameters.Single();
|
||||
|
||||
Assert.AreEqual(expectedSymbol, arguments.Symbol);
|
||||
Assert.AreEqual(resolution, arguments.Resolution);
|
||||
if (resolution < Resolution.Hour)
|
||||
{
|
||||
// 1 day
|
||||
Assert.AreEqual(date.ConvertToUtc(timezone), arguments.StartUtc);
|
||||
Assert.AreEqual(date.AddDays(1).ConvertToUtc(timezone), arguments.EndUtc);
|
||||
}
|
||||
else
|
||||
{
|
||||
// the whole history
|
||||
Assert.AreEqual(expectedLowResolutionStart.ConvertToUtc(timezone), arguments.StartUtc);
|
||||
Assert.AreEqual(expectedLowResolutionEndUtc, arguments.EndUtc);
|
||||
}
|
||||
}
|
||||
|
||||
private class DataDownloaderTest : IDataDownloader
|
||||
{
|
||||
public List<BaseData> Data { get; } = new();
|
||||
public List<DataDownloaderGetParameters > DataDownloaderGetParameters { get; } = new ();
|
||||
public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
|
||||
{
|
||||
lock(DataDownloaderGetParameters)
|
||||
{
|
||||
DataDownloaderGetParameters.Add(dataDownloaderGetParameters);
|
||||
|
||||
if (dataDownloaderGetParameters.Symbol.ID.Symbol.StartsWith("TEST", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
// let's create some more data based on the symbol, so we can assert it's merged
|
||||
var id = int.Parse(dataDownloaderGetParameters.Symbol.ID.Symbol.RemoveFromStart("TEST"));
|
||||
return Data.Select(bar =>
|
||||
{
|
||||
var result = bar.Clone();
|
||||
result.Time = bar.Time.AddDays(id);
|
||||
result.EndTime = bar.EndTime.AddDays(id);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
return Data;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using QuantConnect.Tests.Common.Securities;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class AuxiliaryDataEnumeratorTests
|
||||
{
|
||||
private SubscriptionDataConfig _config;
|
||||
private TestTradableDayNotifier _tradableDayNotifier;
|
||||
private Delisting _delistingEvent;
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
_config = SecurityTests.CreateTradeBarConfig();
|
||||
_tradableDayNotifier = new TestTradableDayNotifier();
|
||||
_tradableDayNotifier.Symbol = _config.Symbol;
|
||||
_delistingEvent = new Delisting(_config.Symbol, new DateTime(2009, 1, 1), 1, DelistingType.Delisted);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void IsSetToNullIfNoDataAlwaysReturnsTrue()
|
||||
{
|
||||
var eventProvider = new TestableEventProvider();
|
||||
var enumerator = new AuxiliaryDataEnumerator(
|
||||
_config,
|
||||
null,
|
||||
null,
|
||||
new ITradableDateEventProvider[] { eventProvider },
|
||||
_tradableDayNotifier,
|
||||
DateTime.UtcNow
|
||||
);
|
||||
|
||||
eventProvider.Data.Enqueue(_delistingEvent);
|
||||
_tradableDayNotifier.TriggerEvent();
|
||||
|
||||
Assert.Null(enumerator.Current);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.NotNull(enumerator.Current);
|
||||
Assert.AreEqual(_delistingEvent, enumerator.Current);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.Null(enumerator.Current);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.Null(enumerator.Current);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
class TestableEventProvider : ITradableDateEventProvider
|
||||
{
|
||||
public readonly Queue<BaseData> Data = new Queue<BaseData>();
|
||||
|
||||
public IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
|
||||
{
|
||||
yield return Data.Dequeue();
|
||||
}
|
||||
|
||||
public void Initialize(SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, IMapFileProvider mapFileProvider, DateTime startTime)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
class TestTradableDayNotifier : ITradableDatesNotifier
|
||||
{
|
||||
public event EventHandler<NewTradableDateEventArgs> NewTradableDate;
|
||||
public DateTime TradableDate { get; set; }
|
||||
public BaseData LastBaseData { get; set; }
|
||||
public Symbol Symbol { get; set; }
|
||||
|
||||
public void TriggerEvent()
|
||||
{
|
||||
NewTradableDate?.Invoke(this, new NewTradableDateEventArgs(TradableDate, LastBaseData, Symbol, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class BaseDataCollectionAggregatorEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void AggregatesUntilNull()
|
||||
{
|
||||
var time = new DateTime(2015, 10, 20);
|
||||
var underlying = Enumerable.Range(0, 5).Select(x => new Tick { Time = time }).ToList();
|
||||
underlying.AddRange(new Tick[] { null, null, null });
|
||||
|
||||
var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), Symbols.SPY);
|
||||
|
||||
Assert.IsTrue(aggregator.MoveNext());
|
||||
Assert.IsNotNull(aggregator.Current);
|
||||
Assert.AreEqual(5, aggregator.Current.Data.Count);
|
||||
|
||||
aggregator.Dispose();
|
||||
}
|
||||
[Test]
|
||||
public void AggregatesUntilTimeChange()
|
||||
{
|
||||
var time = new DateTime(2015, 10, 20);
|
||||
var underlying = Enumerable.Range(0, 5).Select(x => new Tick { Time = time }).ToList();
|
||||
underlying.AddRange(Enumerable.Range(0, 5).Select(x => new Tick {Time = time.AddSeconds(1)}));
|
||||
|
||||
var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), Symbols.SPY);
|
||||
|
||||
Assert.IsTrue(aggregator.MoveNext());
|
||||
Assert.IsNotNull(aggregator.Current);
|
||||
Assert.AreEqual(5, aggregator.Current.Data.Count);
|
||||
|
||||
aggregator.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using System.Collections;
|
||||
using QuantConnect.Data.Market;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class ConcatEnumeratorTests
|
||||
{
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void SkipsBasedOnEndTime(bool skipsBasedOnEndTime)
|
||||
{
|
||||
var time = new DateTime(2020, 1, 1);
|
||||
var enumerator1 = new List<BaseData> { new Tick(time, Symbols.SPY, 10, 10) }.GetEnumerator();
|
||||
var enumerator2 = new List<BaseData>
|
||||
{
|
||||
new Tick(time.AddSeconds(-1), Symbols.SPY, 20, 20), //should be skipped because end time is before previous tick
|
||||
new Tick(time.AddSeconds(1), Symbols.SPY, 30 , 30)
|
||||
}.GetEnumerator();
|
||||
|
||||
var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, enumerator2);
|
||||
|
||||
Assert.IsTrue(concat.MoveNext());
|
||||
Assert.AreEqual(10, (concat.Current as Tick).AskPrice);
|
||||
|
||||
if (!skipsBasedOnEndTime)
|
||||
{
|
||||
Assert.IsTrue(concat.MoveNext());
|
||||
Assert.AreEqual(20, (concat.Current as Tick).AskPrice);
|
||||
}
|
||||
|
||||
Assert.IsTrue(concat.MoveNext());
|
||||
Assert.AreEqual(30, (concat.Current as Tick).AskPrice);
|
||||
|
||||
Assert.IsFalse(concat.MoveNext());
|
||||
Assert.IsNull(concat.Current);
|
||||
|
||||
concat.Dispose();
|
||||
}
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void EmptyNullEnumerators(bool skipsBasedOnEndTime)
|
||||
{
|
||||
var time = new DateTime(2020, 1, 1);
|
||||
// empty enumerators
|
||||
var enumerator1 = new List<BaseData>().GetEnumerator();
|
||||
var enumerator2 = new List<BaseData>().GetEnumerator();
|
||||
|
||||
var enumerator3 = new List<BaseData>
|
||||
{
|
||||
new Tick(time, Symbols.SPY, 10, 10),
|
||||
new Tick(time.AddSeconds(-1), Symbols.SPY, 20, 20),
|
||||
new Tick(time.AddSeconds(1), Symbols.SPY, 30 , 30)
|
||||
}.GetEnumerator();
|
||||
|
||||
var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, null, enumerator2, enumerator3);
|
||||
|
||||
Assert.IsTrue(concat.MoveNext());
|
||||
Assert.AreEqual(10, (concat.Current as Tick).AskPrice);
|
||||
|
||||
Assert.IsTrue(concat.MoveNext());
|
||||
Assert.AreEqual(20, (concat.Current as Tick).AskPrice);
|
||||
|
||||
Assert.IsTrue(concat.MoveNext());
|
||||
Assert.AreEqual(30, (concat.Current as Tick).AskPrice);
|
||||
|
||||
Assert.IsFalse(concat.MoveNext());
|
||||
Assert.IsNull(concat.Current);
|
||||
|
||||
concat.Dispose();
|
||||
}
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void DropsEnumeratorsReturningNullAndTrue(bool skipsBasedOnEndTime)
|
||||
{
|
||||
var enumerator1 = new TestEnumerator();
|
||||
var enumerator2 = new TestEnumerator();
|
||||
|
||||
var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, null, enumerator2);
|
||||
|
||||
Assert.IsTrue(concat.MoveNext());
|
||||
|
||||
Assert.IsNull(concat.Current);
|
||||
Assert.IsTrue(enumerator1.Disposed);
|
||||
Assert.IsFalse(enumerator2.Disposed);
|
||||
Assert.AreEqual(1, enumerator2.MoveNextCount);
|
||||
|
||||
Assert.IsTrue(concat.MoveNext());
|
||||
|
||||
// we assert it just keeps the last enumerator and drops the rest
|
||||
Assert.IsTrue(enumerator1.Disposed);
|
||||
Assert.IsFalse(enumerator2.Disposed);
|
||||
Assert.IsNull(concat.Current);
|
||||
Assert.AreEqual(2, enumerator2.MoveNextCount);
|
||||
|
||||
concat.Dispose();
|
||||
}
|
||||
|
||||
private class TestEnumerator : IEnumerator<BaseData>
|
||||
{
|
||||
public bool Disposed { get; private set; }
|
||||
public int MoveNextCount { get; private set; }
|
||||
public BaseData Current => null;
|
||||
|
||||
object IEnumerator.Current => null;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Disposed = true;
|
||||
}
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
MoveNextCount++;
|
||||
return true;
|
||||
}
|
||||
|
||||
public void Reset()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class DelistingEnumeratorTests
|
||||
{
|
||||
private SubscriptionDataConfig _config;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
var symbol = Symbol.CreateFuture("ASD", Market.USA, new DateTime(2018, 01, 01));
|
||||
|
||||
_config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
symbol,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmitsBothEventsIfDateIsPastDelisted()
|
||||
{
|
||||
var eventProvider = new DelistingEventProvider();
|
||||
eventProvider.Initialize(_config,
|
||||
null,
|
||||
null,
|
||||
DateTime.UtcNow);
|
||||
|
||||
var enumerator = eventProvider.GetEvents(
|
||||
new NewTradableDateEventArgs(
|
||||
DateTime.UtcNow,
|
||||
new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
|
||||
_config.Symbol,
|
||||
null
|
||||
)
|
||||
).GetEnumerator();
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current as Delisting);
|
||||
Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
|
||||
Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
|
||||
Assert.AreEqual(_config.Symbol.ID.Date, (enumerator.Current as Delisting).Time.Date);
|
||||
Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current as Delisting);
|
||||
Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
|
||||
Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
|
||||
Assert.AreEqual(_config.Symbol.ID.Date.AddDays(1), (enumerator.Current as Delisting).Time.Date);
|
||||
Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmitsWarningAsOffDelistingDate()
|
||||
{
|
||||
var eventProvider = new DelistingEventProvider();
|
||||
eventProvider.Initialize(_config,
|
||||
null,
|
||||
null,
|
||||
DateTime.UtcNow);
|
||||
|
||||
// should NOT emit
|
||||
var enumerator = eventProvider.GetEvents(
|
||||
new NewTradableDateEventArgs(
|
||||
_config.Symbol.ID.Date.Subtract(TimeSpan.FromMinutes(1)),
|
||||
new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
|
||||
_config.Symbol,
|
||||
null
|
||||
)
|
||||
).GetEnumerator();
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
|
||||
// should emit
|
||||
enumerator = eventProvider.GetEvents(
|
||||
new NewTradableDateEventArgs(
|
||||
_config.Symbol.ID.Date,
|
||||
new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
|
||||
_config.Symbol,
|
||||
null
|
||||
)
|
||||
).GetEnumerator();
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current as Delisting);
|
||||
Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
|
||||
Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
|
||||
Assert.AreEqual(_config.Symbol.ID.Date, (enumerator.Current as Delisting).Time.Date);
|
||||
Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmitsDelistedAfterDelistingDate()
|
||||
{
|
||||
var eventProvider = new DelistingEventProvider();
|
||||
eventProvider.Initialize(_config,
|
||||
null,
|
||||
null,
|
||||
DateTime.UtcNow);
|
||||
|
||||
// should emit warning
|
||||
var enumerator = eventProvider.GetEvents(
|
||||
new NewTradableDateEventArgs(
|
||||
_config.Symbol.ID.Date,
|
||||
new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
|
||||
_config.Symbol,
|
||||
null
|
||||
)
|
||||
).GetEnumerator();
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current as Delisting);
|
||||
Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
|
||||
|
||||
// should NOT emit if not AFTER delisting date
|
||||
enumerator = eventProvider.GetEvents(
|
||||
new NewTradableDateEventArgs(
|
||||
_config.Symbol.ID.Date,
|
||||
new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
|
||||
_config.Symbol,
|
||||
null
|
||||
)
|
||||
).GetEnumerator();
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
|
||||
// should emit AFTER delisting date
|
||||
enumerator = eventProvider.GetEvents(
|
||||
new NewTradableDateEventArgs(
|
||||
_config.Symbol.ID.Date.AddMinutes(1),
|
||||
new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
|
||||
_config.Symbol,
|
||||
null
|
||||
)
|
||||
).GetEnumerator();
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current as Delisting);
|
||||
Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
|
||||
Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
|
||||
Assert.AreEqual(_config.Symbol.ID.Date.AddDays(1), (enumerator.Current as Delisting).Time.Date);
|
||||
Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmitsDelistedWarningOnNonTradableDay()
|
||||
{
|
||||
// Unit test to simulate and reproduce #5545
|
||||
|
||||
// Give us two tradable days before and after expiration
|
||||
var tradableDays = new List<DateTime> { new DateTime(2021, 01, 01), new DateTime(2021, 01, 04) };
|
||||
|
||||
// Set expiration as 1/2/21 a Saturday, not included in our tradble days
|
||||
var expiration = new DateTime(2021, 01, 02);
|
||||
var symbol = Symbol.CreateFuture("ASD", Market.USA, expiration);
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
symbol,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
|
||||
var eventProvider = new DelistingEventProvider();
|
||||
eventProvider.Initialize(config,
|
||||
null,
|
||||
null,
|
||||
DateTime.UtcNow);
|
||||
|
||||
var tradableDateEvents = tradableDays.Select(day => new NewTradableDateEventArgs(
|
||||
day,
|
||||
new Tick(day, config.Symbol, 10, 5),
|
||||
config.Symbol,
|
||||
null
|
||||
)).GetEnumerator();
|
||||
|
||||
// Pass in the day before expiration should be nothing
|
||||
tradableDateEvents.MoveNext();
|
||||
var enumerator = eventProvider.GetEvents(tradableDateEvents.Current).GetEnumerator();
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
|
||||
// Pass in the following monday, should be both but warning first and still scheduled for saturday
|
||||
tradableDateEvents.MoveNext();
|
||||
enumerator = eventProvider.GetEvents(tradableDateEvents.Current).GetEnumerator();
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current as Delisting);
|
||||
Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
|
||||
Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
|
||||
Assert.AreEqual(config.Symbol.ID.Date, (enumerator.Current as Delisting).Time.Date);
|
||||
Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current as Delisting);
|
||||
Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
|
||||
Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
|
||||
Assert.AreEqual(config.Symbol.ID.Date.AddDays(1), (enumerator.Current as Delisting).Time.Date);
|
||||
Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
|
||||
tradableDateEvents.Dispose();
|
||||
enumerator.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using System.Globalization;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class DividendEventProviderTests
|
||||
{
|
||||
// From https://www.nasdaq.com/market-activity/stocks/aapl/dividend-history
|
||||
[TestCase("20121106", 2.65)]
|
||||
[TestCase("20130206", 2.65)]
|
||||
[TestCase("20130508", 3.05)]
|
||||
[TestCase("20130807", 3.05)]
|
||||
[TestCase("20131105", 3.05)]
|
||||
[TestCase("20140205", 3.05)]
|
||||
[TestCase("20140507", 3.29)]
|
||||
[TestCase("20140806", 0.47)]
|
||||
[TestCase("20141105", 0.47)]
|
||||
[TestCase("20150204", 0.47)]
|
||||
[TestCase("20150506", 0.52)]
|
||||
[TestCase("20150805", 0.52)]
|
||||
[TestCase("20151104", 0.52)]
|
||||
[TestCase("20160203", 0.52)]
|
||||
[TestCase("20160504", 0.57)]
|
||||
[TestCase("20160803", 0.57)]
|
||||
[TestCase("20161102", 0.57)]
|
||||
[TestCase("20170208", 0.57)]
|
||||
[TestCase("20170510", 0.63)]
|
||||
[TestCase("20170809", 0.63)]
|
||||
[TestCase("20171109", 0.63)]
|
||||
[TestCase("20180208", 0.63)]
|
||||
[TestCase("20180510", 0.73)]
|
||||
public void DividendsDistribution(string exDividendDateStr, decimal expectedDistribution)
|
||||
{
|
||||
var dividendProvider = new DividendEventProvider();
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork,
|
||||
false, false, false);
|
||||
|
||||
var start = new DateTime(1998, 01, 02);
|
||||
dividendProvider.Initialize(config, TestGlobals.FactorFileProvider, TestGlobals.MapFileProvider, start);
|
||||
|
||||
var exDividendDate = DateTime.ParseExact(exDividendDateStr, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
|
||||
|
||||
var events = dividendProvider
|
||||
.GetEvents(new NewTradableDateEventArgs(exDividendDate, null, Symbols.AAPL, null))
|
||||
.ToList();
|
||||
// ex dividend date does not emit anything
|
||||
Assert.AreEqual(0, events.Count);
|
||||
|
||||
events = dividendProvider
|
||||
.GetEvents(new NewTradableDateEventArgs(exDividendDate.AddDays(1), null, Symbols.AAPL, null))
|
||||
.ToList();
|
||||
|
||||
Assert.AreEqual(1, events.Count);
|
||||
var dividend = events[0] as Dividend;
|
||||
Assert.IsNotNull(dividend);
|
||||
|
||||
Assert.AreEqual(expectedDistribution, dividend.Distribution);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThrowsWhenEmptyReferencePrice()
|
||||
{
|
||||
var dividendProvider = new DividendEventProvider();
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork,
|
||||
false, false, false);
|
||||
var start = new DateTime(1998, 01, 02);
|
||||
var row1 = new DateTime(2000, 01, 02);
|
||||
var row2 = new DateTime(2001, 01, 02);
|
||||
var row3 = new DateTime(2002, 01, 02);
|
||||
|
||||
var factorFileProvider = new TestFactorFileProvider
|
||||
{
|
||||
FactorFile = new CorporateFactorProvider("AAPL", new []
|
||||
{
|
||||
new CorporateFactorRow(row1, 0.693m, 1),
|
||||
new CorporateFactorRow(row2, 0.77m, 1),
|
||||
new CorporateFactorRow(row3, 0.85555m, 1)
|
||||
}, start)
|
||||
};
|
||||
|
||||
dividendProvider.Initialize(config, factorFileProvider, TestGlobals.MapFileProvider, start);
|
||||
|
||||
foreach (var row in factorFileProvider.FactorFile.Take(1))
|
||||
{
|
||||
var lastRawPrice = 100;
|
||||
var events = dividendProvider
|
||||
.GetEvents(new NewTradableDateEventArgs(row.Date, null, Symbols.AAPL, lastRawPrice))
|
||||
.ToList();
|
||||
// ex dividend date does not emit anything
|
||||
Assert.AreEqual(0, events.Count);
|
||||
|
||||
Assert.Throws<InvalidOperationException>(() =>
|
||||
{
|
||||
dividendProvider
|
||||
.GetEvents(new NewTradableDateEventArgs(row.Date.AddDays(1), null, Symbols.AAPL, lastRawPrice))
|
||||
.ToList();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private class TestFactorFileProvider : IFactorFileProvider
|
||||
{
|
||||
public CorporateFactorProvider FactorFile { get; set; }
|
||||
public void Initialize(IMapFileProvider mapFileProvider, IDataProvider dataProvider)
|
||||
{
|
||||
}
|
||||
|
||||
public IFactorProvider Get(Symbol symbol)
|
||||
{
|
||||
return FactorFile;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class EnqueableEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void PassesTicksStraightThrough()
|
||||
{
|
||||
var enumerator = new EnqueueableEnumerator<Tick>();
|
||||
|
||||
// add some ticks
|
||||
var currentTime = new DateTime(2015, 10, 08);
|
||||
|
||||
// returns true even if no data present until stop is called
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) {Quantity = 10};
|
||||
enumerator.Enqueue(tick1);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(tick1, enumerator.Current);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick2 = new Tick(currentTime, Symbols.SPY, 199.56m, 199.21m, 200.02m) {Quantity = 5};
|
||||
enumerator.Enqueue(tick2);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(tick2, enumerator.Current);
|
||||
|
||||
enumerator.Stop();
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RecordsInternalQueueCount()
|
||||
{
|
||||
var enumerator = new EnqueueableEnumerator<Tick>();
|
||||
|
||||
var currentTime = new DateTime(2015, 12, 01);
|
||||
var tick = new Tick(currentTime, Symbols.SPY, 100, 101);
|
||||
enumerator.Enqueue(tick);
|
||||
Assert.AreEqual(1, enumerator.Count);
|
||||
|
||||
tick = new Tick(currentTime, Symbols.SPY, 100, 101);
|
||||
enumerator.Enqueue(tick);
|
||||
Assert.AreEqual(2, enumerator.Count);
|
||||
|
||||
enumerator.MoveNext();
|
||||
Assert.AreEqual(1, enumerator.Count);
|
||||
|
||||
enumerator.MoveNext();
|
||||
Assert.AreEqual(0, enumerator.Count);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test, Category("TravisExclude")]
|
||||
public void MoveNextBlocks()
|
||||
{
|
||||
using var finished = new ManualResetEvent(false);
|
||||
var enumerator = new EnqueueableEnumerator<Tick>(true);
|
||||
|
||||
// producer
|
||||
int count = 0;
|
||||
Task.Run(() =>
|
||||
{
|
||||
while (!finished.WaitOne(TimeSpan.FromMilliseconds(50)))
|
||||
{
|
||||
enumerator.Enqueue(new Tick(DateTime.Now, Symbols.SPY, 100, 101));
|
||||
count++;
|
||||
|
||||
// 5 data points is plenty
|
||||
if (count > 5)
|
||||
{
|
||||
finished.Set();
|
||||
enumerator.Stop();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// consumer
|
||||
int dequeuedCount = 0;
|
||||
bool encounteredError = false;
|
||||
using var consumerTaskFinished = new ManualResetEvent(false);
|
||||
Task.Run(() =>
|
||||
{
|
||||
while (enumerator.MoveNext())
|
||||
{
|
||||
dequeuedCount++;
|
||||
if (enumerator.Current == null)
|
||||
{
|
||||
encounteredError = true;
|
||||
}
|
||||
}
|
||||
consumerTaskFinished.Set();
|
||||
});
|
||||
|
||||
finished.WaitOne(Timeout.Infinite);
|
||||
consumerTaskFinished.WaitOne(Timeout.Infinite);
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsFalse(encounteredError);
|
||||
Assert.AreEqual(count, dequeuedCount);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Brokerages;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Fundamental;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Securities;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
|
||||
{
|
||||
[TestFixture]
|
||||
public class BaseDataCollectionSubscriptionEnumeratorFactoryTests
|
||||
{
|
||||
// This test reports higher memory usage when ran with Travis, so we exclude it for now
|
||||
[Test, Category("TravisExclude")]
|
||||
public void DoesNotLeakMemory()
|
||||
{
|
||||
var symbolFactory = new FundamentalUniverse();
|
||||
var symbol = symbolFactory.UniverseSymbol();
|
||||
var config = new SubscriptionDataConfig(typeof(FundamentalUniverse), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
|
||||
var security = new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
|
||||
config,
|
||||
new Cash(Currencies.USD, 0, 1),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
|
||||
var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
|
||||
var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
|
||||
using var universe = new CoarseFundamentalUniverse(universeSettings, x => new List<Symbol>{ Symbols.AAPL });
|
||||
|
||||
var factory = new BaseDataCollectionSubscriptionEnumeratorFactory(null);
|
||||
|
||||
GC.Collect();
|
||||
var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;
|
||||
|
||||
var date = new DateTime(2014, 3, 25);
|
||||
|
||||
const int iterations = 1000;
|
||||
for (var i = 0; i < iterations; i++)
|
||||
{
|
||||
var request = new SubscriptionRequest(true, universe, security, config, date, date);
|
||||
using (var enumerator = factory.CreateEnumerator(request, TestGlobals.DataProvider))
|
||||
{
|
||||
enumerator.MoveNext();
|
||||
}
|
||||
}
|
||||
|
||||
GC.Collect();
|
||||
var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed;
|
||||
|
||||
Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB");
|
||||
|
||||
Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReturnsExpectedTimestamps()
|
||||
{
|
||||
var symbolFactory = new FundamentalUniverse();
|
||||
var symbol = symbolFactory.UniverseSymbol();
|
||||
var config = new SubscriptionDataConfig(typeof(FundamentalUniverse), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
|
||||
var security = new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
|
||||
config,
|
||||
new Cash(Currencies.USD, 0, 1),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
|
||||
var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
|
||||
var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
|
||||
using var universe = new CoarseFundamentalUniverse(universeSettings, x => new List<Symbol> { Symbols.AAPL });
|
||||
|
||||
var factory = new BaseDataCollectionSubscriptionEnumeratorFactory(null);
|
||||
|
||||
var dateStart = new DateTime(2014, 3, 26);
|
||||
var dateEnd = new DateTime(2014, 3, 27);
|
||||
var days = (dateEnd - dateStart).Days + 1;
|
||||
|
||||
var request = new SubscriptionRequest(true, universe, security, config, dateStart, dateEnd);
|
||||
|
||||
using (var enumerator = factory.CreateEnumerator(request, TestGlobals.DataProvider))
|
||||
{
|
||||
for (var i = 0; i < days; i++)
|
||||
{
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
|
||||
var current = enumerator.Current as BaseDataCollection;
|
||||
Assert.IsNotNull(current);
|
||||
Assert.AreEqual(dateStart.AddDays(i), current.Time);
|
||||
Assert.AreEqual(dateStart.AddDays(i), current.EndTime);
|
||||
Assert.AreEqual(dateStart.AddDays(i - 1), current.Data[0].Time);
|
||||
Assert.AreEqual(dateStart.AddDays(i), current.Data[0].EndTime);
|
||||
}
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
;
|
||||
+613
@@ -0,0 +1,613 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Securities.Equity;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
|
||||
{
|
||||
[TestFixture]
|
||||
public class LiveCustomDataSubscriptionEnumeratorFactoryTests
|
||||
{
|
||||
[TestFixture]
|
||||
public class WhenCreatingEnumeratorForRestData
|
||||
{
|
||||
private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
|
||||
private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
|
||||
|
||||
private ManualTimeProvider _timeProvider;
|
||||
private IEnumerator<BaseData> _enumerator;
|
||||
private Mock<ISubscriptionDataSourceReader> _dataSourceReader;
|
||||
|
||||
[SetUp]
|
||||
public void Given()
|
||||
{
|
||||
_timeProvider = new ManualTimeProvider(_referenceUtc);
|
||||
|
||||
_dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
|
||||
_dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
|
||||
sds.Source == "rest.source" &&
|
||||
sds.TransportMedium == SubscriptionTransportMedium.Rest &&
|
||||
sds.Format == FileFormat.Csv))
|
||||
)
|
||||
.Returns(Enumerable.Range(0, 100)
|
||||
.Select(i => new RestData
|
||||
{
|
||||
EndTime = _referenceLocal.AddSeconds(i)
|
||||
}))
|
||||
.Verifiable();
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(RestData), Symbols.SPY, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
|
||||
var request = GetSubscriptionRequest(config, _referenceUtc.AddSeconds(-1), _referenceUtc.AddDays(1));
|
||||
|
||||
var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _dataSourceReader.Object);
|
||||
_enumerator = factory.CreateEnumerator(request, null);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_enumerator?.DisposeSafely();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void YieldsDataEachSecondAsTimePasses()
|
||||
{
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
|
||||
_timeProvider.AdvanceSeconds(1);
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal.AddSeconds(1), _enumerator.Current.EndTime);
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
|
||||
VerifyGetSourceInvocationCount(_dataSourceReader, 1, "rest.source", SubscriptionTransportMedium.Rest, FileFormat.Csv);
|
||||
}
|
||||
}
|
||||
|
||||
[TestFixture]
|
||||
public class WhenCreatingEnumeratorForRestCollectionData
|
||||
{
|
||||
private const int DataPerTimeStep = 3;
|
||||
private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
|
||||
private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
|
||||
|
||||
private ManualTimeProvider _timeProvider;
|
||||
private IEnumerator<BaseData> _enumerator;
|
||||
private Mock<ISubscriptionDataSourceReader> _dataSourceReader;
|
||||
|
||||
[SetUp]
|
||||
public void Given()
|
||||
{
|
||||
_timeProvider = new ManualTimeProvider(_referenceUtc);
|
||||
|
||||
_dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
|
||||
_dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
|
||||
sds.Source == "rest.collection.source" &&
|
||||
sds.TransportMedium == SubscriptionTransportMedium.Rest &&
|
||||
sds.Format == FileFormat.UnfoldingCollection))
|
||||
)
|
||||
.Returns(Enumerable.Range(0, 100)
|
||||
.Select(i => new BaseDataCollection(_referenceLocal.AddSeconds(i), Symbols.SPY, Enumerable.Range(0, DataPerTimeStep)
|
||||
.Select(_ => new RestCollectionData {EndTime = _referenceLocal.AddSeconds(i)})))
|
||||
)
|
||||
.Verifiable();
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(RestCollectionData), Symbols.SPY, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
|
||||
var request = GetSubscriptionRequest(config, _referenceUtc.AddSeconds(-4), _referenceUtc.AddDays(1));
|
||||
|
||||
var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _dataSourceReader.Object);
|
||||
_enumerator = factory.CreateEnumerator(request, null);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_enumerator?.DisposeSafely();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void YieldsGroupOfDataEachSecond()
|
||||
{
|
||||
for (int i = 0; i < DataPerTimeStep; i++)
|
||||
{
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current, $"Index {i} is null.");
|
||||
Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
|
||||
}
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
|
||||
_timeProvider.AdvanceSeconds(1);
|
||||
|
||||
for (int i = 0; i < DataPerTimeStep; i++)
|
||||
{
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal.AddSeconds(1), _enumerator.Current.EndTime);
|
||||
}
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
|
||||
VerifyGetSourceInvocationCount(_dataSourceReader, 1, "rest.collection.source", SubscriptionTransportMedium.Rest, FileFormat.UnfoldingCollection);
|
||||
}
|
||||
}
|
||||
|
||||
[TestFixture]
|
||||
public class WhenCreatingEnumeratorForRemoteCollectionData
|
||||
{
|
||||
private const int DataPerTimeStep = 3;
|
||||
private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
|
||||
private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
|
||||
|
||||
private ManualTimeProvider _timeProvider;
|
||||
private IEnumerator<BaseData> _enumerator;
|
||||
|
||||
[SetUp]
|
||||
public void Given()
|
||||
{
|
||||
_timeProvider = new ManualTimeProvider(_referenceUtc);
|
||||
var dataSourceReader = new TestISubscriptionDataSourceReader
|
||||
{
|
||||
TimeProvider = _timeProvider
|
||||
};
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(RemoteCollectionData), Symbols.SPY, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
|
||||
var request = GetSubscriptionRequest(config, _referenceUtc.AddSeconds(-4), _referenceUtc.AddDays(1));
|
||||
|
||||
var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, dataSourceReader);
|
||||
_enumerator = factory.CreateEnumerator(request, null);
|
||||
}
|
||||
|
||||
private class TestISubscriptionDataSourceReader : ISubscriptionDataSourceReader
|
||||
{
|
||||
public ManualTimeProvider TimeProvider;
|
||||
public event EventHandler<InvalidSourceEventArgs> InvalidSource;
|
||||
|
||||
public IEnumerable<BaseData> Read(SubscriptionDataSource source)
|
||||
{
|
||||
var currentLocalTime = TimeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
|
||||
var data = Enumerable.Range(0, DataPerTimeStep).Select(_ => new RemoteCollectionData { EndTime = currentLocalTime });
|
||||
|
||||
// let's add some old data which should be ignored
|
||||
data = data.Concat(Enumerable.Range(0, DataPerTimeStep).Select(_ => new RemoteCollectionData { EndTime = currentLocalTime.AddSeconds(-1) }));
|
||||
return new BaseDataCollection(currentLocalTime, Symbols.SPY, data);
|
||||
}
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_enumerator?.DisposeSafely();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void YieldsGroupOfDataEachSecond()
|
||||
{
|
||||
for (int i = 0; i < DataPerTimeStep; i++)
|
||||
{
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current, $"Index {i} is null.");
|
||||
Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
|
||||
}
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
|
||||
_timeProvider.AdvanceSeconds(1);
|
||||
|
||||
for (int i = 0; i < DataPerTimeStep; i++)
|
||||
{
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal.AddSeconds(1), _enumerator.Current.EndTime);
|
||||
}
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
}
|
||||
}
|
||||
|
||||
[TestFixture]
|
||||
public class WhenCreatingEnumeratorForSecondRemoteFileData
|
||||
{
|
||||
private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
|
||||
private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
|
||||
|
||||
private ManualTimeProvider _timeProvider;
|
||||
private IEnumerator<BaseData> _enumerator;
|
||||
private Mock<ISubscriptionDataSourceReader> _dataSourceReader;
|
||||
|
||||
[SetUp]
|
||||
public void Given()
|
||||
{
|
||||
_timeProvider = new ManualTimeProvider(_referenceUtc);
|
||||
|
||||
_dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
|
||||
_dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
|
||||
sds.Source == "remote.file.source" &&
|
||||
sds.TransportMedium == SubscriptionTransportMedium.RemoteFile &&
|
||||
sds.Format == FileFormat.Csv))
|
||||
)
|
||||
.Returns(Enumerable.Range(0, 100)
|
||||
.Select(i => new RemoteFileData
|
||||
{
|
||||
// include past data
|
||||
EndTime = _referenceLocal.AddSeconds(i - 95)
|
||||
}))
|
||||
.Verifiable();
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(RemoteFileData), Symbols.SPY, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
|
||||
var request = GetSubscriptionRequest(config, _referenceUtc.AddSeconds(-6), _referenceUtc.AddDays(1));
|
||||
|
||||
var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _dataSourceReader.Object);
|
||||
_enumerator = factory.CreateEnumerator(request, null);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_enumerator?.DisposeSafely();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void YieldsDataEachSecondAsTimePasses()
|
||||
{
|
||||
// most recent 5 seconds of data
|
||||
for (int i = 5; i > 0; i--)
|
||||
{
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal.AddSeconds(-i), _enumerator.Current.EndTime);
|
||||
}
|
||||
|
||||
// first data point
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
|
||||
|
||||
_timeProvider.AdvanceSeconds(1);
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal.AddSeconds(1), _enumerator.Current.EndTime);
|
||||
|
||||
VerifyGetSourceInvocationCount(_dataSourceReader, 1, "remote.file.source", SubscriptionTransportMedium.RemoteFile, FileFormat.Csv);
|
||||
}
|
||||
}
|
||||
|
||||
[TestFixture]
|
||||
public class WhenCreatingEnumeratorForDailyRemoteFileData
|
||||
{
|
||||
private int _dataPointsAfterReference = 1;
|
||||
private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
|
||||
private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
|
||||
|
||||
private ManualTimeProvider _timeProvider;
|
||||
private IEnumerator<BaseData> _enumerator;
|
||||
private Mock<ISubscriptionDataSourceReader> _dataSourceReader;
|
||||
|
||||
[SetUp]
|
||||
public void Given()
|
||||
{
|
||||
_timeProvider = new ManualTimeProvider(_referenceUtc);
|
||||
|
||||
_dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
|
||||
_dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
|
||||
sds.Source == "remote.file.source" &&
|
||||
sds.TransportMedium == SubscriptionTransportMedium.RemoteFile &&
|
||||
sds.Format == FileFormat.Csv))
|
||||
)
|
||||
.Returns(() => Enumerable.Range(0, 100)
|
||||
.Select(i => new RemoteFileData
|
||||
{
|
||||
// include past data
|
||||
EndTime = _referenceLocal.Add(TimeSpan.FromDays(i - (100 - _dataPointsAfterReference - 1)))
|
||||
}))
|
||||
.Verifiable();
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(RemoteFileData), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
|
||||
var request = GetSubscriptionRequest(config, _referenceUtc.AddDays(-2), _referenceUtc.AddDays(1));
|
||||
|
||||
var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _dataSourceReader.Object);
|
||||
_enumerator = factory.CreateEnumerator(request, null);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_enumerator?.DisposeSafely();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void YieldsDataEachDayAsTimePasses()
|
||||
{
|
||||
// previous point is exactly one resolution step behind, so it emits
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal.AddDays(-1), _enumerator.Current.EndTime);
|
||||
VerifyGetSourceInvocation(1);
|
||||
|
||||
// yields the data for the current time
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
|
||||
VerifyGetSourceInvocation(0);
|
||||
|
||||
_timeProvider.Advance(Time.OneDay);
|
||||
|
||||
// now we can yield the next data point as it has passed frontier time
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal.AddDays(1), _enumerator.Current.EndTime);
|
||||
VerifyGetSourceInvocation(0);
|
||||
|
||||
// this call exhaused the enumerator stack and yields a null result
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
VerifyGetSourceInvocation(0);
|
||||
|
||||
// this call refrshes the enumerator stack but finds no data ahead of the frontier
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
VerifyGetSourceInvocation(1);
|
||||
|
||||
_timeProvider.Advance(TimeSpan.FromMinutes(30));
|
||||
|
||||
// time advances 30 minutes so we'll try to refresh again
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
VerifyGetSourceInvocation(1);
|
||||
|
||||
_timeProvider.Advance(Time.OneDay);
|
||||
|
||||
// now to the next day, we'll try again and get data
|
||||
_dataPointsAfterReference++;
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal.AddDays(2), _enumerator.Current.EndTime);
|
||||
VerifyGetSourceInvocation(1);
|
||||
|
||||
_timeProvider.Advance(TimeSpan.FromHours(1));
|
||||
|
||||
// out of data
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
VerifyGetSourceInvocation(0);
|
||||
|
||||
_timeProvider.Advance(TimeSpan.FromHours(1));
|
||||
|
||||
// time advanced so we'll try to refresh the souce again, but exhaust the stack because no data
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
VerifyGetSourceInvocation(1);
|
||||
|
||||
// move forward to next whole day, midnight
|
||||
_timeProvider.Advance(Time.OneDay.Subtract(TimeSpan.FromHours(2.5)));
|
||||
|
||||
// the day elapsed but there's still no data available
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
VerifyGetSourceInvocation(1);
|
||||
|
||||
// this is rate limited by the 30 minute guard for daily data
|
||||
_timeProvider.Advance(TimeSpan.FromMinutes(29));
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
VerifyGetSourceInvocation(0);
|
||||
|
||||
// another 30 minutes elapsed and now there's data available
|
||||
_dataPointsAfterReference++;
|
||||
_timeProvider.Advance(TimeSpan.FromMinutes(1));
|
||||
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNotNull(_enumerator.Current);
|
||||
Assert.AreEqual(_referenceLocal.AddDays(3), _enumerator.Current.EndTime);
|
||||
VerifyGetSourceInvocation(1);
|
||||
|
||||
// exhausted the stack
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
VerifyGetSourceInvocation(0);
|
||||
|
||||
// rate limited
|
||||
Assert.IsTrue(_enumerator.MoveNext());
|
||||
Assert.IsNull(_enumerator.Current);
|
||||
VerifyGetSourceInvocation(0);
|
||||
}
|
||||
|
||||
private int _runningCount;
|
||||
private void VerifyGetSourceInvocation(int count)
|
||||
{
|
||||
_runningCount += count;
|
||||
VerifyGetSourceInvocationCount(_dataSourceReader, _runningCount, "remote.file.source", SubscriptionTransportMedium.RemoteFile, FileFormat.Csv);
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase(10)]
|
||||
[TestCase(60)]
|
||||
[TestCase(0)]
|
||||
public void AllowsSpecifyingIntervalCheck(int intervalCheck)
|
||||
{
|
||||
var referenceLocal = new DateTime(2017, 10, 12);
|
||||
var referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
|
||||
|
||||
var timeProvider = new ManualTimeProvider(referenceUtc);
|
||||
|
||||
var callCount = 0;
|
||||
var dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
|
||||
dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
|
||||
sds.Source == "local.file.source" &&
|
||||
sds.TransportMedium == SubscriptionTransportMedium.LocalFile &&
|
||||
sds.Format == FileFormat.Csv))
|
||||
)
|
||||
.Returns(() => new []{ new LocalFileData { EndTime = referenceLocal.AddSeconds(++callCount) } })
|
||||
.Verifiable();
|
||||
|
||||
var config = new SubscriptionDataConfig(typeof(LocalFileData), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
|
||||
var request = GetSubscriptionRequest(config, referenceUtc.AddSeconds(-1), referenceUtc.AddDays(1));
|
||||
|
||||
var intervalCalls = intervalCheck == 0 ? (TimeSpan?) null : TimeSpan.FromMinutes(intervalCheck);
|
||||
|
||||
var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(timeProvider, dataSourceReader.Object, intervalCalls);
|
||||
var enumerator = factory.CreateEnumerator(request, null);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
Assert.AreEqual(referenceLocal.AddSeconds(callCount), enumerator.Current.EndTime);
|
||||
|
||||
VerifyGetSourceInvocationCount(dataSourceReader, 1, "local.file.source", SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
|
||||
|
||||
// time didn't pass so should refresh
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
VerifyGetSourceInvocationCount(dataSourceReader, 1, "local.file.source", SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
|
||||
|
||||
var expectedInterval = intervalCalls ?? TimeSpan.FromMinutes(30);
|
||||
|
||||
timeProvider.Advance(expectedInterval.Add(-TimeSpan.FromSeconds(2)));
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
VerifyGetSourceInvocationCount(dataSourceReader, 1, "local.file.source", SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
|
||||
|
||||
timeProvider.Advance(TimeSpan.FromSeconds(2));
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
Assert.AreEqual(referenceLocal.AddSeconds(callCount), enumerator.Current.EndTime);
|
||||
VerifyGetSourceInvocationCount(dataSourceReader, 2, "local.file.source", SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
|
||||
}
|
||||
|
||||
private static void VerifyGetSourceInvocationCount(Mock<ISubscriptionDataSourceReader> dataSourceReader, int count, string source, SubscriptionTransportMedium medium, FileFormat fileFormat)
|
||||
{
|
||||
dataSourceReader.Verify(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
|
||||
sds.Source == source && sds.TransportMedium == medium && sds.Format == fileFormat)), Times.Exactly(count));
|
||||
}
|
||||
|
||||
private static SubscriptionRequest GetSubscriptionRequest(SubscriptionDataConfig config, DateTime startTime, DateTime endTime)
|
||||
{
|
||||
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
|
||||
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, Symbols.SPY, SecurityType.Equity);
|
||||
var security = new Equity(
|
||||
Symbols.SPY,
|
||||
exchangeHours,
|
||||
quoteCurrency,
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
return new SubscriptionRequest(false, null, security, config, startTime, endTime);
|
||||
}
|
||||
|
||||
|
||||
class RestData : BaseData
|
||||
{
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
return new SubscriptionDataSource("rest.source", SubscriptionTransportMedium.Rest);
|
||||
}
|
||||
}
|
||||
|
||||
class RemoteCollectionData : BaseData
|
||||
{
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
return new SubscriptionDataSource("remote.collection.source", SubscriptionTransportMedium.RemoteFile, FileFormat.UnfoldingCollection);
|
||||
}
|
||||
}
|
||||
|
||||
class RestCollectionData : BaseData
|
||||
{
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
return new SubscriptionDataSource("rest.collection.source", SubscriptionTransportMedium.Rest, FileFormat.UnfoldingCollection);
|
||||
}
|
||||
}
|
||||
|
||||
class RemoteFileData : BaseData
|
||||
{
|
||||
public override DateTime EndTime
|
||||
{
|
||||
get { return Time + QuantConnect.Time.OneDay; }
|
||||
set { Time = value - QuantConnect.Time.OneDay; }
|
||||
}
|
||||
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
return new SubscriptionDataSource("remote.file.source", SubscriptionTransportMedium.RemoteFile);
|
||||
}
|
||||
}
|
||||
|
||||
class LocalFileData : BaseData
|
||||
{
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
return new SubscriptionDataSource("local.file.source", SubscriptionTransportMedium.LocalFile);
|
||||
}
|
||||
}
|
||||
|
||||
class TestableLiveCustomDataSubscriptionEnumeratorFactory : LiveCustomDataSubscriptionEnumeratorFactory
|
||||
{
|
||||
private readonly ISubscriptionDataSourceReader _dataSourceReader;
|
||||
|
||||
public TestableLiveCustomDataSubscriptionEnumeratorFactory(ITimeProvider timeProvider, ISubscriptionDataSourceReader dataSourceReader, TimeSpan? minimumIntervalCheck = null)
|
||||
: base(timeProvider, null, minimumIntervalCheck: minimumIntervalCheck)
|
||||
{
|
||||
_dataSourceReader = dataSourceReader;
|
||||
}
|
||||
|
||||
protected override ISubscriptionDataSourceReader GetSubscriptionDataSourceReader(SubscriptionDataSource source,
|
||||
IDataCacheProvider dataCacheProvider,
|
||||
SubscriptionDataConfig config,
|
||||
DateTime date,
|
||||
BaseData baseData,
|
||||
IDataProvider dataProvider)
|
||||
{
|
||||
return _dataSourceReader;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class FastForwardEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void FastForwardsOldData()
|
||||
{
|
||||
var start = new DateTime(2015, 10, 10, 13, 0, 0);
|
||||
var data = new List<Tick>
|
||||
{
|
||||
new Tick {Time = start.AddMinutes(-1)},
|
||||
new Tick {Time = start.AddSeconds(-1)},
|
||||
new Tick {Time = start.AddSeconds(0)},
|
||||
new Tick {Time = start.AddSeconds(1)},
|
||||
};
|
||||
|
||||
var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
|
||||
var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(0.5));
|
||||
|
||||
Assert.IsTrue(fastForward.MoveNext());
|
||||
Assert.AreEqual(start, fastForward.Current.Time);
|
||||
fastForward.Dispose();
|
||||
}
|
||||
[Test]
|
||||
public void FastForwardsOldDataAllowsEquals()
|
||||
{
|
||||
var start = new DateTime(2015, 10, 10, 13, 0, 0);
|
||||
var data = new List<Tick>
|
||||
{
|
||||
new Tick {Time = start.AddMinutes(-1)},
|
||||
new Tick {Time = start.AddSeconds(-1)},
|
||||
new Tick {Time = start.AddSeconds(0)},
|
||||
new Tick {Time = start.AddSeconds(1)},
|
||||
};
|
||||
|
||||
var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
|
||||
var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(1));
|
||||
|
||||
Assert.IsTrue(fastForward.MoveNext());
|
||||
Assert.AreEqual(start.AddSeconds(-1), fastForward.Current.Time);
|
||||
fastForward.Dispose();
|
||||
}
|
||||
[Test]
|
||||
public void FiltersOutPastData()
|
||||
{
|
||||
var start = new DateTime(2015, 10, 10, 13, 0, 0);
|
||||
var data = new List<Tick>
|
||||
{
|
||||
new Tick {Time = start.AddMinutes(-1)},
|
||||
new Tick {Time = start.AddSeconds(-1)},
|
||||
new Tick {Time = start.AddSeconds(1)},
|
||||
new Tick {Time = start.AddSeconds(0)},
|
||||
new Tick {Time = start.AddSeconds(2)}
|
||||
};
|
||||
|
||||
var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
|
||||
var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(0.5));
|
||||
|
||||
Assert.IsTrue(fastForward.MoveNext());
|
||||
Assert.AreEqual(start.AddSeconds(1), fastForward.Current.Time);
|
||||
|
||||
Assert.IsTrue(fastForward.MoveNext());
|
||||
Assert.AreEqual(start.AddSeconds(2), fastForward.Current.Time);
|
||||
fastForward.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CurrentIsNullWhenEnumeratorReturnsFalse()
|
||||
{
|
||||
var start = new DateTime(2015, 10, 10, 13, 0, 0);
|
||||
var data = new List<Tick>
|
||||
{
|
||||
new Tick {Time = start.AddSeconds(0)}
|
||||
};
|
||||
|
||||
var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
|
||||
var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(0.5));
|
||||
|
||||
Assert.IsTrue(fastForward.MoveNext());
|
||||
Assert.IsNotNull(fastForward.Current);
|
||||
|
||||
Assert.IsFalse(fastForward.MoveNext());
|
||||
Assert.IsNull(fastForward.Current);
|
||||
fastForward.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using NodaTime;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class FrontierAwareEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void ReturnsTrueWhenNextDataIsAheadOfFrontier()
|
||||
{
|
||||
var currentTime = new DateTime(2015, 10, 13);
|
||||
var timeProvider = new ManualTimeProvider(currentTime);
|
||||
var underlying = new List<Tick>
|
||||
{
|
||||
new Tick {Time = currentTime.AddSeconds(1)}
|
||||
};
|
||||
|
||||
var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
|
||||
var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
|
||||
|
||||
Assert.IsTrue(frontierAware.MoveNext());
|
||||
Assert.IsNull(frontierAware.Current);
|
||||
frontierAware.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void YieldsDataWhenFrontierPasses()
|
||||
{
|
||||
var currentTime = new DateTime(2015, 10, 13);
|
||||
var timeProvider = new ManualTimeProvider(currentTime);
|
||||
var underlying = new List<Tick>
|
||||
{
|
||||
new Tick {Time = currentTime.AddSeconds(1)}
|
||||
};
|
||||
|
||||
var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
|
||||
var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
|
||||
|
||||
timeProvider.AdvanceSeconds(1);
|
||||
|
||||
Assert.IsTrue(frontierAware.MoveNext());
|
||||
Assert.IsNotNull(frontierAware.Current);
|
||||
Assert.AreEqual(underlying[0], frontierAware.Current);
|
||||
frontierAware.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void YieldsFutureDataAtCorrectTime()
|
||||
{
|
||||
var currentTime = new DateTime(2015, 10, 13);
|
||||
var timeProvider = new ManualTimeProvider(currentTime);
|
||||
var underlying = new List<Tick>
|
||||
{
|
||||
new Tick {Time = currentTime.AddSeconds(10)}
|
||||
};
|
||||
|
||||
var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
|
||||
var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
|
||||
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
timeProvider.AdvanceSeconds(1);
|
||||
Assert.IsTrue(frontierAware.MoveNext());
|
||||
if (i < 9)
|
||||
{
|
||||
Assert.IsNull(frontierAware.Current);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.IsNotNull(frontierAware.Current);
|
||||
Assert.AreEqual(underlying[0], frontierAware.Current);
|
||||
}
|
||||
}
|
||||
frontierAware.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using System.Globalization;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Data.Market;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class LiveAuxiliaryDataEnumeratorTests
|
||||
{
|
||||
[TestCase(DataMappingMode.OpenInterest, "20130616", false)]
|
||||
[TestCase(DataMappingMode.FirstDayMonth, "20130602", false)]
|
||||
[TestCase(DataMappingMode.LastTradingDay, "20130623", false)]
|
||||
[TestCase(DataMappingMode.OpenInterest, "20130616", true)]
|
||||
[TestCase(DataMappingMode.FirstDayMonth, "20130602", true)]
|
||||
[TestCase(DataMappingMode.LastTradingDay, "20130623", true)]
|
||||
public void EmitsMappingEventsBasedOnCurrentMapFileAndTime(DataMappingMode dataMappingMode, string mappingDate, bool delayed)
|
||||
{
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.ES_Future_Chain,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
dataMappingMode: dataMappingMode);
|
||||
var symbolMaps = new List<SubscriptionDataConfig.NewSymbolEventArgs>();
|
||||
config.NewSymbol += (sender, args) => symbolMaps.Add(args);
|
||||
var time = new DateTime(2013, 05, 28);
|
||||
var cache = new SecurityCache();
|
||||
|
||||
cache.AddData(new Tick(time, config.Symbol, 20, 10));
|
||||
var timeProvider = new ManualTimeProvider(time);
|
||||
|
||||
var futureTicker1 = "es vhle2yxr5blt";
|
||||
TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, new[]
|
||||
{
|
||||
new MapFileRow(Time.BeginningOfTime, Futures.Indices.SP500EMini, Exchange.CME),
|
||||
new MapFileRow(new DateTime(2013,06,01), futureTicker1, Exchange.CME, DataMappingMode.FirstDayMonth),
|
||||
new MapFileRow(new DateTime(2013,06,15), futureTicker1, Exchange.CME, DataMappingMode.OpenInterest),
|
||||
new MapFileRow(new DateTime(2013,06,22), futureTicker1, Exchange.CME, DataMappingMode.LastTradingDay),
|
||||
});
|
||||
|
||||
IEnumerator<BaseData> enumerator;
|
||||
Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, cache, new TestMapFileProvider(), TestGlobals.FactorFileProvider, time, out enumerator));
|
||||
|
||||
// get's mapped right away!
|
||||
Assert.AreEqual(futureTicker1.ToUpper(), config.MappedSymbol);
|
||||
|
||||
Assert.AreEqual(1, symbolMaps.Count);
|
||||
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].Old);
|
||||
Assert.AreEqual(Futures.Indices.SP500EMini, symbolMaps[0].Old.ID.Symbol);
|
||||
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].New);
|
||||
Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[0].New.Underlying.ID.ToString());
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var expectedMappingDate = DateTime.ParseExact(mappingDate, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
|
||||
if (delayed)
|
||||
{
|
||||
// we advance to the mapping date, without any new mapFile!
|
||||
timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
|
||||
}
|
||||
else
|
||||
{
|
||||
// just advance a day to show nothing happens until mapping time
|
||||
timeProvider.Advance(TimeSpan.FromDays(1));
|
||||
}
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var futureTicker2 = "es vk2zrh843z7l";
|
||||
TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, TestMapFileResolver.MapFile.Concat(
|
||||
new[]
|
||||
{
|
||||
new MapFileRow(new DateTime(2013,09,01), futureTicker2, Exchange.CME, DataMappingMode.FirstDayMonth),
|
||||
new MapFileRow(new DateTime(2013,09,14), futureTicker2, Exchange.CME, DataMappingMode.OpenInterest),
|
||||
new MapFileRow(new DateTime(2013,09,21), futureTicker2, Exchange.CME, DataMappingMode.LastTradingDay),
|
||||
}));
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
if (delayed)
|
||||
{
|
||||
// we got a new mapFile! advance the date and expect mapping to have happened
|
||||
timeProvider.Advance(TimeSpan.FromDays(1));
|
||||
}
|
||||
else
|
||||
{
|
||||
// we advance to the mapping date
|
||||
timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
|
||||
}
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
|
||||
Assert.AreEqual(2, symbolMaps.Count);
|
||||
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].Old);
|
||||
Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[1].Old.Underlying.ID.ToString());
|
||||
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].New);
|
||||
Assert.AreEqual(futureTicker2.ToUpper(), symbolMaps[1].New.Underlying.ID.ToString());
|
||||
|
||||
Assert.AreEqual(futureTicker2.ToUpper(), config.MappedSymbol);
|
||||
|
||||
Assert.AreEqual(futureTicker2.ToUpper(), (enumerator.Current as SymbolChangedEvent).NewSymbol);
|
||||
Assert.AreEqual(futureTicker1.ToUpper(), (enumerator.Current as SymbolChangedEvent).OldSymbol);
|
||||
Assert.AreEqual(config.Symbol, (enumerator.Current as SymbolChangedEvent).Symbol);
|
||||
Assert.AreEqual(timeProvider.GetUtcNow().Date, (enumerator.Current as SymbolChangedEvent).Time);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmitsDelistingEventsBasedOnCurrentTime()
|
||||
{
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.SPY_C_192_Feb19_2016,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
var delistingDate = config.Symbol.GetDelistingDate();
|
||||
var time = delistingDate.AddDays(-10);
|
||||
var cache = new SecurityCache();
|
||||
cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 20, 10));
|
||||
var timeProvider = new ManualTimeProvider(time);
|
||||
|
||||
IEnumerator<BaseData> enumerator;
|
||||
Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, cache, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, config.Symbol.ID.Date, out enumerator));
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
// advance until delisting date, take into account 5 hour offset of NY + TradableDateOffset
|
||||
timeProvider.Advance(TimeSpan.FromDays(10));
|
||||
timeProvider.Advance(TimeSpan.FromHours(5));
|
||||
timeProvider.Advance(Time.LiveAuxiliaryDataOffset);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
|
||||
Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
|
||||
Assert.AreEqual(delistingDate, (enumerator.Current as Delisting).Time);
|
||||
Assert.AreEqual(15, (enumerator.Current as Delisting).Price);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
// when the day ends the delisted event will pass through, respecting the offset
|
||||
timeProvider.Advance(TimeSpan.FromDays(1));
|
||||
|
||||
cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 40, 20));
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
|
||||
Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
|
||||
Assert.AreEqual(delistingDate.AddDays(1), (enumerator.Current as Delisting).Time);
|
||||
Assert.AreEqual(30, (enumerator.Current as Delisting).Price);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public void EquityEmitsDelistingEventsBasedOnCurrentTime(bool delayed)
|
||||
{
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
var time = new DateTime(2013, 05, 28);
|
||||
var cache = new SecurityCache();
|
||||
|
||||
cache.AddData(new Tick(time, config.Symbol, 20, 10));
|
||||
var timeProvider = new ManualTimeProvider(time);
|
||||
|
||||
TestMapFileResolver.MapFile = new MapFile(config.Symbol.ID.Symbol, new[]
|
||||
{
|
||||
new MapFileRow(Time.BeginningOfTime, config.Symbol.ID.Symbol),
|
||||
new MapFileRow(Time.EndOfTime, config.Symbol.ID.Symbol),
|
||||
});
|
||||
|
||||
IEnumerator<BaseData> enumerator;
|
||||
Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, cache, new TestMapFileProvider(), TestGlobals.FactorFileProvider, time, out enumerator));
|
||||
|
||||
// get's mapped right away!
|
||||
Assert.AreEqual("SPY", config.MappedSymbol);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var delistingDate = time.AddDays(2);
|
||||
var delistedMapFile = new MapFile(config.Symbol.ID.Symbol, new[]
|
||||
{
|
||||
new MapFileRow(Time.BeginningOfTime, config.Symbol.ID.Symbol),
|
||||
new MapFileRow(delistingDate, config.Symbol.ID.Symbol),
|
||||
});
|
||||
|
||||
// just advance a day to show nothing happens until mapping time
|
||||
timeProvider.Advance(TimeSpan.FromDays(1));
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
if (!delayed)
|
||||
{
|
||||
TestMapFileResolver.MapFile = delistedMapFile;
|
||||
}
|
||||
|
||||
// we advance to the mapping date, without any new mapFile!
|
||||
timeProvider.Advance(delistingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
|
||||
|
||||
if (delayed)
|
||||
{
|
||||
// nothing happens
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
TestMapFileResolver.MapFile = delistedMapFile;
|
||||
|
||||
timeProvider.Advance(Time.OneDay);
|
||||
}
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
|
||||
var delisted = enumerator.Current as Delisting;
|
||||
Assert.IsNotNull(delisted);
|
||||
Assert.AreEqual(DelistingType.Warning, delisted.Type);
|
||||
|
||||
if (!delayed)
|
||||
{
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
// delisting passed
|
||||
timeProvider.Advance(TimeSpan.FromDays(1));
|
||||
}
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
|
||||
delisted = enumerator.Current as Delisting;
|
||||
Assert.IsNotNull(delisted);
|
||||
Assert.AreEqual(DelistingType.Delisted, delisted.Type);
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
private class TestMapFileProvider : IMapFileProvider
|
||||
{
|
||||
public void Initialize(IDataProvider dataProvider)
|
||||
{
|
||||
}
|
||||
|
||||
public MapFileResolver Get(AuxiliaryDataKey auxiliaryDataKey)
|
||||
{
|
||||
return new TestMapFileResolver();
|
||||
}
|
||||
}
|
||||
|
||||
private class TestMapFileResolver : MapFileResolver
|
||||
{
|
||||
public static MapFile MapFile { get; set; }
|
||||
public TestMapFileResolver()
|
||||
: base(new[] { MapFile })
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NodaTime;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using QuantConnect.Logging;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.All)]
|
||||
public class LiveEquityDataSynchronizingEnumeratorTests
|
||||
{
|
||||
// this test case generates data points in the past, will complete very quickly
|
||||
[TestCase(-15, 1)]
|
||||
// this test case generates data points in the future, will require at least 10 seconds to complete
|
||||
[TestCase(0, 11)]
|
||||
public void SynchronizesData(int timeOffsetSeconds, int testTimeSeconds)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
var end = start.AddSeconds(testTimeSeconds);
|
||||
|
||||
var time = start;
|
||||
var tickList1 = Enumerable.Range(0, 10).Select(x => new Tick { Time = time.AddSeconds(x * 1 + timeOffsetSeconds), Value = x }).ToList();
|
||||
var tickList2 = Enumerable.Range(0, 5).Select(x => new Tick { Time = time.AddSeconds(x * 2 + timeOffsetSeconds), Value = x + 100 }).ToList();
|
||||
var stream1 = tickList1.GetEnumerator();
|
||||
var stream2 = tickList2.GetEnumerator();
|
||||
|
||||
var count1 = 0;
|
||||
var count2 = 0;
|
||||
var previous = DateTime.MinValue;
|
||||
var synchronizer = new LiveAuxiliaryDataSynchronizingEnumerator(new RealTimeProvider(), DateTimeZone.Utc, stream1, new List<IEnumerator<BaseData>> { stream2 });
|
||||
while (synchronizer.MoveNext() && DateTime.UtcNow < end)
|
||||
{
|
||||
if (synchronizer.Current != null)
|
||||
{
|
||||
if (synchronizer.Current.Value < 100)
|
||||
{
|
||||
Assert.AreEqual(count1, synchronizer.Current.Value);
|
||||
count1++;
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.AreEqual(count2 + 100, synchronizer.Current.Value);
|
||||
count2++;
|
||||
}
|
||||
|
||||
Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous));
|
||||
previous = synchronizer.Current.EndTime;
|
||||
|
||||
Log.Trace($"Data point emitted: {synchronizer.Current.EndTime:O} - {synchronizer.Current}");
|
||||
}
|
||||
}
|
||||
|
||||
Log.Trace($"Total point count: {count1 + count2}");
|
||||
|
||||
Assert.AreEqual(tickList1.Count, count1);
|
||||
Assert.AreEqual(tickList2.Count, count2);
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,577 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Data.Market;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class LiveFillForwardEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void FillsForwardOnNulls()
|
||||
{
|
||||
var reference = new DateTime(2015, 10, 08);
|
||||
var period = Time.OneSecond;
|
||||
var underlying = new List<BaseData>
|
||||
{
|
||||
// 0 seconds
|
||||
new TradeBar(reference, Symbols.SPY, 10, 20, 5, 15, 123456, period),
|
||||
// 1 seconds
|
||||
null,
|
||||
// 3 seconds
|
||||
new TradeBar(reference.AddSeconds(2), Symbols.SPY, 100, 200, 50, 150, 1234560, period),
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
};
|
||||
|
||||
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
|
||||
timeProvider.SetCurrentTime(reference);
|
||||
var exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
|
||||
var fillForward = new LiveFillForwardEnumerator(timeProvider, underlying.GetEnumerator(), exchange, Ref.Create(Time.OneSecond), false, reference, Time.EndOfTime, Resolution.Second, exchange.TimeZone, false);
|
||||
|
||||
// first point is always emitted
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.AreEqual(underlying[0], fillForward.Current);
|
||||
Assert.AreEqual(123456, ((TradeBar)fillForward.Current).Volume);
|
||||
|
||||
// stepping again without advancing time does nothing, but we'll still
|
||||
// return true as per IEnumerator contract
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.IsNull(fillForward.Current);
|
||||
|
||||
timeProvider.SetCurrentTime(reference.AddSeconds(2));
|
||||
|
||||
// non-null next will fill forward in between
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.AreEqual(underlying[0].EndTime, fillForward.Current.Time);
|
||||
Assert.AreEqual(underlying[0].Value, fillForward.Current.Value);
|
||||
Assert.IsTrue(fillForward.Current.IsFillForward);
|
||||
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
|
||||
|
||||
// even without stepping the time this will advance since non-null data is ready
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.AreEqual(underlying[2], fillForward.Current);
|
||||
Assert.AreEqual(1234560, ((TradeBar)fillForward.Current).Volume);
|
||||
|
||||
// step ahead into null data territory
|
||||
timeProvider.SetCurrentTime(reference.AddSeconds(4));
|
||||
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
|
||||
Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork).RoundDown(Time.OneSecond), fillForward.Current.EndTime);
|
||||
Assert.IsTrue(fillForward.Current.IsFillForward);
|
||||
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
|
||||
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.IsNull(fillForward.Current);
|
||||
|
||||
timeProvider.SetCurrentTime(reference.AddSeconds(5));
|
||||
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
|
||||
Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork).RoundDown(Time.OneSecond), fillForward.Current.EndTime);
|
||||
Assert.IsTrue(fillForward.Current.IsFillForward);
|
||||
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
|
||||
|
||||
timeProvider.SetCurrentTime(reference.AddSeconds(6));
|
||||
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
|
||||
Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork).RoundDown(Time.OneSecond), fillForward.Current.EndTime);
|
||||
Assert.IsTrue(fillForward.Current.IsFillForward);
|
||||
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
|
||||
|
||||
fillForward.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void HandlesDaylightSavingTimeChange()
|
||||
{
|
||||
// In 2018, Daylight Saving Time (DST) began at 2 AM on Sunday, March 11
|
||||
// This means that clocks were moved forward one hour on March 11
|
||||
var reference = new DateTime(2018, 3, 10);
|
||||
var period = Time.OneDay;
|
||||
var underlying = new List<TradeBar>
|
||||
{
|
||||
new TradeBar(reference, Symbols.SPY, 10, 20, 5, 15, 123456, period),
|
||||
// Daylight Saving Time change, the data still goes from midnight to midnight
|
||||
new TradeBar(reference.AddDays(1), Symbols.SPY, 100, 200, 50, 150, 1234560, period)
|
||||
};
|
||||
|
||||
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
|
||||
timeProvider.SetCurrentTime(reference);
|
||||
var exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
|
||||
var fillForward = new LiveFillForwardEnumerator(
|
||||
timeProvider,
|
||||
underlying.GetEnumerator(),
|
||||
exchange,
|
||||
Ref.Create(Time.OneDay),
|
||||
false,
|
||||
reference,
|
||||
Time.EndOfTime,
|
||||
Resolution.Daily,
|
||||
exchange.TimeZone, false);
|
||||
|
||||
// first point is always emitted
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.IsFalse(fillForward.Current.IsFillForward);
|
||||
Assert.AreEqual(underlying[0], fillForward.Current);
|
||||
//Assert.AreEqual(underlying[0].EndTime, fillForward.Current.EndTime);
|
||||
Assert.AreEqual(123456, ((TradeBar)fillForward.Current).Volume);
|
||||
|
||||
// Daylight Saving Time change -> add 1 hour
|
||||
timeProvider.SetCurrentTime(reference.AddDays(1).AddHours(1));
|
||||
|
||||
// second data point emitted
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.IsFalse(fillForward.Current.IsFillForward);
|
||||
Assert.AreEqual(underlying[1], fillForward.Current);
|
||||
//Assert.AreEqual(underlying[1].EndTime, fillForward.Current.EndTime);
|
||||
Assert.AreEqual(1234560, ((TradeBar)fillForward.Current).Volume);
|
||||
|
||||
Assert.IsTrue(fillForward.MoveNext());
|
||||
Assert.IsTrue(fillForward.Current.IsFillForward);
|
||||
Assert.AreEqual(underlying[1].EndTime, fillForward.Current.Time);
|
||||
Assert.AreEqual(underlying[1].Value, fillForward.Current.Value);
|
||||
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
|
||||
|
||||
fillForward.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LiveFillForwardEnumeratorDoesNotStall()
|
||||
{
|
||||
var reference = new DateTime(2020, 5, 21, 9, 40, 0, 100);
|
||||
var timeProvider = new ManualTimeProvider(reference, TimeZones.NewYork);
|
||||
|
||||
using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, reference.Date, Resolution.Minute, out var enqueueableEnumerator, false);
|
||||
var openingBar = new TradeBar
|
||||
{
|
||||
Open = 0.01m,
|
||||
High = 0.01m,
|
||||
Low = 0.01m,
|
||||
Close = 0.01m,
|
||||
Volume = 1,
|
||||
EndTime = new DateTime(2020, 5, 21, 9, 40, 0),
|
||||
Symbol = Symbols.AAPL
|
||||
};
|
||||
var secondBar = new TradeBar
|
||||
{
|
||||
Open = 1m,
|
||||
High = 2m,
|
||||
Low = 1m,
|
||||
Close = 2m,
|
||||
Volume = 100,
|
||||
EndTime = new DateTime(2020, 5, 21, 9, 42, 0),
|
||||
Symbol = Symbols.AAPL
|
||||
};
|
||||
|
||||
|
||||
// Enqueue the first point, which will be emitted ASAP.
|
||||
enqueueableEnumerator.Enqueue(openingBar);
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.NotNull(fillForwardEnumerator.Current);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
|
||||
|
||||
// Advance the time, we expect a fill-forward bar.
|
||||
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 41, 0, 100) + LiveFillForwardEnumerator.GetMaximumDataTimeout(Resolution.Minute));
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(openingBar.EndTime.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
|
||||
|
||||
// Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
|
||||
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 42, 0, 100));
|
||||
enqueueableEnumerator.Enqueue(secondBar);
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
|
||||
enqueueableEnumerator.Dispose();
|
||||
}
|
||||
|
||||
private static IEnumerable<TestCaseData> TimeOutTestCases
|
||||
{
|
||||
get
|
||||
{
|
||||
// Hour resolution, fill forward to market open
|
||||
yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 10, 0, 0), true, true);
|
||||
yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 10, 0, 0), false, false);
|
||||
yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 10, 0, 0), null, true, false);
|
||||
yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 10, 0, 0), null, false, false);
|
||||
// over a weekend (22th is a Friday and 25th Monday is a holiday)
|
||||
yield return new(Resolution.Hour, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 10, 0, 0), true, true);
|
||||
yield return new(Resolution.Hour, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 10, 0, 0), false, false);
|
||||
// market close
|
||||
yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 15, 0, 0), null, true, true);
|
||||
yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 15, 0, 0), null, false, false);
|
||||
|
||||
// Minute resolution, fill forward to market open
|
||||
yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 9, 31, 0), true, true);
|
||||
yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 9, 31, 0), false, false);
|
||||
yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 9, 31, 0), null, true, false);
|
||||
yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 9, 31, 0), null, false, false);
|
||||
// over a weekend
|
||||
yield return new(Resolution.Minute, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 9, 31, 0), true, true);
|
||||
yield return new(Resolution.Minute, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 9, 31, 0), false, false);
|
||||
// market close
|
||||
yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 15, 59, 0), null, true, true);
|
||||
yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 15, 59, 0), null, false, false);
|
||||
|
||||
// Second resolution, fill forward to market open
|
||||
yield return new(Resolution.Second, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 9, 30, 1), true, true);
|
||||
yield return new(Resolution.Second, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 9, 30, 1), false, false);
|
||||
yield return new(Resolution.Second, new DateTime(2020, 5, 21, 9, 30, 1), null, true, false);
|
||||
yield return new(Resolution.Second, new DateTime(2020, 5, 21, 9, 30, 1), null, false, false);
|
||||
// over a weekend
|
||||
yield return new(Resolution.Second, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 9, 30, 1), true, true);
|
||||
yield return new(Resolution.Second, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 9, 30, 1), false, false);
|
||||
// market close
|
||||
yield return new(Resolution.Second, new DateTime(2020, 5, 21, 15, 59, 59), null, true, true);
|
||||
yield return new(Resolution.Second, new DateTime(2020, 5, 21, 15, 59, 59), null, false, false);
|
||||
}
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(TimeOutTestCases))]
|
||||
public void TakesIntoAccountTimeOut(Resolution resolution, DateTime previousDataEndTime, DateTime? expectedNextBarEndTime, bool dataArrivedLate, bool shouldHaveTimeout)
|
||||
{
|
||||
var timeProvider = new ManualTimeProvider(previousDataEndTime, TimeZones.NewYork);
|
||||
|
||||
using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, previousDataEndTime.Date, resolution, out var enqueueableEnumerator, dailyStrictEndTimeEnabled: false);
|
||||
var period = resolution.ToTimeSpan();
|
||||
var openingBar = new TradeBar(previousDataEndTime.Subtract(period), Symbols.AAPL, 0.01m, 0.01m, 0.01m, 0.01m, 1, period);
|
||||
|
||||
expectedNextBarEndTime ??= previousDataEndTime.Add(resolution.ToTimeSpan());
|
||||
var secondBar = new TradeBar(openingBar.EndTime, Symbols.AAPL, 1m, 2m, 1m, 2m, 100, period);
|
||||
|
||||
// Enqueue the first point, which will be emitted ASAP.
|
||||
enqueueableEnumerator.Enqueue(openingBar);
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.NotNull(fillForwardEnumerator.Current);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
|
||||
|
||||
// Advance the time, we don't expect a fill-forward bar because the timeout amount has not passed yet
|
||||
timeProvider.SetCurrentTime(expectedNextBarEndTime.Value);
|
||||
if (dataArrivedLate)
|
||||
{
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
|
||||
if (shouldHaveTimeout)
|
||||
{
|
||||
Assert.IsNull(fillForwardEnumerator.Current);
|
||||
|
||||
// Advance the time, including the expected timout, we expect a fill-forward bar.
|
||||
timeProvider.Advance(LiveFillForwardEnumerator.GetMaximumDataTimeout(resolution));
|
||||
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
}
|
||||
|
||||
Assert.IsNotNull(fillForwardEnumerator.Current);
|
||||
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(expectedNextBarEndTime, fillForwardEnumerator.Current.EndTime);
|
||||
|
||||
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(expectedNextBarEndTime, fillForwardEnumerator.Current.EndTime);
|
||||
|
||||
}
|
||||
|
||||
// Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
|
||||
enqueueableEnumerator.Enqueue(secondBar);
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
|
||||
enqueueableEnumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TakesIntoAccountTimeOutWhenThereAreBigGaps([Values(Resolution.Second, Resolution.Minute, Resolution.Hour)] Resolution resolution)
|
||||
{
|
||||
var start = new DateTime(2020, 5, 20, 12, 0, 0);
|
||||
var end = new DateTime(2020, 5, 22, 16, 0, 0);
|
||||
|
||||
var timeProvider = new ManualTimeProvider(start, TimeZones.NewYork);
|
||||
|
||||
using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, start, resolution, out var enqueueableEnumerator, dailyStrictEndTimeEnabled: false);
|
||||
var period = resolution.ToTimeSpan();
|
||||
var openingBar = new TradeBar(start.Subtract(period), Symbols.AAPL, 0.01m, 0.01m, 0.01m, 0.01m, 1, period);
|
||||
|
||||
// Enqueue the first point, which will be emitted ASAP.
|
||||
enqueueableEnumerator.Enqueue(openingBar);
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.NotNull(fillForwardEnumerator.Current);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
|
||||
|
||||
timeProvider.Advance(period);
|
||||
|
||||
var previous = (TradeBar)fillForwardEnumerator.Current;
|
||||
var currentIsMarketOpen = false;
|
||||
var currentIsMarketClose = false;
|
||||
while (previous.EndTime < end)
|
||||
{
|
||||
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
|
||||
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext(), $"Previous: {previous.EndTime}");
|
||||
|
||||
if (currentIsMarketOpen || currentIsMarketClose)
|
||||
{
|
||||
Assert.IsNull(fillForwardEnumerator.Current, $"Previous: {previous.EndTime}");
|
||||
|
||||
// Advance the time, including the expected timout, we expect a fill-forward bar.
|
||||
timeProvider.Advance(LiveFillForwardEnumerator.GetMaximumDataTimeout(resolution));
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext(), $"Previous: {previous.EndTime}");
|
||||
}
|
||||
|
||||
Assert.IsNotNull(fillForwardEnumerator.Current, $"Previous: {previous.EndTime}");
|
||||
|
||||
var current = (TradeBar)fillForwardEnumerator.Current;
|
||||
Assert.IsTrue(current.IsFillForward, $"Current: {previous.EndTime}");
|
||||
Assert.AreEqual(previous.Open, current.Open, $"Current: {previous.EndTime}");
|
||||
Assert.AreEqual(previous.Price, current.Price, $"Current: {previous.EndTime}");
|
||||
|
||||
var expectedEndTime = previous.EndTime.Add(period);
|
||||
if (currentIsMarketOpen)
|
||||
{
|
||||
expectedEndTime = expectedEndTime.Date.AddDays(1).AddHours(9).AddMinutes(30).Add(period);
|
||||
if (resolution == Resolution.Hour)
|
||||
{
|
||||
expectedEndTime = expectedEndTime.RoundDown(period);
|
||||
}
|
||||
}
|
||||
Assert.AreEqual(expectedEndTime, current.EndTime, $"Current: {previous.EndTime}");
|
||||
|
||||
currentIsMarketOpen = current.EndTime.TimeOfDay == TimeSpan.FromHours(16);
|
||||
currentIsMarketClose = current.EndTime.TimeOfDay == TimeSpan.FromHours(16) - period;
|
||||
previous = current;
|
||||
|
||||
// Advance the time provider
|
||||
var nextTime = current.EndTime.Add(period);
|
||||
if (nextTime.TimeOfDay > TimeSpan.FromHours(16))
|
||||
{
|
||||
// If the next time is after market close, we need to advance to the next day
|
||||
nextTime = nextTime.Date.AddDays(1).AddHours(9).AddMinutes(30).Add(period);
|
||||
if (resolution == Resolution.Hour)
|
||||
{
|
||||
nextTime = nextTime.RoundDown(period);
|
||||
}
|
||||
}
|
||||
timeProvider.SetCurrentTime(nextTime);
|
||||
}
|
||||
|
||||
enqueueableEnumerator.Dispose();
|
||||
}
|
||||
|
||||
[TestCase(true, true)]
|
||||
[TestCase(false, true)]
|
||||
[TestCase(true, false)]
|
||||
[TestCase(false, false)]
|
||||
public void TakesIntoAccountTimeOutDaily(bool dailyStrictEndTimeEnabled, bool dataArrivedLate)
|
||||
{
|
||||
var resolution = Resolution.Daily;
|
||||
var referenceOpenTime = new DateTime(2020, 5, 21, 9, 30, 0);
|
||||
var referenceTime = new DateTime(2020, 5, 21, 16, 0, 0);
|
||||
if (!dailyStrictEndTimeEnabled)
|
||||
{
|
||||
referenceOpenTime = referenceOpenTime.Date;
|
||||
referenceTime = referenceTime.Date.AddDays(1);
|
||||
}
|
||||
var timeProvider = new ManualTimeProvider(referenceTime, TimeZones.NewYork);
|
||||
using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, referenceTime.Date, resolution, out var enqueueableEnumerator, dailyStrictEndTimeEnabled);
|
||||
var openingBar = new TradeBar
|
||||
{
|
||||
Open = 0.01m,
|
||||
High = 0.01m,
|
||||
Low = 0.01m,
|
||||
Close = 0.01m,
|
||||
Volume = 1,
|
||||
Time = referenceOpenTime,
|
||||
EndTime = referenceTime,
|
||||
Symbol = Symbols.AAPL
|
||||
};
|
||||
var secondBar = new TradeBar
|
||||
{
|
||||
Open = 1m,
|
||||
High = 2m,
|
||||
Low = 1m,
|
||||
Close = 2m,
|
||||
Volume = 100,
|
||||
Time = referenceOpenTime.AddDays(1),
|
||||
EndTime = referenceTime.AddDays(1),
|
||||
Symbol = Symbols.AAPL
|
||||
};
|
||||
|
||||
// Enqueue the first point, which will be emitted ASAP.
|
||||
enqueueableEnumerator.Enqueue(openingBar);
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.NotNull(fillForwardEnumerator.Current);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
|
||||
|
||||
// Advance the time, we don't expect a fill-forward bar because the timeout amount has not passed yet
|
||||
timeProvider.SetCurrentTime(secondBar.EndTime);
|
||||
|
||||
if (dataArrivedLate)
|
||||
{
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.IsNull(fillForwardEnumerator.Current);
|
||||
|
||||
// Advance the time, including the expected timout, we expect a fill-forward bar.
|
||||
timeProvider.Advance(LiveFillForwardEnumerator.GetMaximumDataTimeout(resolution));
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(secondBar.EndTime, fillForwardEnumerator.Current.EndTime);
|
||||
}
|
||||
|
||||
// Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
|
||||
enqueueableEnumerator.Enqueue(secondBar);
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
|
||||
Assert.AreEqual(secondBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(secondBar.EndTime, fillForwardEnumerator.Current.EndTime);
|
||||
enqueueableEnumerator.Dispose();
|
||||
}
|
||||
|
||||
[TestCase(Resolution.Minute)]
|
||||
[TestCase(Resolution.Hour)]
|
||||
public void MultiResolutionSmallerFillForwardResolution(Resolution resolution)
|
||||
{
|
||||
var ffResolution = Resolution.Second;
|
||||
var referenceOpenTime = new DateTime(2020, 5, 21, 14, 0, 0);
|
||||
if (resolution == Resolution.Minute)
|
||||
{
|
||||
referenceOpenTime = new DateTime(2020, 5, 21, 15, 59, 0);
|
||||
}
|
||||
var referenceTime = new DateTime(2020, 5, 21, 15, 0, 0);
|
||||
|
||||
var timeProvider = new ManualTimeProvider(referenceTime, TimeZones.NewYork);
|
||||
using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, referenceTime.Date, resolution, out var enqueueableEnumerator, true, ffResolution);
|
||||
var openingBar = new TradeBar
|
||||
{
|
||||
Open = 0.01m,
|
||||
High = 0.01m,
|
||||
Low = 0.01m,
|
||||
Close = 0.01m,
|
||||
Volume = 1,
|
||||
Time = referenceOpenTime,
|
||||
EndTime = referenceTime,
|
||||
Symbol = Symbols.AAPL
|
||||
};
|
||||
|
||||
// Enqueue the first point, which will be emitted ASAP.
|
||||
enqueueableEnumerator.Enqueue(openingBar);
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.NotNull(fillForwardEnumerator.Current);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
|
||||
|
||||
// Advance the time, we expect a fill-forward bar
|
||||
|
||||
for (var i = 0; i < 60; i++)
|
||||
{
|
||||
timeProvider.Advance(Time.OneSecond);
|
||||
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.NotNull(fillForwardEnumerator.Current);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork), fillForwardEnumerator.Current.EndTime);
|
||||
}
|
||||
|
||||
enqueueableEnumerator.Dispose();
|
||||
}
|
||||
|
||||
[TestCase(Resolution.Daily, Resolution.Minute)]
|
||||
[TestCase(Resolution.Hour, Resolution.Minute)]
|
||||
[TestCase(Resolution.Daily, Resolution.Second)]
|
||||
[TestCase(Resolution.Hour, Resolution.Second)]
|
||||
public void MultiResolutionMarketClose(Resolution resolution, Resolution ffResolution)
|
||||
{
|
||||
var referenceOpenTime = new DateTime(2020, 5, 21, 9, 30, 0);
|
||||
if (resolution == Resolution.Hour)
|
||||
{
|
||||
referenceOpenTime = new DateTime(2020, 5, 21, 15, 0, 0);
|
||||
}
|
||||
var referenceTime = new DateTime(2020, 5, 21, 16, 0, 0);
|
||||
var timeProvider = new ManualTimeProvider(referenceTime, TimeZones.NewYork);
|
||||
using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, referenceTime.Date, resolution, out var enqueueableEnumerator, true, ffResolution);
|
||||
var openingBar = new TradeBar
|
||||
{
|
||||
Open = 0.01m,
|
||||
High = 0.01m,
|
||||
Low = 0.01m,
|
||||
Close = 0.01m,
|
||||
Volume = 1,
|
||||
Time = referenceOpenTime,
|
||||
EndTime = referenceTime,
|
||||
Symbol = Symbols.AAPL
|
||||
};
|
||||
|
||||
// Enqueue the first point, which will be emitted ASAP.
|
||||
enqueueableEnumerator.Enqueue(openingBar);
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.NotNull(fillForwardEnumerator.Current);
|
||||
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
|
||||
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
|
||||
|
||||
for (var i = 0; i < 600; i++)
|
||||
{
|
||||
// Advance the time, we don't expect a fill-forward bar, we've emitted our daily bar already and market is closed
|
||||
timeProvider.Advance(ffResolution.ToTimeSpan());
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
Assert.IsNull(fillForwardEnumerator.Current);
|
||||
}
|
||||
|
||||
enqueueableEnumerator.Dispose();
|
||||
}
|
||||
|
||||
private static LiveFillForwardEnumerator GetLiveFillForwardEnumerator(ITimeProvider timeProvider, DateTime startTime, Resolution resolution,
|
||||
out EnqueueableEnumerator<BaseData> enqueueableEnumerator, bool dailyStrictEndTimeEnabled, Resolution? ffResolution = null)
|
||||
{
|
||||
enqueueableEnumerator = new EnqueueableEnumerator<BaseData>();
|
||||
var fillForwardEnumerator = new LiveFillForwardEnumerator(
|
||||
timeProvider,
|
||||
enqueueableEnumerator,
|
||||
new SecurityExchange(MarketHoursDatabase.FromDataFolder()
|
||||
.ExchangeHoursListing
|
||||
.First(kvp => kvp.Key.Market == Market.USA && kvp.Key.SecurityType == SecurityType.Equity)
|
||||
.Value
|
||||
.ExchangeHours),
|
||||
Ref.CreateReadOnly(() => (ffResolution ?? resolution).ToTimeSpan()),
|
||||
false,
|
||||
startTime,
|
||||
Time.EndOfTime,
|
||||
resolution,
|
||||
TimeZones.NewYork,
|
||||
dailyStrictEndTimeEnabled: dailyStrictEndTimeEnabled
|
||||
);
|
||||
|
||||
return fillForwardEnumerator;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using NodaTime;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Data.Market;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.All)]
|
||||
public class LiveSubscriptionEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void HandlesSymbolMapping()
|
||||
{
|
||||
var canonical = Symbols.Fut_SPY_Feb19_2016.Canonical;
|
||||
var dataQueue = new TestDataQueueHandler
|
||||
{
|
||||
DataPerSymbol = new Dictionary<Symbol, List<BaseData>>
|
||||
{
|
||||
{ Symbols.Fut_SPY_Feb19_2016,
|
||||
new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Feb19_2016, 1, 1)} },
|
||||
{ Symbols.Fut_SPY_Mar19_2016,
|
||||
new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Mar19_2016, 2, 2)} },
|
||||
}
|
||||
};
|
||||
var config = new SubscriptionDataConfig(typeof(Tick), canonical, Resolution.Tick,
|
||||
DateTimeZone.Utc, DateTimeZone.Utc, false, false, false)
|
||||
{
|
||||
MappedSymbol = Symbols.Fut_SPY_Feb19_2016.ID.ToString()
|
||||
};
|
||||
|
||||
var compositeDataQueueHandler = new TestDataQueueHandlerManager(new AlgorithmSettings());
|
||||
compositeDataQueueHandler.ExposedDataHandlers.Add(dataQueue);
|
||||
var data = new LiveSubscriptionEnumerator(config, compositeDataQueueHandler, (_, _) => {}, (_) => false);
|
||||
|
||||
Assert.IsTrue(data.MoveNext());
|
||||
Assert.AreEqual(1, (data.Current as Tick).AskPrice);
|
||||
Assert.AreEqual(canonical, (data.Current as Tick).Symbol);
|
||||
|
||||
Assert.IsFalse(data.MoveNext());
|
||||
Assert.IsNull(data.Current);
|
||||
|
||||
Assert.AreEqual(2, dataQueue.DataPerSymbol.Count);
|
||||
config.MappedSymbol = Symbols.Fut_SPY_Mar19_2016.ID.ToString();
|
||||
Assert.AreEqual(1, dataQueue.DataPerSymbol.Count);
|
||||
|
||||
Assert.IsTrue(data.MoveNext());
|
||||
Assert.AreEqual(2, (data.Current as Tick).AskPrice);
|
||||
Assert.AreEqual(canonical, (data.Current as Tick).Symbol);
|
||||
Assert.AreNotEqual(canonical, dataQueue.DataPerSymbol[Symbols.Fut_SPY_Mar19_2016].Single().Symbol);
|
||||
|
||||
Assert.IsFalse(data.MoveNext());
|
||||
Assert.IsNull(data.Current);
|
||||
|
||||
Assert.AreEqual(1, dataQueue.DataPerSymbol.Count);
|
||||
|
||||
data.Dispose();
|
||||
compositeDataQueueHandler.Dispose();
|
||||
}
|
||||
|
||||
internal class TestDataQueueHandler : IDataQueueHandler
|
||||
{
|
||||
public bool IsConnected => true;
|
||||
|
||||
public Dictionary<Symbol, List<BaseData>> DataPerSymbol;
|
||||
|
||||
public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
|
||||
{
|
||||
if (DataPerSymbol.TryGetValue(dataConfig.Symbol, out var baseDatas))
|
||||
{
|
||||
return baseDatas.GetEnumerator();
|
||||
}
|
||||
throw new Exception($"Failed to find a data enumerator for symbol {dataConfig.Symbol}!");
|
||||
}
|
||||
public void Unsubscribe(SubscriptionDataConfig dataConfig)
|
||||
{
|
||||
DataPerSymbol.Remove(dataConfig.Symbol);
|
||||
}
|
||||
public void SetJob(LiveNodePacket job)
|
||||
{
|
||||
}
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private class TestDataQueueHandlerManager : DataQueueHandlerManager
|
||||
{
|
||||
public List<IDataQueueHandler> ExposedDataHandlers => DataHandlers;
|
||||
public TestDataQueueHandlerManager(IAlgorithmSettings settings) : base(settings)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class MappingEventProviderTests
|
||||
{
|
||||
private SubscriptionDataConfig _config;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
var symbol = Symbol.Create("TFCFA", SecurityType.Equity, Market.USA);
|
||||
|
||||
_config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
symbol,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void InitialMapping()
|
||||
{
|
||||
var provider = new MappingEventProvider();
|
||||
|
||||
Assert.AreEqual("TFCFA", _config.MappedSymbol);
|
||||
|
||||
provider.Initialize(_config,
|
||||
null,
|
||||
TestGlobals.MapFileProvider,
|
||||
new DateTime(2006, 1, 1));
|
||||
|
||||
Assert.AreEqual("NWSA", _config.MappedSymbol);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MappingEvent()
|
||||
{
|
||||
var provider = new MappingEventProvider();
|
||||
provider.Initialize(_config,
|
||||
null,
|
||||
TestGlobals.MapFileProvider,
|
||||
new DateTime(2006, 1, 1));
|
||||
|
||||
Assert.AreEqual("NWSA", _config.MappedSymbol);
|
||||
|
||||
var symbolEvent = (SymbolChangedEvent)provider
|
||||
.GetEvents(new NewTradableDateEventArgs(
|
||||
new DateTime(2013, 6, 29),
|
||||
null,
|
||||
_config.Symbol,
|
||||
null)).Single();
|
||||
|
||||
Assert.AreEqual("FOXA", symbolEvent.NewSymbol);
|
||||
Assert.AreEqual("NWSA", symbolEvent.OldSymbol);
|
||||
Assert.AreEqual("FOXA", _config.MappedSymbol);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using System.Collections;
|
||||
using QuantConnect.Data.Market;
|
||||
using System.Collections.Generic;
|
||||
using Tick = QuantConnect.Data.Market.Tick;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using System.Linq;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class PriceScaleFactorEnumeratorTests
|
||||
{
|
||||
private SubscriptionDataConfig _config;
|
||||
private RawDataEnumerator _rawDataEnumerator;
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
_config = GetConfig(Symbols.SPY, Resolution.Daily);
|
||||
_rawDataEnumerator = new RawDataEnumerator();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EquityTradeBar()
|
||||
{
|
||||
var enumerator = new PriceScaleFactorEnumerator(
|
||||
_rawDataEnumerator,
|
||||
_config,
|
||||
TestGlobals.FactorFileProvider);
|
||||
_rawDataEnumerator.CurrentValue = new TradeBar(
|
||||
new DateTime(2018, 1, 1),
|
||||
_config.Symbol,
|
||||
10,
|
||||
10,
|
||||
10,
|
||||
10,
|
||||
100);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var tradeBar = enumerator.Current as TradeBar;
|
||||
var expectedValue = 10 * _config.PriceScaleFactor;
|
||||
|
||||
Assert.Less(expectedValue, 10);
|
||||
Assert.AreEqual(expectedValue, tradeBar.Price);
|
||||
Assert.AreEqual(expectedValue, tradeBar.Open);
|
||||
Assert.AreEqual(expectedValue, tradeBar.Close);
|
||||
Assert.AreEqual(expectedValue, tradeBar.High);
|
||||
Assert.AreEqual(expectedValue, tradeBar.Low);
|
||||
Assert.AreEqual(expectedValue, tradeBar.Value);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EquityQuoteBar()
|
||||
{
|
||||
var enumerator = new PriceScaleFactorEnumerator(
|
||||
_rawDataEnumerator,
|
||||
_config,
|
||||
TestGlobals.FactorFileProvider);
|
||||
_rawDataEnumerator.CurrentValue = new QuoteBar(
|
||||
new DateTime(2018, 1, 1),
|
||||
_config.Symbol,
|
||||
new Bar(10, 10, 10, 10),
|
||||
100,
|
||||
new Bar(10, 10, 10, 10),
|
||||
100);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var quoteBar = enumerator.Current as QuoteBar;
|
||||
var expectedValue = 10 * _config.PriceScaleFactor;
|
||||
|
||||
Assert.Less(expectedValue, 10);
|
||||
|
||||
Assert.AreEqual(expectedValue, quoteBar.Price);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Value);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Open);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Close);
|
||||
Assert.AreEqual(expectedValue, quoteBar.High);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Low);
|
||||
// bid
|
||||
Assert.AreEqual(expectedValue, quoteBar.Bid.Open);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Bid.Close);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Bid.High);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Bid.Low);
|
||||
// ask
|
||||
Assert.AreEqual(expectedValue, quoteBar.Ask.Open);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Ask.Close);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Ask.High);
|
||||
Assert.AreEqual(expectedValue, quoteBar.Ask.Low);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EquityTick()
|
||||
{
|
||||
var enumerator = new PriceScaleFactorEnumerator(
|
||||
_rawDataEnumerator,
|
||||
_config,
|
||||
TestGlobals.FactorFileProvider);
|
||||
_rawDataEnumerator.CurrentValue = new Tick(
|
||||
new DateTime(2018, 1, 1),
|
||||
_config.Symbol,
|
||||
10,
|
||||
10,
|
||||
10);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var tick = enumerator.Current as Tick;
|
||||
var expectedValue = 10 * _config.PriceScaleFactor;
|
||||
|
||||
Assert.Less(expectedValue, 10);
|
||||
Assert.AreEqual(expectedValue, tick.Price);
|
||||
Assert.AreEqual(expectedValue, tick.Value);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FactorFileIsNull()
|
||||
{
|
||||
var enumerator = new PriceScaleFactorEnumerator(
|
||||
_rawDataEnumerator,
|
||||
_config,
|
||||
null);
|
||||
_rawDataEnumerator.CurrentValue = new Tick(
|
||||
new DateTime(2018, 1, 1),
|
||||
_config.Symbol,
|
||||
10,
|
||||
10,
|
||||
10);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var tick = enumerator.Current as Tick;
|
||||
Assert.AreEqual(10, tick.Price);
|
||||
Assert.AreEqual(10, tick.Value);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RawEnumeratorReturnsFalse()
|
||||
{
|
||||
var enumerator = new PriceScaleFactorEnumerator(
|
||||
_rawDataEnumerator,
|
||||
_config,
|
||||
TestGlobals.FactorFileProvider);
|
||||
_rawDataEnumerator.CurrentValue = new Tick(
|
||||
new DateTime(2018, 1, 1),
|
||||
_config.Symbol,
|
||||
10,
|
||||
10,
|
||||
10);
|
||||
_rawDataEnumerator.MoveNextReturnValue = false;
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.AreEqual(_rawDataEnumerator.CurrentValue, enumerator.Current);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RawEnumeratorCurrentIsNull()
|
||||
{
|
||||
var enumerator = new PriceScaleFactorEnumerator(
|
||||
_rawDataEnumerator,
|
||||
_config,
|
||||
TestGlobals.FactorFileProvider);
|
||||
_rawDataEnumerator.CurrentValue = null;
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UpdatesFactorFileCorrectly()
|
||||
{
|
||||
var dateBeforeUpadate = new DateTime(2018, 3, 14);
|
||||
var dateAtUpadate = new DateTime(2018, 3, 15);
|
||||
var dateAfterUpadate = new DateTime(2018, 3, 16);
|
||||
|
||||
var enumerator = new PriceScaleFactorEnumerator(
|
||||
_rawDataEnumerator,
|
||||
_config,
|
||||
TestGlobals.FactorFileProvider);
|
||||
|
||||
// Before factor file update date (2018, 3, 15)
|
||||
_rawDataEnumerator.CurrentValue = new Tick(
|
||||
dateBeforeUpadate,
|
||||
_config.Symbol,
|
||||
10,
|
||||
10,
|
||||
10);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var factorFile = TestGlobals.FactorFileProvider.Get(_config.Symbol);
|
||||
var expectedFactor = factorFile.GetPriceFactor(dateBeforeUpadate, DataNormalizationMode.Adjusted);
|
||||
var tick = enumerator.Current as Tick;
|
||||
Assert.AreEqual(expectedFactor, _config.PriceScaleFactor);
|
||||
Assert.AreEqual(10 * expectedFactor, tick.Price);
|
||||
Assert.AreEqual(10 * expectedFactor, tick.Value);
|
||||
|
||||
// At factor file update date (2018, 3, 15)
|
||||
_rawDataEnumerator.CurrentValue = new Tick(
|
||||
dateAtUpadate,
|
||||
_config.Symbol,
|
||||
10,
|
||||
10,
|
||||
10);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var expectedFactor2 = factorFile.GetPriceFactor(dateAtUpadate, DataNormalizationMode.Adjusted);
|
||||
var tick2 = enumerator.Current as Tick;
|
||||
Assert.AreEqual(expectedFactor2, _config.PriceScaleFactor);
|
||||
Assert.AreEqual(10 * expectedFactor2, tick2.Price);
|
||||
Assert.AreEqual(10 * expectedFactor2, tick2.Value);
|
||||
|
||||
// After factor file update date (2018, 3, 15)
|
||||
_rawDataEnumerator.CurrentValue = new Tick(
|
||||
dateAfterUpadate,
|
||||
_config.Symbol,
|
||||
10,
|
||||
10,
|
||||
10);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var expectedFactor3 = factorFile.GetPriceFactor(dateAfterUpadate, DataNormalizationMode.Adjusted);
|
||||
var tick3 = enumerator.Current as Tick;
|
||||
Assert.AreEqual(expectedFactor3, _config.PriceScaleFactor);
|
||||
Assert.AreEqual(10 * expectedFactor3, tick3.Price);
|
||||
Assert.AreEqual(10 * expectedFactor3, tick3.Value);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PricesAreProperlyAdjustedForLookAheadScaledRawDataNormalizationMode()
|
||||
{
|
||||
var factorFileEntries = new[]
|
||||
{
|
||||
new DateTime(2005, 02, 25),
|
||||
new DateTime(2012, 08, 08),
|
||||
new DateTime(2013, 05, 08),
|
||||
new DateTime(2014, 08, 06),
|
||||
new DateTime(2015, 08, 05)
|
||||
};
|
||||
var endDate = factorFileEntries.Last().AddDays(1);
|
||||
|
||||
var config = GetConfig(Symbols.AAPL, Resolution.Daily);
|
||||
config.DataNormalizationMode = DataNormalizationMode.ScaledRaw;
|
||||
|
||||
using var enumerator = new PriceScaleFactorEnumerator(
|
||||
_rawDataEnumerator,
|
||||
config,
|
||||
TestGlobals.FactorFileProvider,
|
||||
endDate: endDate);
|
||||
|
||||
var price = 100m;
|
||||
var factorFile = TestGlobals.FactorFileProvider.Get(config.Symbol);
|
||||
var endDateFactor = factorFile.GetPriceFactor(endDate, config.DataNormalizationMode);
|
||||
|
||||
var performAssertions = (DateTime date) =>
|
||||
{
|
||||
var expectedFactor = factorFile.GetPriceFactor(date, config.DataNormalizationMode);
|
||||
Assert.AreEqual(expectedFactor / endDateFactor, config.PriceScaleFactor);
|
||||
|
||||
var tradeBar = enumerator.Current as TradeBar;
|
||||
var expectedValue = price * config.PriceScaleFactor;
|
||||
Assert.AreEqual(expectedValue, tradeBar.Price);
|
||||
Assert.AreEqual(expectedValue, tradeBar.Open);
|
||||
Assert.AreEqual(expectedValue, tradeBar.Close);
|
||||
Assert.AreEqual(expectedValue, tradeBar.High);
|
||||
Assert.AreEqual(expectedValue, tradeBar.Low);
|
||||
Assert.AreEqual(expectedValue, tradeBar.Value);
|
||||
|
||||
return expectedFactor;
|
||||
};
|
||||
|
||||
foreach (var factorFileDate in factorFileEntries)
|
||||
{
|
||||
// before split
|
||||
var dateBeforeSplit = factorFileDate.AddDays(-1);
|
||||
_rawDataEnumerator.CurrentValue = new TradeBar(dateBeforeSplit, config.Symbol, price, price, price, price, price);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var expectedFactorBeforeSplit = performAssertions(dateBeforeSplit);
|
||||
|
||||
// at split
|
||||
_rawDataEnumerator.CurrentValue = new TradeBar(factorFileDate, config.Symbol, price, price, price, price, price);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var expectedFactorAtSplit = performAssertions(factorFileDate);
|
||||
Assert.AreEqual(expectedFactorBeforeSplit, expectedFactorAtSplit);
|
||||
|
||||
// after split
|
||||
var dateAfterSplit = factorFileDate.AddDays(1);
|
||||
_rawDataEnumerator.CurrentValue = new TradeBar(dateAfterSplit, config.Symbol, price, price, price, price, price);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
var expectedFactorAfterSplit = performAssertions(dateAfterSplit);
|
||||
Assert.AreNotEqual(expectedFactorAtSplit, expectedFactorAfterSplit);
|
||||
|
||||
if (factorFileDate == factorFileEntries.Last())
|
||||
{
|
||||
// prices should have been adjusted to the end date prices, instead of the latest factor file entry (today),
|
||||
// So the last factor should be 1.
|
||||
Assert.AreEqual(1m, config.PriceScaleFactor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static SubscriptionDataConfig GetConfig(Symbol symbol, Resolution resolution)
|
||||
{
|
||||
return new SubscriptionDataConfig(typeof(TradeBar),
|
||||
symbol,
|
||||
resolution,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
}
|
||||
|
||||
private class RawDataEnumerator : IEnumerator<BaseData>
|
||||
{
|
||||
public bool MoveNextReturnValue { get; set; }
|
||||
public BaseData CurrentValue { get; set; }
|
||||
|
||||
public BaseData Current => CurrentValue;
|
||||
|
||||
object IEnumerator.Current => CurrentValue;
|
||||
|
||||
public RawDataEnumerator()
|
||||
{
|
||||
MoveNextReturnValue = true;
|
||||
}
|
||||
public bool MoveNext()
|
||||
{
|
||||
return MoveNextReturnValue;
|
||||
}
|
||||
|
||||
public void Reset()
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class QuoteBarFillForwardEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void FillsForwardBidAskBars()
|
||||
{
|
||||
var bar1 = new QuoteBar
|
||||
{
|
||||
Bid = new Bar(3m, 4m, 1m, 2m),
|
||||
Ask = new Bar(3.1m, 4.1m, 1.1m, 2.1m),
|
||||
};
|
||||
|
||||
var bar2 = new QuoteBar
|
||||
{
|
||||
Bid = null,
|
||||
Ask = null,
|
||||
};
|
||||
|
||||
var data = new[] { bar1, bar2 }.ToList();
|
||||
var enumerator = data.GetEnumerator();
|
||||
|
||||
var fillForwardEnumerator = new QuoteBarFillForwardEnumerator(enumerator);
|
||||
|
||||
// 9:31
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
var quoteBar1 = (QuoteBar)fillForwardEnumerator.Current;
|
||||
Assert.AreSame(bar1.Bid, quoteBar1.Bid);
|
||||
Assert.AreSame(bar1.Ask, quoteBar1.Ask);
|
||||
|
||||
// 9:32
|
||||
Assert.IsTrue(fillForwardEnumerator.MoveNext());
|
||||
var quoteBar2 = (QuoteBar)fillForwardEnumerator.Current;
|
||||
Assert.AreSame(quoteBar1.Bid, quoteBar2.Bid);
|
||||
Assert.AreSame(quoteBar1.Ask, quoteBar2.Ask);
|
||||
|
||||
fillForwardEnumerator.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class RateLimitEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void LimitsBasedOnTimeBetweenCalls()
|
||||
{
|
||||
var currentTime = new DateTime(2015, 10, 10, 13, 6, 0);
|
||||
var timeProvider = new ManualTimeProvider(currentTime, TimeZones.Utc);
|
||||
var data = Enumerable.Range(0, 100).Select(x => new Tick {Symbol = CreateSymbol(x)}).GetEnumerator();
|
||||
var rateLimit = new RateLimitEnumerator<BaseData>(data, timeProvider, Time.OneSecond);
|
||||
|
||||
Assert.IsTrue(rateLimit.MoveNext());
|
||||
|
||||
while (rateLimit.MoveNext() && rateLimit.Current == null)
|
||||
{
|
||||
timeProvider.AdvanceSeconds(0.1);
|
||||
}
|
||||
|
||||
var delta = (timeProvider.GetUtcNow() - currentTime).TotalSeconds;
|
||||
|
||||
Assert.AreEqual(1, delta);
|
||||
|
||||
Assert.AreEqual("1", data.Current.Symbol.Value);
|
||||
|
||||
rateLimit.Dispose();
|
||||
}
|
||||
|
||||
private static Symbol CreateSymbol(int x)
|
||||
{
|
||||
return new Symbol(
|
||||
SecurityIdentifier.GenerateBase(null, x.ToString(CultureInfo.InvariantCulture), Market.USA),
|
||||
x.ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using Moq;
|
||||
using System.IO;
|
||||
using NUnit.Framework;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class RefreshEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void StaleFileHandleException()
|
||||
{
|
||||
var fakeEnumerator = new Mock<IEnumerator<int?>>();
|
||||
fakeEnumerator.Setup(e => e.MoveNext()).Throws(new IOException("stale file handle"));
|
||||
fakeEnumerator.Setup(e => e.Dispose()).Verifiable();
|
||||
var refresher = new RefreshEnumerator<int?>(() => fakeEnumerator.Object);
|
||||
|
||||
// does not throw exception but disposes of enumerator
|
||||
refresher.MoveNext();
|
||||
|
||||
fakeEnumerator.Verify(enumerator => enumerator.Dispose(), Times.Once);
|
||||
|
||||
refresher.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RefreshesEnumeratorOnFirstMoveNext()
|
||||
{
|
||||
var refreshed = false;
|
||||
var refresher = new RefreshEnumerator<int?>(() =>
|
||||
{
|
||||
refreshed = true;
|
||||
return new List<int?>().GetEnumerator();
|
||||
});
|
||||
|
||||
refresher.MoveNext();
|
||||
Assert.IsTrue(refreshed);
|
||||
|
||||
refresher.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MoveNextReturnsTrueWhenUnderlyingEnumeratorReturnsFalse()
|
||||
{
|
||||
var refresher = new RefreshEnumerator<int?>(() => new List<int?>().GetEnumerator());
|
||||
Assert.IsTrue(refresher.MoveNext());
|
||||
|
||||
refresher.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CurrentIsDefault_T_WhenUnderlyingEnumeratorReturnsFalse()
|
||||
{
|
||||
var refresher = new RefreshEnumerator<int?>(() => new List<int?>().GetEnumerator());
|
||||
refresher.MoveNext();
|
||||
Assert.AreEqual(default(int?), refresher.Current);
|
||||
|
||||
refresher.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UnderlyingEnumeratorDisposed_WhenUnderlyingEnumeratorReturnsFalse()
|
||||
{
|
||||
var fakeEnumerator = new Mock<IEnumerator<int?>>();
|
||||
fakeEnumerator.Setup(e => e.MoveNext()).Returns(false);
|
||||
fakeEnumerator.Setup(e => e.Dispose()).Verifiable();
|
||||
var refresher = new RefreshEnumerator<int?>(() => fakeEnumerator.Object);
|
||||
refresher.MoveNext();
|
||||
|
||||
fakeEnumerator.Verify(enumerator => enumerator.Dispose(), Times.Once);
|
||||
|
||||
refresher.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DisposeCallsUnderlyingDispose()
|
||||
{
|
||||
var fakeEnumerator = new Mock<IEnumerator<int?>>();
|
||||
fakeEnumerator.Setup(e => e.MoveNext()).Returns(true);
|
||||
fakeEnumerator.Setup(e => e.Dispose()).Verifiable();
|
||||
var refresher = new RefreshEnumerator<int?>(() => fakeEnumerator.Object);
|
||||
refresher.MoveNext();
|
||||
refresher.Dispose();
|
||||
|
||||
fakeEnumerator.Verify(enumerator => enumerator.Dispose(), Times.Once);
|
||||
|
||||
refresher.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ResetCallsUnderlyingReset()
|
||||
{
|
||||
var fakeEnumerator = new Mock<IEnumerator<int?>>();
|
||||
fakeEnumerator.Setup(e => e.MoveNext()).Returns(true);
|
||||
fakeEnumerator.Setup(e => e.Reset()).Verifiable();
|
||||
var refresher = new RefreshEnumerator<int?>(() => fakeEnumerator.Object);
|
||||
refresher.MoveNext();
|
||||
refresher.Reset();
|
||||
|
||||
fakeEnumerator.Verify(enumerator => enumerator.Reset(), Times.Once);
|
||||
|
||||
refresher.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RefreshesAfterMoveNextReturnsFalse()
|
||||
{
|
||||
var refreshCount = 0;
|
||||
var list = new List<int?> {1, 2};
|
||||
var refresher = new RefreshEnumerator<int?>(() =>
|
||||
{
|
||||
refreshCount++;
|
||||
return list.GetEnumerator();
|
||||
});
|
||||
|
||||
Assert.IsTrue(refresher.MoveNext());
|
||||
Assert.AreEqual(1, refreshCount);
|
||||
Assert.AreEqual(1, refresher.Current);
|
||||
|
||||
Assert.IsTrue(refresher.MoveNext());
|
||||
Assert.AreEqual(1, refreshCount);
|
||||
Assert.AreEqual(2, refresher.Current);
|
||||
|
||||
Assert.IsTrue(refresher.MoveNext());
|
||||
Assert.AreEqual(1, refreshCount);
|
||||
Assert.AreEqual(default(int?), refresher.Current);
|
||||
|
||||
Assert.IsTrue(refresher.MoveNext());
|
||||
Assert.AreEqual(2, refreshCount);
|
||||
Assert.AreEqual(1, refresher.Current);
|
||||
|
||||
Assert.IsTrue(refresher.MoveNext());
|
||||
Assert.AreEqual(2, refreshCount);
|
||||
Assert.AreEqual(2, refresher.Current);
|
||||
|
||||
refresher.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using NodaTime;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data.Consolidators;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.All)]
|
||||
public class ScannableEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void PassesTicksStraightThrough()
|
||||
{
|
||||
var currentTime = new DateTime(2000, 01, 01);
|
||||
using var identityDataConsolidator = new IdentityDataConsolidator<Tick>();
|
||||
var enumerator = new ScannableEnumerator<Tick>(
|
||||
identityDataConsolidator,
|
||||
DateTimeZone.ForOffset(Offset.FromHours(-5)),
|
||||
new ManualTimeProvider(currentTime),
|
||||
(s, e) => { }
|
||||
);
|
||||
|
||||
// returns true even if no data present until stop is called
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) { Quantity = 10 };
|
||||
enumerator.Update(tick1);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(tick1, enumerator.Current);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick2 = new Tick(currentTime, Symbols.SPY, 199.56m, 199.21m, 200.02m) { Quantity = 5 };
|
||||
enumerator.Update(tick2);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(tick2, enumerator.Current);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NewDataAvailableShouldFire()
|
||||
{
|
||||
var currentTime = new DateTime(2000, 01, 01);
|
||||
var available = false;
|
||||
using var identityDataConsolidator = new IdentityDataConsolidator<Tick>();
|
||||
var enumerator = new ScannableEnumerator<Tick>(
|
||||
identityDataConsolidator,
|
||||
DateTimeZone.ForOffset(Offset.FromHours(-5)),
|
||||
new ManualTimeProvider(currentTime),
|
||||
(s, e) => { available = true; }
|
||||
);
|
||||
|
||||
// returns true even if no data present until stop is called
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
Assert.IsFalse(available);
|
||||
|
||||
var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) { Quantity = 10 };
|
||||
enumerator.Update(tick1);
|
||||
Assert.IsTrue(available);
|
||||
|
||||
enumerator.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AggregatesNewQuoteBarProperly()
|
||||
{
|
||||
var reference = DateTime.Today;
|
||||
|
||||
using var tickQuoteBarConsolidator = new TickQuoteBarConsolidator(4);
|
||||
using var enumerator = new ScannableEnumerator<Data.BaseData>(
|
||||
tickQuoteBarConsolidator,
|
||||
DateTimeZone.ForOffset(Offset.FromHours(-5)),
|
||||
new ManualTimeProvider(reference),
|
||||
(s, e) => { }
|
||||
);
|
||||
|
||||
var tick1 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference,
|
||||
BidPrice = 10,
|
||||
BidSize = 20,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
enumerator.Update(tick1);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick2 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.AddMinutes(1),
|
||||
AskPrice = 20,
|
||||
AskSize = 10,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
|
||||
var badTick = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.AddMinutes(1),
|
||||
AskPrice = 25,
|
||||
AskSize = 100,
|
||||
BidPrice = -100,
|
||||
BidSize = 2,
|
||||
Value = 50,
|
||||
Quantity = 1234,
|
||||
TickType = TickType.Trade
|
||||
};
|
||||
enumerator.Update(badTick);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
enumerator.Update(tick2);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick3 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.AddMinutes(2),
|
||||
BidPrice = 12,
|
||||
BidSize = 50,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
enumerator.Update(tick3);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick4 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.AddMinutes(3),
|
||||
AskPrice = 17,
|
||||
AskSize = 15,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
enumerator.Update(tick4);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
|
||||
QuoteBar quoteBar = enumerator.Current as QuoteBar;
|
||||
Assert.IsNotNull(quoteBar);
|
||||
|
||||
Assert.AreEqual(Symbols.SPY, quoteBar.Symbol);
|
||||
Assert.AreEqual(tick1.Time, quoteBar.Time);
|
||||
Assert.AreEqual(tick4.EndTime, quoteBar.EndTime);
|
||||
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open);
|
||||
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Low);
|
||||
Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.High);
|
||||
Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.Close);
|
||||
Assert.AreEqual(tick3.BidSize, quoteBar.LastBidSize);
|
||||
|
||||
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open);
|
||||
Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Low);
|
||||
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.High);
|
||||
Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Close);
|
||||
Assert.AreEqual(tick4.AskSize, quoteBar.LastAskSize);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ForceScanQuoteBar()
|
||||
{
|
||||
var reference = new DateTime(2020, 2, 2, 1, 0, 0);
|
||||
var timeProvider = new ManualTimeProvider(reference);
|
||||
var dateTimeZone = DateTimeZone.ForOffset(Offset.FromHours(-5));
|
||||
using var tickQuoteBarConsolidator = new TickQuoteBarConsolidator(TimeSpan.FromMinutes(1));
|
||||
using var enumerator = new ScannableEnumerator<Data.BaseData>(
|
||||
tickQuoteBarConsolidator,
|
||||
dateTimeZone,
|
||||
timeProvider,
|
||||
(s, e) => { }
|
||||
);
|
||||
|
||||
var tick1 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.ConvertFromUtc(dateTimeZone),
|
||||
BidPrice = 10,
|
||||
BidSize = 20,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
enumerator.Update(tick1);
|
||||
|
||||
var tick2 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.AddSeconds(1).ConvertFromUtc(dateTimeZone),
|
||||
AskPrice = 20,
|
||||
AskSize = 10,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
|
||||
enumerator.Update(tick2);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick3 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.AddSeconds(2).ConvertFromUtc(dateTimeZone),
|
||||
BidPrice = 12,
|
||||
BidSize = 50,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
enumerator.Update(tick3);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick4 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.AddSeconds(3).ConvertFromUtc(dateTimeZone),
|
||||
AskPrice = 17,
|
||||
AskSize = 15,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
enumerator.Update(tick4);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
timeProvider.AdvanceSeconds(120);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
QuoteBar quoteBar = enumerator.Current as QuoteBar;
|
||||
Assert.IsNotNull(quoteBar);
|
||||
|
||||
Assert.AreEqual(Symbols.SPY, quoteBar.Symbol);
|
||||
Assert.AreEqual(tick1.Time, quoteBar.Time);
|
||||
Assert.AreNotEqual(tick4.EndTime, quoteBar.EndTime);
|
||||
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open);
|
||||
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Low);
|
||||
Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.High);
|
||||
Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.Close);
|
||||
Assert.AreEqual(tick3.BidSize, quoteBar.LastBidSize);
|
||||
|
||||
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open);
|
||||
Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Low);
|
||||
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.High);
|
||||
Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Close);
|
||||
Assert.AreEqual(tick4.AskSize, quoteBar.LastAskSize);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MoveNextScanQuoteBar()
|
||||
{
|
||||
var offset = Offset.FromHours(-5);
|
||||
var timeZone = DateTimeZone.ForOffset(offset);
|
||||
var utc = new DateTimeOffset(DateTime.Today);
|
||||
var reference = utc.ToOffset(offset.ToTimeSpan());
|
||||
var timeProvider = new ManualTimeProvider(reference.DateTime, timeZone);
|
||||
|
||||
using var tickQuoteBarConsolidator = new TickQuoteBarConsolidator(TimeSpan.FromMinutes(1));
|
||||
using var enumerator = new ScannableEnumerator<Data.BaseData>(
|
||||
tickQuoteBarConsolidator,
|
||||
timeZone,
|
||||
timeProvider,
|
||||
(s, e) => { }
|
||||
);
|
||||
|
||||
var tick1 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.DateTime,
|
||||
BidPrice = 10,
|
||||
BidSize = 20,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
enumerator.Update(tick1);
|
||||
|
||||
var tick2 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.DateTime.AddSeconds(1),
|
||||
AskPrice = 20,
|
||||
AskSize = 10,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
|
||||
enumerator.Update(tick2);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick3 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.DateTime.AddSeconds(2),
|
||||
BidPrice = 12,
|
||||
BidSize = 50,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
enumerator.Update(tick3);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
var tick4 = new Tick
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = reference.DateTime.AddSeconds(3),
|
||||
AskPrice = 17,
|
||||
AskSize = 15,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
enumerator.Update(tick4);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
timeProvider.SetCurrentTime(reference.DateTime.AddMinutes(2));
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
QuoteBar quoteBar = enumerator.Current as QuoteBar;
|
||||
Assert.IsNotNull(quoteBar);
|
||||
|
||||
Assert.AreEqual(Symbols.SPY, quoteBar.Symbol);
|
||||
Assert.AreEqual(tick1.Time, quoteBar.Time);
|
||||
Assert.AreNotEqual(tick4.EndTime, quoteBar.EndTime);
|
||||
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open);
|
||||
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Low);
|
||||
Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.High);
|
||||
Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.Close);
|
||||
Assert.AreEqual(tick3.BidSize, quoteBar.LastBidSize);
|
||||
|
||||
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open);
|
||||
Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Low);
|
||||
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.High);
|
||||
Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Close);
|
||||
Assert.AreEqual(tick4.AskSize, quoteBar.LastAskSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.All)]
|
||||
public class ScheduledEnumeratorTests
|
||||
{
|
||||
private readonly DateTime _referenceTime = new DateTime(2019, 1, 1);
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void RespectsPredicateTimeProvider(bool newDataArrivedInTime)
|
||||
{
|
||||
var scheduledDate = _referenceTime.AddDays(1);
|
||||
using var underlyingEnumerator = new TestEnumerator
|
||||
{
|
||||
MoveNextReturn = true,
|
||||
MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
|
||||
{
|
||||
new Tick(scheduledDate.AddDays(-1), Symbols.SPY, 1, 1)
|
||||
})
|
||||
};
|
||||
var timeProvider = new ManualTimeProvider(_referenceTime);
|
||||
|
||||
using var enumerator = new ScheduledEnumerator(
|
||||
underlyingEnumerator,
|
||||
new List<DateTime> { scheduledDate },
|
||||
new PredicateTimeProvider(timeProvider, (currentDateTime) => {
|
||||
// will only let time advance after it's passed the 7/8 hour frontier
|
||||
return currentDateTime.TimeOfDay > TimeSpan.FromMinutes(7 * 60 + DateTime.UtcNow.Second);
|
||||
}),
|
||||
TimeZones.Utc,
|
||||
DateTime.MinValue);
|
||||
|
||||
// still null since frontier is still behind schedule
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
timeProvider.SetCurrentTimeUtc(scheduledDate);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
timeProvider.SetCurrentTimeUtc(scheduledDate.AddHours(2));
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
if (newDataArrivedInTime)
|
||||
{
|
||||
// New data comes in!
|
||||
underlyingEnumerator.MoveNextNewValues.Enqueue(new Tick(scheduledDate, Symbols.SPY, 10, 10));
|
||||
}
|
||||
|
||||
timeProvider.SetCurrentTimeUtc(scheduledDate.AddHours(8));
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(scheduledDate, enumerator.Current.Time);
|
||||
Assert.AreEqual(newDataArrivedInTime ? 10 : 1, (enumerator.Current as Tick).BidPrice);
|
||||
|
||||
// schedule ended so enumerator will end too
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ScheduleSkipsOldDates()
|
||||
{
|
||||
using var testEnumerator = new TestEnumerator();
|
||||
using var enumerator = new ScheduledEnumerator(
|
||||
testEnumerator,
|
||||
new List<DateTime> { _referenceTime },
|
||||
new ManualTimeProvider(_referenceTime),
|
||||
TimeZones.Utc,
|
||||
_referenceTime.AddDays(1));
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmptyScheduleThrowsNoException()
|
||||
{
|
||||
ScheduledEnumerator enumerator = null;
|
||||
Assert.DoesNotThrow(() => enumerator = new ScheduledEnumerator(
|
||||
new TestEnumerator(),
|
||||
new List<DateTime>(),
|
||||
new ManualTimeProvider(_referenceTime),
|
||||
TimeZones.Utc,
|
||||
DateTime.MinValue));
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReturnsTrueEvenIfUnderlyingIsNullButReturnsTrue()
|
||||
{
|
||||
using var underlyingEnumerator = new TestEnumerator { MoveNextReturn = true };
|
||||
var timeProvider = new ManualTimeProvider(_referenceTime);
|
||||
|
||||
using var enumerator = new ScheduledEnumerator(
|
||||
underlyingEnumerator,
|
||||
new List<DateTime> { _referenceTime.AddDays(1) },
|
||||
timeProvider,
|
||||
TimeZones.Utc,
|
||||
DateTime.MinValue);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReturnsFalseWhenUnderlyingReturnsFalse()
|
||||
{
|
||||
using var underlyingEnumerator = new TestEnumerator { MoveNextReturn = false };
|
||||
var timeProvider = new ManualTimeProvider(_referenceTime);
|
||||
|
||||
using var enumerator = new ScheduledEnumerator(
|
||||
underlyingEnumerator,
|
||||
new List<DateTime> { _referenceTime.AddDays(1) },
|
||||
timeProvider,
|
||||
TimeZones.Utc,
|
||||
DateTime.MinValue);
|
||||
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ForwardsDataToFitSchedule()
|
||||
{
|
||||
var scheduledDate = _referenceTime.AddDays(1);
|
||||
using var underlyingEnumerator = new TestEnumerator
|
||||
{
|
||||
MoveNextReturn = true,
|
||||
MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
|
||||
{
|
||||
new Tick(scheduledDate, Symbols.SPY, 1, 1),
|
||||
// way in the future compared with the schedule
|
||||
new Tick(scheduledDate.AddYears(1), Symbols.SPY, 10, 10)
|
||||
})
|
||||
};
|
||||
var timeProvider = new ManualTimeProvider(_referenceTime);
|
||||
|
||||
using var enumerator = new ScheduledEnumerator(
|
||||
underlyingEnumerator,
|
||||
new List<DateTime> { scheduledDate, scheduledDate.AddDays(1) },
|
||||
timeProvider,
|
||||
TimeZones.Utc,
|
||||
DateTime.MinValue);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
// still null since frontier is still behind schedule
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
timeProvider.SetCurrentTimeUtc(scheduledDate);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(scheduledDate, enumerator.Current.Time);
|
||||
Assert.AreEqual(1, (enumerator.Current as Tick).BidPrice);
|
||||
|
||||
// it will forward previous available value to fit the schedule
|
||||
timeProvider.SetCurrentTimeUtc(scheduledDate.AddDays(1));
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(scheduledDate.AddDays(1), enumerator.Current.Time);
|
||||
Assert.AreEqual(1, (enumerator.Current as Tick).BidPrice);
|
||||
|
||||
// schedule ended so enumerator will end too
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UpdatesCurrentBasedOnSchedule()
|
||||
{
|
||||
var scheduledDate = _referenceTime.AddDays(1);
|
||||
using var underlyingEnumerator = new TestEnumerator
|
||||
{
|
||||
MoveNextReturn = true,
|
||||
MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
|
||||
{
|
||||
new Tick(scheduledDate, Symbols.SPY, 1, 1)
|
||||
})
|
||||
};
|
||||
var timeProvider = new ManualTimeProvider(_referenceTime);
|
||||
|
||||
using var enumerator = new ScheduledEnumerator(
|
||||
underlyingEnumerator,
|
||||
new List<DateTime> { scheduledDate },
|
||||
timeProvider,
|
||||
TimeZones.Utc,
|
||||
DateTime.MinValue);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
// still null since frontier is still behind schedule
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
timeProvider.SetCurrentTimeUtc(scheduledDate);
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(scheduledDate, enumerator.Current.Time);
|
||||
Assert.AreEqual(1, (enumerator.Current as Tick).BidPrice);
|
||||
|
||||
// schedule ended so enumerator will end too
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WillUseLatestDataPoint()
|
||||
{
|
||||
using var underlyingEnumerator = new TestEnumerator
|
||||
{
|
||||
MoveNextReturn = true,
|
||||
MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
|
||||
{
|
||||
new Tick(new DateTime(2019, 1, 15), Symbols.SPY, 1, 1),
|
||||
new Tick(new DateTime(2019, 1, 20), Symbols.SPY, 2, 1),
|
||||
new Tick(new DateTime(2019, 1, 25), Symbols.SPY, 3, 1)
|
||||
})
|
||||
};
|
||||
var timeProvider = new ManualTimeProvider(_referenceTime);
|
||||
|
||||
using var enumerator = new ScheduledEnumerator(
|
||||
underlyingEnumerator,
|
||||
new List<DateTime> { new DateTime(2019, 2, 1) },
|
||||
timeProvider,
|
||||
TimeZones.Utc,
|
||||
DateTime.MinValue);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
// still null since frontier is still behind schedule
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
// frontier is now a month after the scheduled time!
|
||||
timeProvider.SetCurrentTimeUtc(new DateTime(2019, 3, 1));
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
// it uses the last available data point in the enumerator
|
||||
Assert.AreEqual(new DateTime(2019, 2, 1), enumerator.Current.Time);
|
||||
Assert.AreEqual(3, (enumerator.Current as Tick).BidPrice);
|
||||
|
||||
Assert.IsNull(underlyingEnumerator.Current);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WillUseLatestDataPointOnlyIfBeforeOrAtSchedule()
|
||||
{
|
||||
using var underlyingEnumerator = new TestEnumerator
|
||||
{
|
||||
MoveNextReturn = true,
|
||||
MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
|
||||
{
|
||||
new Tick(new DateTime(2019, 1, 20), Symbols.SPY, 2, 1),
|
||||
new Tick(new DateTime(2019, 1, 25), Symbols.SPY, 3, 1),
|
||||
// this guys is in 2020
|
||||
new Tick(new DateTime(2020, 1, 1), Symbols.SPY, 4, 1)
|
||||
})
|
||||
};
|
||||
var timeProvider = new ManualTimeProvider(_referenceTime);
|
||||
|
||||
using var enumerator = new ScheduledEnumerator(
|
||||
underlyingEnumerator,
|
||||
new List<DateTime> { new DateTime(2019, 2, 1), new DateTime(2020, 2, 1) },
|
||||
timeProvider,
|
||||
TimeZones.Utc,
|
||||
DateTime.MinValue);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
// still null since frontier is still behind schedule
|
||||
Assert.IsNull(enumerator.Current);
|
||||
|
||||
// frontier is now a month after the scheduled time!
|
||||
timeProvider.SetCurrentTimeUtc(new DateTime(2019, 3, 1));
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
// it uses the last available data point in the enumerator that is before the schedule
|
||||
Assert.AreEqual(new DateTime(2019, 2, 1), enumerator.Current.Time);
|
||||
Assert.AreEqual(3, (enumerator.Current as Tick).BidPrice);
|
||||
|
||||
// the underlying enumerator hold the next data point
|
||||
Assert.AreEqual(new DateTime(2020, 1, 1), underlyingEnumerator.Current.Time);
|
||||
|
||||
// now lets test fetching the last data point
|
||||
timeProvider.SetCurrentTimeUtc(new DateTime(2021, 3, 1));
|
||||
|
||||
// the underlying will end but should still emit the data point it has
|
||||
underlyingEnumerator.MoveNextReturn = false;
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(new DateTime(2020, 2, 1), enumerator.Current.Time);
|
||||
Assert.AreEqual(4, (enumerator.Current as Tick).BidPrice);
|
||||
|
||||
Assert.IsNull(underlyingEnumerator.Current);
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NoTimeProvider()
|
||||
{
|
||||
using var underlyingEnumerator = new TestEnumerator
|
||||
{
|
||||
MoveNextReturn = true,
|
||||
MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
|
||||
{
|
||||
new Tick(new DateTime(2019, 1, 20), Symbols.SPY, 2, 1),
|
||||
new Tick(new DateTime(2019, 1, 25), Symbols.SPY, 3, 1),
|
||||
|
||||
new Tick(new DateTime(2020, 1, 1), Symbols.SPY, 4, 1)
|
||||
})
|
||||
};
|
||||
using var enumerator = new ScheduledEnumerator(
|
||||
underlyingEnumerator,
|
||||
new List<DateTime> { new DateTime(2019, 2, 1), new DateTime(2020, 2, 1) },
|
||||
null,
|
||||
TimeZones.Utc,
|
||||
DateTime.MinValue);
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
// it uses the last available data point in the enumerator that is before the schedule
|
||||
Assert.AreEqual(new DateTime(2019, 2, 1), enumerator.Current.Time);
|
||||
Assert.AreEqual(3, (enumerator.Current as Tick).BidPrice);
|
||||
|
||||
// the underlying enumerator hold the next data point
|
||||
Assert.AreEqual(new DateTime(2020, 1, 1), underlyingEnumerator.Current.Time);
|
||||
|
||||
// the underlying will end but should still emit the data point it has
|
||||
underlyingEnumerator.MoveNextReturn = false;
|
||||
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.AreEqual(new DateTime(2020, 2, 1), enumerator.Current.Time);
|
||||
Assert.AreEqual(4, (enumerator.Current as Tick).BidPrice);
|
||||
|
||||
Assert.IsNull(underlyingEnumerator.Current);
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
Assert.IsNull(enumerator.Current);
|
||||
}
|
||||
|
||||
private class TestEnumerator : IEnumerator<BaseData>
|
||||
{
|
||||
public Queue<BaseData> MoveNextNewValues { get; set; }
|
||||
public BaseData Current { get; private set; }
|
||||
|
||||
object IEnumerator.Current => Current;
|
||||
|
||||
public bool MoveNextReturn { get; set; }
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
if (MoveNextNewValues != null && MoveNextNewValues.Count > 0)
|
||||
{
|
||||
Current = MoveNextNewValues.Dequeue();
|
||||
}
|
||||
else
|
||||
{
|
||||
Current = null;
|
||||
}
|
||||
return MoveNextReturn;
|
||||
}
|
||||
|
||||
public void Reset()
|
||||
{}
|
||||
public void Dispose()
|
||||
{}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using QuantConnect.Securities;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class SubscriptionDataEnumeratorTests
|
||||
{
|
||||
|
||||
[TestCase(typeof(TradeBar), true)]
|
||||
[TestCase(typeof(OpenInterest), false)]
|
||||
[TestCase(typeof(QuoteBar), false)]
|
||||
public void EnumeratorEmitsAuxData(Type typeOfConfig, bool shouldReceiveAuxData)
|
||||
{
|
||||
var config = CreateConfig(Resolution.Hour, typeOfConfig);
|
||||
var security = GetSecurity(config);
|
||||
var time = new DateTime(2010, 1, 1);
|
||||
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, time, time.AddDays(1));
|
||||
|
||||
|
||||
// Make a aux data stream; for this testing case we will just use delisting data points
|
||||
var totalPoints = 8;
|
||||
var stream = Enumerable.Range(0, totalPoints).Select(x => new Delisting { Time = time.AddHours(x) }).GetEnumerator();
|
||||
using var enumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, tzOffsetProvider, stream, false, false);
|
||||
|
||||
// Test our SubscriptionDataEnumerator to see if it emits the aux data
|
||||
int dataReceivedCount = 0;
|
||||
while (enumerator.MoveNext())
|
||||
{
|
||||
dataReceivedCount++;
|
||||
if (enumerator.Current != null && enumerator.Current.Data.DataType == MarketDataType.Auxiliary)
|
||||
{
|
||||
Assert.IsTrue(shouldReceiveAuxData);
|
||||
}
|
||||
}
|
||||
|
||||
// If it should receive aux data it should have emitted all points
|
||||
// otherwise none should have been emitted
|
||||
if (shouldReceiveAuxData)
|
||||
{
|
||||
Assert.AreEqual(totalPoints, dataReceivedCount);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.AreEqual(0, dataReceivedCount);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static Security GetSecurity(SubscriptionDataConfig config)
|
||||
{
|
||||
return new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
|
||||
config,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
}
|
||||
|
||||
private static SubscriptionDataConfig CreateConfig(Resolution resolution, Type type)
|
||||
{
|
||||
return new SubscriptionDataConfig(type, Symbols.SPY, resolution, TimeZones.NewYork, TimeZones.NewYork, true, true, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class SynchronizingBaseDataEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void SynchronizesData()
|
||||
{
|
||||
var time = new DateTime(2016, 03, 03, 12, 05, 00);
|
||||
var stream1 = Enumerable.Range(0, 10).Select(x => new Tick {Time = time.AddSeconds(x * 1)}).GetEnumerator();
|
||||
var stream2 = Enumerable.Range(0, 5).Select(x => new Tick {Time = time.AddSeconds(x * 2)}).GetEnumerator();
|
||||
var stream3 = Enumerable.Range(0, 20).Select(x => new Tick {Time = time.AddSeconds(x * 0.5)}).GetEnumerator();
|
||||
|
||||
var previous = DateTime.MinValue;
|
||||
var synchronizer = new SynchronizingBaseDataEnumerator(stream1, stream2, stream3);
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous));
|
||||
previous = synchronizer.Current.EndTime;
|
||||
}
|
||||
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveEnumeratorsReturningTrueWithCurrentNull()
|
||||
{
|
||||
var time = new DateTime(2016, 03, 03, 12, 05, 00);
|
||||
var stream1 = Enumerable.Range(0, 20)
|
||||
// return null except the last value and check if its emitted
|
||||
.Select(x => x == 19 ? new Tick {Time = time.AddSeconds(x * 100), Quantity = 998877} : null
|
||||
).GetEnumerator();
|
||||
var stream2 = Enumerable.Range(0, 5).Select(x => new Tick { Time = time.AddSeconds(x * 2) }).GetEnumerator();
|
||||
var stream3 = Enumerable.Range(0, 20).Select(x => new Tick { Time = time.AddSeconds(x * 0.5) }).GetEnumerator();
|
||||
|
||||
var previous = new Tick { Time = DateTime.MinValue };
|
||||
var synchronizer = new SynchronizingBaseDataEnumerator(stream1, stream2, stream3);
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous.EndTime));
|
||||
previous = synchronizer.Current as Tick;
|
||||
}
|
||||
Assert.AreEqual(998877, previous.Quantity);
|
||||
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WillRemoveEnumeratorsReturningFalse()
|
||||
{
|
||||
var time = new DateTime(2016, 03, 03, 12, 05, 00);
|
||||
var stream1 = new TestEnumerator { MoveNextReturnValue = false };
|
||||
var stream2 = Enumerable.Range(0, 10).Select(x => new Tick { Time = time.AddSeconds(x * 2) }).GetEnumerator();
|
||||
var synchronizer = new SynchronizingBaseDataEnumerator(stream1, stream2);
|
||||
var emitted = false;
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
emitted = true;
|
||||
}
|
||||
Assert.IsTrue(emitted);
|
||||
Assert.IsTrue(stream1.MoveNextWasCalled);
|
||||
Assert.AreEqual(1, stream1.MoveNextCallCount);
|
||||
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WillStopIfAllEnumeratorsCurrentIsNullAndReturningTrue()
|
||||
{
|
||||
var stream1 = new TestEnumerator { MoveNextReturnValue = true };
|
||||
var synchronizer = new SynchronizingBaseDataEnumerator(stream1);
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
Assert.Fail();
|
||||
}
|
||||
Assert.IsTrue(stream1.MoveNextWasCalled);
|
||||
Assert.Pass();
|
||||
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WillStopIfAllEnumeratorsCurrentIsNullAndReturningFalse()
|
||||
{
|
||||
var stream1 = new TestEnumerator { MoveNextReturnValue = false };
|
||||
var synchronizer = new SynchronizingBaseDataEnumerator(stream1);
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
Assert.Fail();
|
||||
}
|
||||
Assert.IsTrue(stream1.MoveNextWasCalled);
|
||||
Assert.Pass();
|
||||
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
private class TestEnumerator : IEnumerator<BaseData>
|
||||
{
|
||||
public int MoveNextCallCount { get; set; }
|
||||
public bool MoveNextReturnValue { get; set; }
|
||||
public bool MoveNextWasCalled { get; set; }
|
||||
|
||||
public TestEnumerator()
|
||||
{
|
||||
MoveNextWasCalled = false;
|
||||
}
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
MoveNextCallCount++;
|
||||
MoveNextWasCalled = true;
|
||||
return MoveNextReturnValue;
|
||||
}
|
||||
|
||||
public BaseData Current { get; }
|
||||
|
||||
object IEnumerator.Current => Current;
|
||||
public void Dispose() { }
|
||||
public void Reset() { }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
|
||||
{
|
||||
[TestFixture]
|
||||
public class SynchronizingSliceEnumeratorTests
|
||||
{
|
||||
[Test]
|
||||
public void SynchronizesData()
|
||||
{
|
||||
var time = new DateTime(2016, 03, 03, 12, 05, 00);
|
||||
var stream1 = Enumerable.Range(0, 10).Select(x => new Slice(time.AddSeconds(x * 1), new List<BaseData>(), utcTime: time.AddSeconds(x * 1))).GetEnumerator();
|
||||
var stream2 = Enumerable.Range(0, 5).Select(x => new Slice(time.AddSeconds(x * 2), new List<BaseData>(), utcTime: time.AddSeconds(x * 2))).GetEnumerator();
|
||||
var stream3 = Enumerable.Range(0, 20).Select(x => new Slice(time.AddSeconds(x * 0.5), new List<BaseData>(), utcTime: time.AddSeconds(x * 0.5))).GetEnumerator();
|
||||
|
||||
var previous = DateTime.MinValue;
|
||||
var synchronizer = new SynchronizingSliceEnumerator(stream1, stream2, stream3);
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
Assert.That(synchronizer.Current.UtcTime, Is.GreaterThanOrEqualTo(previous));
|
||||
previous = synchronizer.Current.UtcTime;
|
||||
}
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveEnumeratorsReturningTrueWithCurrentNull()
|
||||
{
|
||||
var time = new DateTime(2016, 03, 03, 12, 05, 00);
|
||||
var tradeBar1 = new TradeBar { Symbol = Symbols.SPY, Time = time };
|
||||
var tradeBar2 = new TradeBar { Symbol = Symbols.AAPL, Time = time, Open = 23 };
|
||||
var stream1 = Enumerable.Range(0, 20)
|
||||
// return null except the last value and check if its emitted
|
||||
.Select(x => x == 19 ? new Slice(time.AddSeconds(x * 1), new BaseData[] { tradeBar1, tradeBar2 }, utcTime: time.AddSeconds(x * 1)) : null
|
||||
).GetEnumerator();
|
||||
var stream2 = Enumerable.Range(0, 5).Select(x => new Slice(time.AddSeconds(x * 2), new List<BaseData>(), utcTime: time.AddSeconds(x * 2))).GetEnumerator();
|
||||
var stream3 = Enumerable.Range(0, 20).Select(x => new Slice(time.AddSeconds(x * 0.5), new List<BaseData>(), utcTime: time.AddSeconds(x * 0.5))).GetEnumerator();
|
||||
|
||||
var previous = new Slice(DateTime.MinValue, new List<BaseData>(), DateTime.MinValue);
|
||||
var synchronizer = new SynchronizingSliceEnumerator(stream1, stream2, stream3);
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
Assert.That(synchronizer.Current.UtcTime, Is.GreaterThanOrEqualTo(previous.UtcTime));
|
||||
previous = synchronizer.Current;
|
||||
}
|
||||
Assert.AreEqual(2, previous.Bars.Count);
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WillRemoveEnumeratorsReturningFalse()
|
||||
{
|
||||
var time = new DateTime(2016, 03, 03, 12, 05, 00);
|
||||
var stream1 = new TestEnumerator { MoveNextReturnValue = false };
|
||||
var stream2 = Enumerable.Range(0, 10).Select(x => new Slice(time.AddSeconds(x * 0.5), new List<BaseData>(), utcTime: time.AddSeconds(x * 0.5))).GetEnumerator();
|
||||
var synchronizer = new SynchronizingSliceEnumerator(stream1, stream2);
|
||||
var emitted = false;
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
emitted = true;
|
||||
}
|
||||
Assert.IsTrue(emitted);
|
||||
Assert.IsTrue(stream1.MoveNextWasCalled);
|
||||
Assert.AreEqual(1, stream1.MoveNextCallCount);
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WillStopIfAllEnumeratorsCurrentIsNullAndReturningTrue()
|
||||
{
|
||||
var stream1 = new TestEnumerator { MoveNextReturnValue = true };
|
||||
var synchronizer = new SynchronizingSliceEnumerator(stream1);
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
Assert.Fail();
|
||||
}
|
||||
Assert.IsTrue(stream1.MoveNextWasCalled);
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WillStopIfAllEnumeratorsCurrentIsNullAndReturningFalse()
|
||||
{
|
||||
var stream1 = new TestEnumerator { MoveNextReturnValue = false };
|
||||
var synchronizer = new SynchronizingSliceEnumerator(stream1);
|
||||
while (synchronizer.MoveNext())
|
||||
{
|
||||
Assert.Fail();
|
||||
}
|
||||
Assert.IsTrue(stream1.MoveNextWasCalled);
|
||||
synchronizer.Dispose();
|
||||
}
|
||||
|
||||
private class TestEnumerator : IEnumerator<Slice>
|
||||
{
|
||||
public int MoveNextCallCount { get; set; }
|
||||
public bool MoveNextReturnValue { get; set; }
|
||||
public bool MoveNextWasCalled { get; set; }
|
||||
|
||||
public TestEnumerator()
|
||||
{
|
||||
MoveNextWasCalled = false;
|
||||
}
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
MoveNextCallCount++;
|
||||
MoveNextWasCalled = true;
|
||||
return MoveNextReturnValue;
|
||||
}
|
||||
|
||||
public Slice Current { get; }
|
||||
|
||||
object IEnumerator.Current => Current;
|
||||
|
||||
public void Dispose()
|
||||
{ }
|
||||
|
||||
public void Reset()
|
||||
{ }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using NUnit.Framework;
|
||||
using System.Threading;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Logging;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Queues;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.All)]
|
||||
public class FakeDataQueueTests
|
||||
{
|
||||
[Test]
|
||||
public void GeneratesDataCorrectly()
|
||||
{
|
||||
using var aggregator = new AggregationManager();
|
||||
var algorithm = new AlgorithmStub();
|
||||
var dataQueue = new FakeDataQueue(aggregator, dataPointsPerSecondPerSymbol: 10);
|
||||
algorithm.AddEquity("SPY", Resolution.Second);
|
||||
List<IEnumerator<BaseData>> enumerators = new();
|
||||
foreach (var config in algorithm.SubscriptionManager.Subscriptions)
|
||||
{
|
||||
using var newDataEvent = new ManualResetEvent(false);
|
||||
enumerators.Add(dataQueue.Subscribe(config, (_, _) => {
|
||||
try
|
||||
{
|
||||
newDataEvent.Set();
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
}
|
||||
}));
|
||||
|
||||
Assert.IsTrue(newDataEvent.WaitOne(15000));
|
||||
|
||||
// let's just generate a single point
|
||||
dataQueue.Unsubscribe(config);
|
||||
}
|
||||
dataQueue.Dispose();
|
||||
|
||||
foreach (var enumerator in enumerators)
|
||||
{
|
||||
// assert each data type generate data correctly
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
Assert.IsNotNull(enumerator.Current);
|
||||
|
||||
Log.Debug($"FakeDataQueueTests.GeneratesDataCorrectly(): {enumerator.Current}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
|
||||
using QuantConnect.Lean.Engine.Results;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Tests.Common.Securities;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class FileSystemDataFeedTests
|
||||
{
|
||||
[Test]
|
||||
public void TestsFileSystemDataFeedSpeed()
|
||||
{
|
||||
var job = new BacktestNodePacket();
|
||||
var resultHandler = new BacktestingResultHandler();
|
||||
|
||||
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
|
||||
var feed = new FileSystemDataFeed();
|
||||
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
|
||||
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
var dataManager = new DataManager(feed,
|
||||
new UniverseSelection(
|
||||
algorithm,
|
||||
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio), algorithm: algorithm),
|
||||
dataPermissionManager,
|
||||
TestGlobals.DataProvider),
|
||||
algorithm,
|
||||
algorithm.TimeKeeper,
|
||||
marketHoursDatabase,
|
||||
false,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
dataPermissionManager);
|
||||
algorithm.SubscriptionManager.SetDataManager(dataManager);
|
||||
using var synchronizer = new Synchronizer();
|
||||
synchronizer.Initialize(algorithm, dataManager, new());
|
||||
|
||||
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, dataManager, synchronizer, dataPermissionManager.DataChannelProvider);
|
||||
algorithm.Initialize();
|
||||
algorithm.PostInitialize();
|
||||
|
||||
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
var count = 0;
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var lastMonth = algorithm.StartDate.Month;
|
||||
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
|
||||
{
|
||||
if (timeSlice.Time.Month != lastMonth)
|
||||
{
|
||||
var elapsed = stopwatch.Elapsed.TotalSeconds;
|
||||
var thousands = count / 1000d;
|
||||
Log.Trace($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands / elapsed}");
|
||||
lastMonth = timeSlice.Time.Month;
|
||||
}
|
||||
count++;
|
||||
}
|
||||
Log.Trace("Count: " + count);
|
||||
stopwatch.Stop();
|
||||
feed.Exit();
|
||||
dataManager.RemoveAllSubscriptions();
|
||||
Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TestDataFeedEnumeratorStackSpeed()
|
||||
{
|
||||
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
|
||||
algorithm.Initialize();
|
||||
algorithm.PostInitialize();
|
||||
|
||||
var resultHandler = new BacktestingResultHandler();
|
||||
using var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider,
|
||||
TestGlobals.FactorFileProvider, TestGlobals.DataCacheProvider, algorithm, enablePriceScaling: false);
|
||||
|
||||
var universe = algorithm.UniverseManager.Single().Value;
|
||||
var security = algorithm.Securities.Single().Value;
|
||||
var securityConfig = security.Subscriptions.First();
|
||||
var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, algorithm.StartDate, algorithm.EndDate);
|
||||
var enumerator = factory.CreateEnumerator(subscriptionRequest, TestGlobals.DataProvider);
|
||||
|
||||
var count = 0;
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var lastMonth = algorithm.StartDate.Month;
|
||||
while (enumerator.MoveNext())
|
||||
{
|
||||
var current = enumerator.Current;
|
||||
if (current == null)
|
||||
{
|
||||
Log.Trace("ERROR: Current is null");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current.Time.Month != lastMonth)
|
||||
{
|
||||
var elapsed = stopwatch.Elapsed.TotalSeconds;
|
||||
var thousands = count / 1000d;
|
||||
Log.Trace($"{DateTime.Now} - Time: {current.Time}: KPS: {thousands / elapsed}");
|
||||
lastMonth = current.Time.Month;
|
||||
}
|
||||
count++;
|
||||
}
|
||||
Log.Trace("Count: " + count);
|
||||
|
||||
stopwatch.Stop();
|
||||
enumerator.Dispose();
|
||||
factory.DisposeSafely();
|
||||
Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ChecksMapFileFirstDate()
|
||||
{
|
||||
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
|
||||
algorithm.Initialize();
|
||||
algorithm.PostInitialize();
|
||||
|
||||
var resultHandler = new TestResultHandler();
|
||||
using var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider,
|
||||
TestGlobals.FactorFileProvider, TestGlobals.DataCacheProvider, algorithm, enablePriceScaling: false);
|
||||
|
||||
var universe = algorithm.UniverseManager.Single().Value;
|
||||
var security = algorithm.AddEquity("AAA", Resolution.Daily);
|
||||
var securityConfig = security.Subscriptions.First();
|
||||
// start date is before the first date in the map file
|
||||
var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, new DateTime(2001, 12, 1),
|
||||
new DateTime(2016, 11, 1));
|
||||
var enumerator = factory.CreateEnumerator(subscriptionRequest, TestGlobals.DataProvider);
|
||||
// should initialize the data source reader
|
||||
enumerator.MoveNext();
|
||||
|
||||
enumerator.Dispose();
|
||||
factory.DisposeSafely();
|
||||
resultHandler.Exit();
|
||||
|
||||
var message = ((DebugPacket)resultHandler.Messages.Single()).Message;
|
||||
Assert.IsTrue(message.Equals(
|
||||
"The starting dates for the following symbols have been adjusted to match their map files first date: [AAA, 2020-09-09]"));
|
||||
}
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void OptionChainEnumerator(bool fillForward)
|
||||
{
|
||||
var job = new BacktestNodePacket();
|
||||
var resultHandler = new BacktestingResultHandler();
|
||||
var feed = new FileSystemDataFeed();
|
||||
var algorithm = new AlgorithmStub(feed);
|
||||
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
|
||||
algorithm.SetStartDate(new DateTime(2014, 06, 06));
|
||||
algorithm.SetEndDate(new DateTime(2014, 06, 09));
|
||||
|
||||
var optionChainProvider = new BacktestingOptionChainProvider();
|
||||
optionChainProvider.Initialize(new(TestGlobals.MapFileProvider, TestGlobals.HistoryProvider));
|
||||
algorithm.SetOptionChainProvider(optionChainProvider);
|
||||
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
using var synchronizer = new Synchronizer();
|
||||
synchronizer.Initialize(algorithm, algorithm.DataManager, new());
|
||||
|
||||
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
|
||||
var option = algorithm.AddOption("AAPL", fillForward: fillForward);
|
||||
option.SetFilter(filter => filter.FrontMonth());
|
||||
algorithm.PostInitialize();
|
||||
|
||||
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
var count = 0;
|
||||
var lastMonth = algorithm.StartDate.Month;
|
||||
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
|
||||
{
|
||||
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0 && timeSlice.Time.Date <= algorithm.EndDate)
|
||||
{
|
||||
var baseDataCollection = timeSlice.UniverseData.Where(x => x.Key is OptionChainUniverse).SingleOrDefault().Value;
|
||||
if (baseDataCollection != null)
|
||||
{
|
||||
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
|
||||
Assert.AreEqual(new TimeSpan(0, 0, 0), nyTime.TimeOfDay, $"Failed on: {nyTime}");
|
||||
|
||||
Assert.AreEqual(nyTime.TimeOfDay, baseDataCollection.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay);
|
||||
Assert.IsNotNull(baseDataCollection.FilteredContracts);
|
||||
CollectionAssert.IsNotEmpty(baseDataCollection.FilteredContracts);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
feed.Exit();
|
||||
algorithm.DataManager.RemoveAllSubscriptions();
|
||||
|
||||
// 2 tradable dates between 2014-06-06 and 2014-06-09 (the 6th and 9th)
|
||||
Assert.AreEqual(2, count);
|
||||
}
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void FutureChainEnumerator(bool fillForward)
|
||||
{
|
||||
var job = new BacktestNodePacket();
|
||||
var resultHandler = new BacktestingResultHandler();
|
||||
var feed = new FileSystemDataFeed();
|
||||
var algorithm = new AlgorithmStub(feed);
|
||||
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
|
||||
algorithm.SetStartDate(new DateTime(2013, 10, 07));
|
||||
algorithm.SetEndDate(new DateTime(2013, 10, 08));
|
||||
|
||||
var optionChainProvider = new BacktestingOptionChainProvider();
|
||||
optionChainProvider.Initialize(new(TestGlobals.MapFileProvider, TestGlobals.HistoryProvider));
|
||||
algorithm.SetOptionChainProvider(optionChainProvider);
|
||||
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
using var synchronizer = new Synchronizer();
|
||||
synchronizer.Initialize(algorithm, algorithm.DataManager, new());
|
||||
|
||||
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider,
|
||||
algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
|
||||
var future = algorithm.AddFuture("ES", fillForward: fillForward, extendedMarketHours: true);
|
||||
future.SetFilter(0, 300);
|
||||
algorithm.PostInitialize();
|
||||
|
||||
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
var count = 0L;
|
||||
var lastMonth = algorithm.StartDate.Month;
|
||||
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
|
||||
{
|
||||
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0 && timeSlice.Time.Date <= algorithm.EndDate)
|
||||
{
|
||||
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
|
||||
var universeData = timeSlice.UniverseData;
|
||||
var chainData = universeData.Where(x => x.Key is FuturesChainUniverse).Single().Value;
|
||||
|
||||
Log.Trace($"{nyTime}. Count: {count}. Universe Data Count {universeData.Count}");
|
||||
Assert.AreEqual(TimeSpan.Zero, nyTime.TimeOfDay, $"Failed on: {nyTime}. Count: {count}");
|
||||
Assert.IsTrue(timeSlice.UniverseData.All(kvp => kvp.Value.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay == nyTime.TimeOfDay));
|
||||
if (chainData.FilteredContracts.IsNullOrEmpty())
|
||||
{
|
||||
Assert.AreEqual(new DateTime(2013, 10, 09), nyTime, $"Unexpected chain FilteredContracts was empty on {nyTime}");
|
||||
}
|
||||
|
||||
if (universeData.Count == 1)
|
||||
{
|
||||
// the chain
|
||||
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
|
||||
}
|
||||
else
|
||||
{
|
||||
// we have 2 universe data, the chain and the continuous future
|
||||
Assert.AreEqual(2, universeData.Count);
|
||||
Assert.IsTrue(universeData.All(kvp => kvp.Key.Configuration.Symbol.SecurityType == SecurityType.Future));
|
||||
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
|
||||
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol.ID.Symbol.Contains("CONTINUOUS", StringComparison.InvariantCultureIgnoreCase)));
|
||||
|
||||
var continuousData = universeData.Where(x => x.Key is ContinuousContractUniverse).Single().Value;
|
||||
Assert.AreEqual(TimeSpan.Zero, nyTime.TimeOfDay, $"Failed on: {nyTime}");
|
||||
Assert.IsTrue(!chainData.FilteredContracts.IsNullOrEmpty());
|
||||
}
|
||||
|
||||
count++;
|
||||
}
|
||||
}
|
||||
feed.Exit();
|
||||
algorithm.DataManager.RemoveAllSubscriptions();
|
||||
|
||||
// 2 tradable days
|
||||
Assert.AreEqual(2, count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ContinuousFutureUniverseSelectionIsPerformedOnExtendedMarketHoursDates([Values] bool extendedMarketHours)
|
||||
{
|
||||
var job = new BacktestNodePacket();
|
||||
var resultHandler = new BacktestingResultHandler();
|
||||
var feed = new FileSystemDataFeed();
|
||||
var algorithm = new AlgorithmStub(feed);
|
||||
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
|
||||
algorithm.SetStartDate(new DateTime(2019, 08, 01));
|
||||
algorithm.SetEndDate(new DateTime(2019, 08, 08));
|
||||
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
using var synchronizer = new Synchronizer();
|
||||
synchronizer.Initialize(algorithm, algorithm.DataManager, new());
|
||||
|
||||
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider,
|
||||
algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
|
||||
var future = algorithm.AddFuture("GC", Resolution.Daily, extendedMarketHours: extendedMarketHours);
|
||||
algorithm.PostInitialize();
|
||||
|
||||
var addedSecurities = new HashSet<Symbol>();
|
||||
var mappingCounts = 0;
|
||||
|
||||
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
|
||||
{
|
||||
if (timeSlice.IsTimePulse) continue;
|
||||
|
||||
var addedSymbols = timeSlice.SecurityChanges.AddedSecurities.Select(x => x.Symbol).ToHashSet();
|
||||
|
||||
if (timeSlice.Slice.SymbolChangedEvents.TryGetValue(future.Symbol, out var symbolChangedEvent))
|
||||
{
|
||||
mappingCounts++;
|
||||
var oldSymbol = algorithm.Symbol(symbolChangedEvent.OldSymbol);
|
||||
var newSymbol = algorithm.Symbol(symbolChangedEvent.NewSymbol);
|
||||
|
||||
Assert.IsTrue(addedSecurities.Contains(oldSymbol));
|
||||
|
||||
Assert.IsTrue(addedSymbols.Contains(newSymbol));
|
||||
}
|
||||
|
||||
addedSecurities.UnionWith(addedSymbols);
|
||||
}
|
||||
|
||||
feed.Exit();
|
||||
algorithm.DataManager.RemoveAllSubscriptions();
|
||||
|
||||
var expectedMappingCounts = extendedMarketHours ? 2 : 1;
|
||||
Assert.AreEqual(expectedMappingCounts, mappingCounts);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DataIsFillForwardedFromWarmupToNormalFeed()
|
||||
{
|
||||
var job = new BacktestNodePacket();
|
||||
var resultHandler = new BacktestingResultHandler();
|
||||
var feed = new FileSystemDataFeed();
|
||||
var algorithm = new AlgorithmStub(feed);
|
||||
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
|
||||
algorithm.SetStartDate(new DateTime(2013, 10, 15));
|
||||
algorithm.SetEndDate(new DateTime(2013, 10, 16));
|
||||
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
using var synchronizer = new Synchronizer();
|
||||
synchronizer.Initialize(algorithm, algorithm.DataManager, new());
|
||||
|
||||
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
|
||||
var equity = algorithm.AddEquity("SPY", fillForward: true, dataNormalizationMode: DataNormalizationMode.Raw);
|
||||
algorithm.SetWarmup(1000);
|
||||
algorithm.PostInitialize();
|
||||
|
||||
QuoteBar lastWarmupQuoteBar = null;
|
||||
TradeBar lastWarmupTradeBar = null;
|
||||
QuoteBar lastQuoteBar = null;
|
||||
TradeBar lastTradeBar = null;
|
||||
|
||||
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
|
||||
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
|
||||
{
|
||||
if (!timeSlice.IsTimePulse && timeSlice.Time.Date <= algorithm.EndDate)
|
||||
{
|
||||
Assert.IsTrue(timeSlice.Slice.QuoteBars.TryGetValue(equity.Symbol, out var quoteBar));
|
||||
Assert.IsTrue(timeSlice.Slice.Bars.TryGetValue(equity.Symbol, out var tradeBar));
|
||||
|
||||
if (timeSlice.Slice.Time <= algorithm.StartDate)
|
||||
{
|
||||
lastWarmupQuoteBar = quoteBar;
|
||||
lastWarmupTradeBar = tradeBar;
|
||||
}
|
||||
else
|
||||
{
|
||||
lastQuoteBar = quoteBar;
|
||||
lastTradeBar = tradeBar;
|
||||
|
||||
// We don't have local data for the start-end range, so we expect all data to be fill-forwarded
|
||||
Assert.IsTrue(lastQuoteBar.IsFillForward);
|
||||
Assert.IsTrue(lastTradeBar.IsFillForward);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
feed.Exit();
|
||||
algorithm.DataManager.RemoveAllSubscriptions();
|
||||
|
||||
// Assert we actually got warmup data
|
||||
Assert.IsNotNull(lastWarmupQuoteBar);
|
||||
Assert.IsNotNull(lastWarmupTradeBar);
|
||||
|
||||
// Assert we got normal data
|
||||
Assert.IsNotNull(lastQuoteBar);
|
||||
Assert.IsNotNull(lastTradeBar);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Tests.Common.Data;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IDataQueueHandler"/> that can be specified
|
||||
/// via a function
|
||||
/// </summary>
|
||||
public class FuncDataQueueHandler : IDataQueueHandler
|
||||
{
|
||||
private readonly HashSet<SubscriptionDataConfig> _subscriptions;
|
||||
private readonly CancellationTokenSource _cancellationTokenSource;
|
||||
private readonly AggregationManager _aggregationManager;
|
||||
private readonly DataQueueHandlerSubscriptionManager _subscriptionManager;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the subscriptions configurations currently being managed by the queue handler
|
||||
/// </summary>
|
||||
public List<SubscriptionDataConfig> SubscriptionDataConfigs
|
||||
{
|
||||
get { lock (_subscriptions) return _subscriptions.ToList(); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the subscriptions Symbols currently being managed by the queue handler
|
||||
/// </summary>
|
||||
public List<Symbol> Subscriptions => _subscriptionManager.GetSubscribedSymbols().ToList();
|
||||
|
||||
/// <summary>
|
||||
/// Returns whether the data provider is connected
|
||||
/// </summary>
|
||||
/// <returns>true if the data provider is connected</returns>
|
||||
public bool IsConnected => true;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FuncDataQueueHandler"/> class
|
||||
/// </summary>
|
||||
/// <param name="getNextTicksFunction">The functional implementation to get ticks function</param>
|
||||
/// <param name="timeProvider">The time provider to use</param>
|
||||
public FuncDataQueueHandler(Func<FuncDataQueueHandler, IEnumerable<BaseData>> getNextTicksFunction, ITimeProvider timeProvider, IAlgorithmSettings algorithmSettings)
|
||||
{
|
||||
_subscriptions = new HashSet<SubscriptionDataConfig>();
|
||||
_cancellationTokenSource = new CancellationTokenSource();
|
||||
_aggregationManager = new TestAggregationManager(timeProvider);
|
||||
_aggregationManager.Initialize(new DataAggregatorInitializeParameters() { AlgorithmSettings = algorithmSettings });
|
||||
_subscriptionManager = new FakeDataQueuehandlerSubscriptionManager((t) => "quote-trade");
|
||||
|
||||
Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (!_cancellationTokenSource.IsCancellationRequested)
|
||||
{
|
||||
var emitted = false;
|
||||
try
|
||||
{
|
||||
foreach (var baseData in getNextTicksFunction(this))
|
||||
{
|
||||
if (_cancellationTokenSource.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
emitted = true;
|
||||
_aggregationManager.Update(baseData);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
if (exception is ObjectDisposedException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
Log.Error(exception);
|
||||
}
|
||||
|
||||
if (!emitted)
|
||||
{
|
||||
Thread.Sleep(50);
|
||||
}
|
||||
else
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
}
|
||||
}
|
||||
}, TaskCreationOptions.LongRunning);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the job we're subscribing for
|
||||
/// </summary>
|
||||
/// <param name="job">Job we're subscribing for</param>
|
||||
public void SetJob(LiveNodePacket job)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds the specified symbols to the subscription
|
||||
/// </summary>
|
||||
/// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
|
||||
/// <param name="newDataAvailableHandler">handler to be fired on new data available</param>
|
||||
/// <returns>The new enumerator for this subscription request</returns>
|
||||
public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
|
||||
{
|
||||
var enumerator = _aggregationManager.Add(dataConfig, newDataAvailableHandler);
|
||||
lock (_subscriptions)
|
||||
{
|
||||
_subscriptions.Add(dataConfig);
|
||||
_subscriptionManager.Subscribe(dataConfig);
|
||||
}
|
||||
return enumerator;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the specified configuration
|
||||
/// </summary>
|
||||
/// <param name="dataConfig">The data config to remove</param>
|
||||
public void Unsubscribe(SubscriptionDataConfig dataConfig)
|
||||
{
|
||||
lock (_subscriptions)
|
||||
{
|
||||
_subscriptions.Remove(dataConfig);
|
||||
_subscriptionManager.Subscribe(dataConfig);
|
||||
}
|
||||
_aggregationManager.Remove(dataConfig);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (!_cancellationTokenSource.IsCancellationRequested)
|
||||
{
|
||||
_cancellationTokenSource.Cancel();
|
||||
}
|
||||
_aggregationManager.DisposeSafely();
|
||||
_cancellationTokenSource.DisposeSafely();
|
||||
}
|
||||
|
||||
private class TestAggregationManager : AggregationManager
|
||||
{
|
||||
public TestAggregationManager(ITimeProvider timeProvider)
|
||||
{
|
||||
TimeProvider = timeProvider;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Interfaces;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IDataQueueHandler"/> and <see cref="IDataQueueUniverseProvider"/>
|
||||
/// that can be specified via functions
|
||||
/// </summary>
|
||||
public class FuncDataQueueHandlerUniverseProvider : FuncDataQueueHandler, IDataQueueUniverseProvider
|
||||
{
|
||||
private readonly Func<Symbol, bool, string, IEnumerable<Symbol>> _lookupSymbolsFunction;
|
||||
private readonly Func<bool> _canPerformSelectionFunction;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FuncDataQueueHandlerUniverseProvider"/> class
|
||||
/// </summary>
|
||||
/// <param name="getNextTicksFunction">The functional implementation for the <see cref="FuncDataQueueHandler.GetNextTicks"/> function</param>
|
||||
/// <param name="lookupSymbolsFunction">The functional implementation for the <see cref="IDataQueueUniverseProvider.LookupSymbols"/> function</param>
|
||||
/// <param name="canPerformSelectionFunction">The functional implementation for the <see cref="IDataQueueUniverseProvider.CanPerformSelection"/> function</param>
|
||||
/// <param name="timeProvider">The time provider instance to use</param>
|
||||
public FuncDataQueueHandlerUniverseProvider(
|
||||
Func<FuncDataQueueHandler, IEnumerable<BaseData>> getNextTicksFunction,
|
||||
Func<Symbol, bool, string, IEnumerable<Symbol>> lookupSymbolsFunction,
|
||||
Func<bool> canPerformSelectionFunction,
|
||||
ITimeProvider timeProvider, IAlgorithmSettings algorithmSettings)
|
||||
: base(getNextTicksFunction, timeProvider, algorithmSettings)
|
||||
{
|
||||
_lookupSymbolsFunction = lookupSymbolsFunction;
|
||||
_canPerformSelectionFunction = canPerformSelectionFunction;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Method returns a collection of Symbols that are available at the data source.
|
||||
/// </summary>
|
||||
/// <param name="symbol">Symbol to lookup</param>
|
||||
/// <param name="includeExpired">Include expired contracts</param>
|
||||
/// <param name="securityCurrency">Expected security currency(if any)</param>
|
||||
/// <returns></returns>
|
||||
public IEnumerable<Symbol> LookupSymbols(Symbol symbol, bool includeExpired, string securityCurrency = null)
|
||||
{
|
||||
return _lookupSymbolsFunction(symbol, includeExpired, securityCurrency);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns whether selection can take place or not.
|
||||
/// </summary>
|
||||
/// <returns>True if selection can take place</returns>
|
||||
public bool CanPerformSelection()
|
||||
{
|
||||
return _canPerformSelectionFunction();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class IndexSubscriptionDataSourceReaderTests
|
||||
{
|
||||
private DateTime _initialDate;
|
||||
private TestDataCacheProvider _dataCacheProvider;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_initialDate = new DateTime(2018, 1, 1);
|
||||
_dataCacheProvider = new TestDataCacheProvider();
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_dataCacheProvider.DisposeSafely();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThrowsIfDataIsNotIndexBased()
|
||||
{
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
|
||||
Assert.Throws<ArgumentException>(() => new IndexSubscriptionDataSourceReader(
|
||||
_dataCacheProvider,
|
||||
config,
|
||||
_initialDate,
|
||||
false,
|
||||
TestGlobals.DataProvider,
|
||||
null));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetsIndexAndSource()
|
||||
{
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TestIndexedBasedFactory),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
|
||||
var reader = new IndexSubscriptionDataSourceReader(
|
||||
_dataCacheProvider,
|
||||
config,
|
||||
_initialDate,
|
||||
false,
|
||||
TestGlobals.DataProvider,
|
||||
null);
|
||||
var source = (new TradeBar()).GetSource(config, _initialDate, false);
|
||||
_dataCacheProvider.Data = "20000101 00:00,2,2,2,2,2";
|
||||
var dataBars = reader.Read(source).First();
|
||||
|
||||
Assert.IsNotNull(dataBars);
|
||||
Assert.IsNotNull(dataBars.Symbol, Symbols.SPY.Value);
|
||||
Assert.AreEqual("20000101 00:00,2,2,2,2,2", TestIndexedBasedFactory.IndexLine);
|
||||
Assert.AreEqual("20000101 00:00,2,2,2,2,2", TestIndexedBasedFactory.ReaderLine);
|
||||
}
|
||||
|
||||
private class TestIndexedBasedFactory : IndexedBaseData
|
||||
{
|
||||
public static string ReaderLine { get; set; }
|
||||
public static string IndexLine { get; set; }
|
||||
|
||||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
|
||||
{
|
||||
ReaderLine = line;
|
||||
var bar = new TradeBar();
|
||||
return bar.Reader(config, line, date, isLiveMode);
|
||||
}
|
||||
public override SubscriptionDataSource GetSourceForAnIndex(SubscriptionDataConfig config, DateTime date, string index, bool isLiveMode)
|
||||
{
|
||||
IndexLine = index;
|
||||
return new SubscriptionDataSource("",
|
||||
SubscriptionTransportMedium.LocalFile,
|
||||
FileFormat.Csv);
|
||||
}
|
||||
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
return new SubscriptionDataSource("",
|
||||
SubscriptionTransportMedium.LocalFile,
|
||||
FileFormat.Csv);
|
||||
}
|
||||
}
|
||||
|
||||
private class TestDataCacheProvider : IDataCacheProvider
|
||||
{
|
||||
private StreamWriter _writer;
|
||||
public string Data { set; get; }
|
||||
public bool IsDataEphemeral => false;
|
||||
|
||||
public Stream Fetch(string key)
|
||||
{
|
||||
var stream = new MemoryStream();
|
||||
_writer = new StreamWriter(stream);
|
||||
_writer.Write(Data);
|
||||
_writer.Flush();
|
||||
stream.Position = 0;
|
||||
return stream;
|
||||
}
|
||||
public void Store(string key, byte[] data)
|
||||
{
|
||||
}
|
||||
public List<string> GetZipEntries(string zipFile)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
public void Dispose()
|
||||
{
|
||||
_writer.DisposeSafely();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Algorithm;
|
||||
using QuantConnect.Brokerages.Paper;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Queues;
|
||||
using QuantConnect.Lean.Engine.Results;
|
||||
using QuantConnect.Lean.Engine.TransactionHandlers;
|
||||
using QuantConnect.Orders;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class InternalSubscriptionManagerTests
|
||||
{
|
||||
private IResultHandler _resultHandler;
|
||||
private Synchronizer _synchronizer;
|
||||
private DataManager _dataManager;
|
||||
private QCAlgorithm _algorithm;
|
||||
private IDataFeed _dataFeed;
|
||||
private AggregationManager _aggregationManager;
|
||||
private PaperBrokerage _paperBrokerage;
|
||||
private ITransactionHandler _transactionHandler;
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
SetupImpl(null, null, null);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_transactionHandler.Exit();
|
||||
_dataFeed.Exit();
|
||||
_dataManager.RemoveAllSubscriptions();
|
||||
_resultHandler.Exit();
|
||||
_synchronizer.Dispose();
|
||||
_aggregationManager.DisposeSafely();
|
||||
_paperBrokerage.DisposeSafely();
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(DataTypeTestCases))]
|
||||
public void CreatesSubscriptions(SubscriptionRequest subscriptionRequest, bool liveMode, bool expectNewSubscription, bool isWarmup)
|
||||
{
|
||||
_algorithm.SetLiveMode(liveMode);
|
||||
if (isWarmup)
|
||||
{
|
||||
_algorithm.SetWarmUp(10, Resolution.Daily);
|
||||
}
|
||||
_algorithm.PostInitialize();
|
||||
|
||||
var added = false;
|
||||
var start = DateTime.UtcNow;
|
||||
using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
|
||||
foreach (var timeSlice in _synchronizer.StreamData(tokenSource.Token))
|
||||
{
|
||||
if (!added)
|
||||
{
|
||||
_algorithm.AddSecurity(subscriptionRequest.Security.Symbol, subscriptionRequest.Configuration.Resolution);
|
||||
}
|
||||
else if (!timeSlice.IsTimePulse)
|
||||
{
|
||||
Assert.AreEqual(
|
||||
expectNewSubscription,
|
||||
_algorithm.SubscriptionManager.SubscriptionDataConfigService
|
||||
.GetSubscriptionDataConfigs(Symbols.BTCUSD, includeInternalConfigs: true).Any(config => config.IsInternalFeed)
|
||||
);
|
||||
|
||||
if (expectNewSubscription)
|
||||
{
|
||||
var utcStartTime = _dataManager.DataFeedSubscriptions
|
||||
.Where(subscription => subscription.Configuration.IsInternalFeed && subscription.Configuration.Symbol == Symbols.BTCUSD)
|
||||
.Select(subscription => subscription.UtcStartTime)
|
||||
.First();
|
||||
Assert.Greater(utcStartTime.Ticks, start.Ticks);
|
||||
|
||||
// let's wait for a data point
|
||||
if (timeSlice.DataPointCount > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (DateTime.UtcNow - start > TimeSpan.FromSeconds(5))
|
||||
{
|
||||
Assert.Fail("Timeout waiting for data point");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
_algorithm.OnEndOfTimeStep();
|
||||
if (!added)
|
||||
{
|
||||
added = true;
|
||||
// give time for the base exchange to pick up the data point that will trigger the universe selection
|
||||
// so next step we assert the internal config is there
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
else
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
}
|
||||
}
|
||||
Assert.IsFalse(tokenSource.IsCancellationRequested);
|
||||
tokenSource.DisposeSafely();
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(DataTypeTestCases))]
|
||||
public void RemoveSubscriptions(SubscriptionRequest subscriptionRequest, bool liveMode, bool expectNewSubscription, bool isWarmup)
|
||||
{
|
||||
if (!expectNewSubscription)
|
||||
{
|
||||
// we only test cases where we expect an internal subscription
|
||||
return;
|
||||
}
|
||||
_algorithm.SetLiveMode(liveMode);
|
||||
if (isWarmup)
|
||||
{
|
||||
_algorithm.SetWarmUp(10, Resolution.Daily);
|
||||
}
|
||||
_algorithm.PostInitialize();
|
||||
var added = false;
|
||||
var shouldRemoved = false;
|
||||
var count = 0;
|
||||
using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
|
||||
foreach (var timeSlice in _synchronizer.StreamData(tokenSource.Token))
|
||||
{
|
||||
if (!added)
|
||||
{
|
||||
_algorithm.AddSecurity(subscriptionRequest.Security.Symbol, subscriptionRequest.Configuration.Resolution);
|
||||
}
|
||||
else if (!timeSlice.IsTimePulse && !shouldRemoved)
|
||||
{
|
||||
Assert.IsTrue(_algorithm.SubscriptionManager.SubscriptionDataConfigService
|
||||
.GetSubscriptionDataConfigs(Symbols.BTCUSD, includeInternalConfigs: true).Any());
|
||||
|
||||
_algorithm.RemoveSecurity(subscriptionRequest.Security.Symbol);
|
||||
shouldRemoved = true;
|
||||
}
|
||||
else if (!timeSlice.IsTimePulse && shouldRemoved)
|
||||
{
|
||||
var result = _algorithm.SubscriptionManager.SubscriptionDataConfigService
|
||||
.GetSubscriptionDataConfigs(Symbols.BTCUSD, includeInternalConfigs: true).Any(config => config.IsInternalFeed);
|
||||
// can take some extra loop till the base exchange thread picks up the data point that will trigger the universe selection
|
||||
if (!result || count++ > 5)
|
||||
{
|
||||
Assert.IsFalse(result);
|
||||
}
|
||||
break;
|
||||
}
|
||||
_algorithm.OnEndOfTimeStep();
|
||||
if (!added)
|
||||
{
|
||||
added = true;
|
||||
// give time for the base exchange to pick up the data point that will trigger the universe selection
|
||||
// so next step we assert the internal config is there
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
else
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
}
|
||||
}
|
||||
Assert.IsFalse(tokenSource.IsCancellationRequested);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PreMarketDataSetsCache()
|
||||
{
|
||||
var dataQueueTest = new FakeDataQueueTest();
|
||||
dataQueueTest.ManualTimeProvider.SetCurrentTimeUtc(new DateTime(2020, 09, 03, 10, 0, 0));
|
||||
TearDown();
|
||||
var liveSynchronizer = new TestableLiveSynchronizer(dataQueueTest.ManualTimeProvider);
|
||||
using var dataAggregator = new TestAggregationManager(dataQueueTest.ManualTimeProvider);
|
||||
SetupImpl(dataQueueTest, liveSynchronizer, dataAggregator);
|
||||
|
||||
_algorithm.SetDateTime(dataQueueTest.ManualTimeProvider.GetUtcNow());
|
||||
_algorithm.SetLiveMode(true);
|
||||
_algorithm.PostInitialize();
|
||||
var added = false;
|
||||
var first = true;
|
||||
var internalDataCount = 0;
|
||||
using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
foreach (var timeSlice in _synchronizer.StreamData(tokenSource.Token))
|
||||
{
|
||||
dataQueueTest.ManualTimeProvider.AdvanceSeconds(60);
|
||||
_algorithm.SetDateTime(dataQueueTest.ManualTimeProvider.GetUtcNow());
|
||||
if (dataQueueTest.ManualTimeProvider.GetUtcNow() >= new DateTime(2020, 09, 03, 13, 0, 0))
|
||||
{
|
||||
Assert.Fail("Timeout expect pre market data to set security prices");
|
||||
}
|
||||
if (!added)
|
||||
{
|
||||
_algorithm.AddEquity("IBM", Resolution.Minute);
|
||||
_algorithm.AddEquity("AAPL", Resolution.Hour);
|
||||
}
|
||||
else if (!timeSlice.IsTimePulse)
|
||||
{
|
||||
if (timeSlice.SecuritiesUpdateData.Count > 0)
|
||||
{
|
||||
internalDataCount += timeSlice.SecuritiesUpdateData.Count(
|
||||
data => data.IsInternalConfig && data.Target.Symbol == Symbols.AAPL
|
||||
);
|
||||
}
|
||||
if (first)
|
||||
{
|
||||
Assert.IsTrue(_algorithm.SubscriptionManager.SubscriptionDataConfigService
|
||||
.GetSubscriptionDataConfigs(Symbols.AAPL, includeInternalConfigs: true).Any(config => config.Resolution == Resolution.Second));
|
||||
Assert.IsFalse(_algorithm.SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(Symbols.IBM, includeInternalConfigs: true)
|
||||
.Any(config => config.IsInternalFeed && config.Resolution == Resolution.Second));
|
||||
first = false;
|
||||
}
|
||||
else if (_algorithm.Securities["AAPL"].Price != 0 && _algorithm.Securities["IBM"].Price != 0)
|
||||
{
|
||||
#pragma warning disable CS0618
|
||||
_algorithm.SetHoldings("AAPL", 0.01);
|
||||
_algorithm.SetHoldings("IBM", 0.01);
|
||||
|
||||
var orders = _algorithm.Transactions.GetOpenOrders("AAPL");
|
||||
Assert.AreEqual(1, orders.Count);
|
||||
Assert.AreEqual(Symbols.AAPL, orders[0].Symbol);
|
||||
Assert.AreEqual(OrderStatus.Submitted, orders[0].Status);
|
||||
|
||||
orders = _algorithm.Transactions.GetOpenOrders("IBM");
|
||||
#pragma warning restore CS0618
|
||||
Assert.AreEqual(1, orders.Count);
|
||||
Assert.AreEqual(Symbols.IBM, orders[0].Symbol);
|
||||
Assert.AreEqual(OrderStatus.Submitted, orders[0].Status);
|
||||
break;
|
||||
}
|
||||
}
|
||||
_algorithm.OnEndOfTimeStep();
|
||||
if (!added)
|
||||
{
|
||||
added = true;
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
else
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
}
|
||||
}
|
||||
Assert.IsFalse(tokenSource.IsCancellationRequested);
|
||||
Assert.AreNotEqual(0, internalDataCount);
|
||||
}
|
||||
|
||||
[Test, Category("TravisExclude")]
|
||||
public void UniverseSelectionAddAndRemove()
|
||||
{
|
||||
_algorithm.SetLiveMode(true);
|
||||
_algorithm.PostInitialize();
|
||||
_algorithm.UniverseSettings.Resolution = Resolution.Hour;
|
||||
_algorithm.UniverseSettings.MinimumTimeInUniverse = TimeSpan.Zero;
|
||||
var added = false;
|
||||
using var manualEvent = new ManualResetEvent(false);
|
||||
using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
|
||||
foreach (var timeSlice in _synchronizer.StreamData(tokenSource.Token))
|
||||
{
|
||||
if (!added)
|
||||
{
|
||||
_algorithm.AddUniverse(SecurityType.Equity,
|
||||
"AUniverse",
|
||||
Resolution.Second,
|
||||
Market.USA,
|
||||
_algorithm.UniverseSettings,
|
||||
time =>
|
||||
{
|
||||
return !manualEvent.WaitOne(0) ? new[] { "IBM" } : new[] { "AAPL" };
|
||||
}
|
||||
);
|
||||
}
|
||||
else if (!timeSlice.IsTimePulse)
|
||||
{
|
||||
if (!manualEvent.WaitOne(0))
|
||||
{
|
||||
Assert.IsTrue(_algorithm.SubscriptionManager.SubscriptionDataConfigService
|
||||
.GetSubscriptionDataConfigs(Symbols.IBM, includeInternalConfigs: true).Any(config => config.Resolution == Resolution.Second),
|
||||
"IBM subscription was not found");
|
||||
Assert.IsFalse(_algorithm.SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(Symbols.AAPL, includeInternalConfigs: true).Any(),
|
||||
"Unexpected AAPL subscription was found");
|
||||
manualEvent.Set();
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.IsTrue(_algorithm.SubscriptionManager.SubscriptionDataConfigService
|
||||
.GetSubscriptionDataConfigs(Symbols.AAPL, includeInternalConfigs: true).Any(config => config.Resolution == Resolution.Second),
|
||||
"AAPL subscription was not found");
|
||||
Assert.IsFalse(_algorithm.SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(Symbols.IBM, includeInternalConfigs: true).Any(),
|
||||
"Unexpected IBM subscription was found");
|
||||
break;
|
||||
}
|
||||
}
|
||||
_algorithm.OnEndOfTimeStep();
|
||||
if (!added)
|
||||
{
|
||||
added = true;
|
||||
// we need to give time for the base data exchange to pick up the new data point which will trigger the selection
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
else
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
}
|
||||
}
|
||||
Assert.IsFalse(tokenSource.IsCancellationRequested, "Test timed out");
|
||||
}
|
||||
|
||||
private static TestCaseData[] DataTypeTestCases
|
||||
{
|
||||
get
|
||||
{
|
||||
var result = new List<TestCaseData>();
|
||||
var config = GetConfig(Symbols.BTCUSD, Resolution.Second);
|
||||
result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, false, false));
|
||||
|
||||
config = GetConfig(Symbols.BTCUSD, Resolution.Minute);
|
||||
result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, false, false));
|
||||
|
||||
config = GetConfig(Symbols.BTCUSD, Resolution.Hour);
|
||||
result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, true, false));
|
||||
|
||||
config = GetConfig(Symbols.BTCUSD, Resolution.Daily);
|
||||
result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, true, false));
|
||||
|
||||
config = GetConfig(Symbols.BTCUSD, Resolution.Daily);
|
||||
result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, true, true));
|
||||
|
||||
result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), false, false, false));
|
||||
|
||||
return result.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
private static Security CreateSecurity(SubscriptionDataConfig config)
|
||||
{
|
||||
return new Security(SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
config,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
}
|
||||
|
||||
private static SubscriptionDataConfig GetConfig(Symbol symbol, Resolution resolution)
|
||||
{
|
||||
return new SubscriptionDataConfig(typeof(TradeBar), symbol, resolution, TimeZones.Utc, TimeZones.Utc, false, false, false);
|
||||
}
|
||||
|
||||
private class FakeDataQueueTest : FakeDataQueue
|
||||
{
|
||||
public ManualTimeProvider ManualTimeProvider { get; } = new ManualTimeProvider();
|
||||
|
||||
protected override ITimeProvider TimeProvider => ManualTimeProvider;
|
||||
}
|
||||
|
||||
private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchronizer, IDataAggregator dataAggregator)
|
||||
{
|
||||
_algorithm = new AlgorithmStub(createDataManager: false);
|
||||
_aggregationManager = new AggregationManager();
|
||||
_dataFeed = new TestableLiveTradingDataFeed(_algorithm.Settings, dataQueueHandler ?? new FakeDataQueue(dataAggregator ?? _aggregationManager));
|
||||
_synchronizer = synchronizer ?? new LiveSynchronizer();
|
||||
_algorithm.SetStartDate(new DateTime(2022, 04, 13));
|
||||
|
||||
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
|
||||
var securityService = new SecurityService(_algorithm.Portfolio.CashBook,
|
||||
MarketHoursDatabase.FromDataFolder(),
|
||||
SymbolPropertiesDatabase.FromDataFolder(),
|
||||
_algorithm,
|
||||
registeredTypesProvider,
|
||||
new SecurityCacheProvider(_algorithm.Portfolio),
|
||||
algorithm: _algorithm);
|
||||
var universeSelection = new UniverseSelection(
|
||||
_algorithm,
|
||||
securityService,
|
||||
new DataPermissionManager(),
|
||||
TestGlobals.DataProvider,
|
||||
Resolution.Second);
|
||||
_dataManager = new DataManager(_dataFeed, universeSelection, _algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork),
|
||||
MarketHoursDatabase.FromDataFolder(),
|
||||
true,
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
new DataPermissionManager());
|
||||
_resultHandler = new TestResultHandler();
|
||||
_synchronizer.Initialize(_algorithm, _dataManager, new());
|
||||
_dataFeed.Initialize(_algorithm,
|
||||
new LiveNodePacket(),
|
||||
_resultHandler,
|
||||
TestGlobals.MapFileProvider,
|
||||
TestGlobals.FactorFileProvider,
|
||||
TestGlobals.DataProvider,
|
||||
_dataManager,
|
||||
_synchronizer,
|
||||
new DataChannelProvider());
|
||||
_algorithm.SubscriptionManager.SetDataManager(_dataManager);
|
||||
_algorithm.Securities.SetSecurityService(securityService);
|
||||
var backtestingTransactionHandler = new SynchronousBacktestingTransactionHandler();
|
||||
_paperBrokerage = new PaperBrokerage(_algorithm, new LiveNodePacket());
|
||||
backtestingTransactionHandler.Initialize(_algorithm, _paperBrokerage, _resultHandler);
|
||||
_algorithm.Transactions.SetOrderProcessor(backtestingTransactionHandler);
|
||||
|
||||
if (_transactionHandler != null)
|
||||
{
|
||||
_transactionHandler.Exit();
|
||||
}
|
||||
_transactionHandler = backtestingTransactionHandler;
|
||||
}
|
||||
|
||||
private class SynchronousBacktestingTransactionHandler : BacktestingTransactionHandler
|
||||
{
|
||||
protected override bool SynchronousProcessing => true;
|
||||
}
|
||||
|
||||
private class TestAggregationManager : AggregationManager
|
||||
{
|
||||
public TestAggregationManager(ITimeProvider timeProvider)
|
||||
{
|
||||
TimeProvider = timeProvider;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.Results;
|
||||
using QuantConnect.Lean.Engine.TransactionHandlers;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Orders;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class LiveCoarseUniverseTests
|
||||
{
|
||||
[Test]
|
||||
public void CoarseUniverseRotatesActiveSecurity()
|
||||
{
|
||||
var startDate = new DateTime(2014, 3, 24);
|
||||
var endDate = new DateTime(2014, 3, 31);
|
||||
|
||||
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
|
||||
timeProvider.SetCurrentTime(startDate);
|
||||
|
||||
var coarseTimes = new List<DateTime>
|
||||
{
|
||||
// coarse files go from the 24th (Monday) to the 28th (Friday)
|
||||
// they are emitted in the next day, excluding saturday
|
||||
new DateTime(2014, 3, 25, 5, 0, 0, 0),
|
||||
new DateTime(2014, 3, 26, 5, 0, 0, 0),
|
||||
new DateTime(2014, 3, 27, 5, 0, 0, 0),
|
||||
new DateTime(2014, 3, 28, 5, 0, 0, 0),
|
||||
// 29th is Saturday
|
||||
new DateTime(2014, 3, 30, 5, 0, 0, 0)
|
||||
}.ToHashSet();
|
||||
|
||||
var coarseSymbols = new List<Symbol> { Symbols.SPY, Symbols.AAPL, Symbols.MSFT };
|
||||
|
||||
using var emitted = new AutoResetEvent(false);
|
||||
using var dataQueueHandler = new FuncDataQueueHandler(fdqh => Enumerable.Empty<BaseData>(), timeProvider, new AlgorithmSettings());
|
||||
|
||||
var feed = new TestableLiveTradingDataFeed(new AlgorithmSettings(), dataQueueHandler);
|
||||
|
||||
var algorithm = new AlgorithmStub(feed);
|
||||
algorithm.SetLiveMode(true);
|
||||
|
||||
var mock = new Mock<ITransactionHandler>();
|
||||
mock.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>())).Returns(new List<Order>());
|
||||
algorithm.Transactions.SetOrderProcessor(mock.Object);
|
||||
|
||||
using var synchronizer = new TestableLiveSynchronizer(timeProvider);
|
||||
synchronizer.Initialize(algorithm, algorithm.DataManager, new());
|
||||
|
||||
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
|
||||
TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, new DataChannelProvider());
|
||||
|
||||
var symbolIndex = 0;
|
||||
var coarseUniverseSelectionCount = 0;
|
||||
algorithm.AddUniverse(
|
||||
coarse =>
|
||||
{
|
||||
Log.Trace($"Emitted at {algorithm.Time}. Coarse {coarse.First().Time} to {coarse.First().EndTime}");
|
||||
Interlocked.Increment(ref coarseUniverseSelectionCount);
|
||||
emitted.Set();
|
||||
|
||||
// rotate single symbol in universe
|
||||
if (symbolIndex == coarseSymbols.Count) symbolIndex = 0;
|
||||
|
||||
return new[] { coarseSymbols[symbolIndex++] };
|
||||
});
|
||||
|
||||
algorithm.PostInitialize();
|
||||
|
||||
using var cancellationTokenSource = new CancellationTokenSource();
|
||||
|
||||
Exception exceptionThrown = null;
|
||||
|
||||
// create a timer to advance time much faster than realtime
|
||||
var timerInterval = TimeSpan.FromMilliseconds(5);
|
||||
var timer = Ref.Create<Timer>(null);
|
||||
timer.Value = new Timer(state =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
|
||||
|
||||
if (currentTime.Date > endDate.Date)
|
||||
{
|
||||
feed.Exit();
|
||||
cancellationTokenSource.Cancel();
|
||||
return;
|
||||
}
|
||||
|
||||
timeProvider.Advance(TimeSpan.FromHours(1));
|
||||
|
||||
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
|
||||
algorithm.SetDateTime(timeProvider.GetUtcNow());
|
||||
if (coarseTimes.Contains(time))
|
||||
{
|
||||
// lets wait for coarse to emit
|
||||
if (!emitted.WaitOne(TimeSpan.FromMilliseconds(15000)))
|
||||
{
|
||||
throw new TimeoutException($"Timeout waiting for coarse to emit at {time}");
|
||||
}
|
||||
}
|
||||
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
|
||||
|
||||
Assert.That(activeSecuritiesCount <= 1);
|
||||
|
||||
// restart the timer
|
||||
timer.Value.Change(timerInterval, Timeout.InfiniteTimeSpan);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Log.Error(exception);
|
||||
exceptionThrown = exception;
|
||||
|
||||
feed.Exit();
|
||||
cancellationTokenSource.Cancel();
|
||||
}
|
||||
|
||||
}, null, timerInterval, Timeout.InfiniteTimeSpan);
|
||||
|
||||
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token)) { }
|
||||
|
||||
timer.Value.DisposeSafely();
|
||||
algorithm.DataManager.RemoveAllSubscriptions();
|
||||
|
||||
if (exceptionThrown != null)
|
||||
{
|
||||
throw new RegressionTestException("Exception in timer: ", exceptionThrown);
|
||||
}
|
||||
|
||||
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount, message: "coarseUniverseSelectionCount");
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.Results;
|
||||
using QuantConnect.Packets;
|
||||
|
||||
namespace QuantConnect.Lean.Engine.DataFeeds
|
||||
{
|
||||
public class MockDataFeed : IDataFeed
|
||||
{
|
||||
private List<SubscriptionData> _dummyData = new List<SubscriptionData>();
|
||||
public bool IsActive { get; }
|
||||
|
||||
public void Initialize(
|
||||
IAlgorithm algorithm,
|
||||
AlgorithmNodePacket job,
|
||||
IResultHandler resultHandler,
|
||||
IMapFileProvider mapFileProvider,
|
||||
IFactorFileProvider factorFileProvider,
|
||||
IDataProvider dataProvider,
|
||||
IDataFeedSubscriptionManager subscriptionManager,
|
||||
IDataFeedTimeProvider dataFeedTimeProvider,
|
||||
IDataChannelProvider dataChannelProvider
|
||||
)
|
||||
{
|
||||
}
|
||||
|
||||
public Subscription CreateSubscription(SubscriptionRequest request)
|
||||
{
|
||||
var offsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone,
|
||||
request.StartTimeUtc,
|
||||
request.EndTimeUtc);
|
||||
return new Subscription(request, _dummyData.GetEnumerator(), offsetProvider);
|
||||
}
|
||||
|
||||
public void RemoveSubscription(Subscription subscription)
|
||||
{
|
||||
}
|
||||
|
||||
public void Exit()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Algorithm.Framework.Portfolio;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Orders;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Tests.Common.Securities;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class PendingRemovalsManagerTests
|
||||
{
|
||||
private static readonly DateTime Noon = new DateTime(2015, 11, 2, 12, 0, 0);
|
||||
private static readonly TimeKeeper TimeKeeper = new TimeKeeper(Noon.ConvertToUtc(TimeZones.NewYork), new[] { TimeZones.NewYork });
|
||||
|
||||
[Test]
|
||||
public void ReturnedRemoved_Add()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
using var universe = new TestUniverse();
|
||||
|
||||
var result = pendingRemovals.TryRemoveMember(member, universe);
|
||||
|
||||
Assert.IsTrue(result.Any());
|
||||
Assert.AreEqual(universe, result.First().Universe);
|
||||
Assert.AreEqual(security, result.First().Security);
|
||||
Assert.IsFalse(pendingRemovals.CheckPendingRemovals(new HashSet<Symbol>(), universe).Any());
|
||||
Assert.AreEqual(0, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(0, pendingRemovals.PendingRemovals.Values.Count());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReturnedRemoved_Check()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
using var universe = new TestUniverse();
|
||||
orderProvider.AddOrder(new LimitOrder(security.Symbol, 1, 1, DateTime.UtcNow));
|
||||
pendingRemovals.TryRemoveMember(member, universe);
|
||||
orderProvider.Clear();
|
||||
|
||||
var result = pendingRemovals.CheckPendingRemovals(new HashSet<Symbol>(), universe);
|
||||
|
||||
Assert.IsTrue(result.Any());
|
||||
Assert.AreEqual(universe, result.First().Universe);
|
||||
Assert.AreEqual(security, result.First().Security);
|
||||
Assert.IsFalse(pendingRemovals.CheckPendingRemovals(new HashSet<Symbol>(), universe).Any());
|
||||
Assert.AreEqual(0, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(0, pendingRemovals.PendingRemovals.Values.Count());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveBecauseOfUnderlying()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var equity = CreateSecurity(Symbols.SPY);
|
||||
var member = new Universe.Member(Noon, equity, false);
|
||||
var equityOption = CreateSecurity(Symbols.SPY_C_192_Feb19_2016);
|
||||
|
||||
// we add an order of the equity option
|
||||
orderProvider.AddOrder(new LimitOrder(equityOption.Symbol, 1, 1, DateTime.UtcNow));
|
||||
using var universe = new TestUniverse();
|
||||
universe.AddMember(DateTime.UtcNow, equity, false);
|
||||
universe.AddMember(DateTime.UtcNow, equityOption, false);
|
||||
|
||||
// we try to remove the equity
|
||||
Assert.IsNull(pendingRemovals.TryRemoveMember(member, universe));
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Values.Count());
|
||||
Assert.AreEqual(universe, pendingRemovals.PendingRemovals.Keys.First());
|
||||
Assert.AreEqual(equity, pendingRemovals.PendingRemovals.Values.First().First().Security);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveBecauseOpenOrder_Add()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
using var universe = new TestUniverse();
|
||||
orderProvider.AddOrder(new LimitOrder(security.Symbol, 1, 1, DateTime.UtcNow));
|
||||
|
||||
Assert.IsNull(pendingRemovals.TryRemoveMember(member, universe));
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Values.Count());
|
||||
Assert.AreEqual(universe, pendingRemovals.PendingRemovals.Keys.First());
|
||||
Assert.AreEqual(security, pendingRemovals.PendingRemovals.Values.First().First().Security);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveBecauseOpenOrder_Check()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
using var universe = new TestUniverse();
|
||||
orderProvider.AddOrder(new LimitOrder(security.Symbol, 1, 1, DateTime.UtcNow));
|
||||
|
||||
Assert.IsNull(pendingRemovals.TryRemoveMember(member, universe));
|
||||
Assert.IsFalse(pendingRemovals.CheckPendingRemovals(new HashSet<Symbol>(), universe).Any());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Values.Count());
|
||||
Assert.AreEqual(universe, pendingRemovals.PendingRemovals.Keys.First());
|
||||
Assert.AreEqual(security, pendingRemovals.PendingRemovals.Values.First().First().Security);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveBecauseHoldings_Add()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
using var universe = new TestUniverse();
|
||||
security.Holdings.SetHoldings(10, 10);
|
||||
|
||||
Assert.IsNull(pendingRemovals.TryRemoveMember(member, universe));
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Values.Count());
|
||||
Assert.AreEqual(universe, pendingRemovals.PendingRemovals.Keys.First());
|
||||
Assert.AreEqual(security, pendingRemovals.PendingRemovals.Values.First().First().Security);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveBecauseHoldings_Check()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
using var universe = new TestUniverse();
|
||||
security.Holdings.SetHoldings(10, 10);
|
||||
|
||||
Assert.IsNull(pendingRemovals.TryRemoveMember(member, universe));
|
||||
Assert.IsFalse(pendingRemovals.CheckPendingRemovals(new HashSet<Symbol>(), universe).Any());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Values.Count());
|
||||
Assert.AreEqual(universe, pendingRemovals.PendingRemovals.Keys.First());
|
||||
Assert.AreEqual(security, pendingRemovals.PendingRemovals.Values.First().First().Security);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveBecauseTarget_Add()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
using var universe = new TestUniverse();
|
||||
security.Holdings.Target = new PortfolioTarget(security.Symbol, 10);
|
||||
|
||||
Assert.IsNull(pendingRemovals.TryRemoveMember(member, universe));
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Values.Count());
|
||||
Assert.AreEqual(universe, pendingRemovals.PendingRemovals.Keys.First());
|
||||
Assert.AreEqual(security, pendingRemovals.PendingRemovals.Values.First().First().Security);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveBecauseTarget_Check()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
using var universe = new TestUniverse();
|
||||
security.Holdings.Target = new PortfolioTarget(security.Symbol, 10);
|
||||
|
||||
Assert.IsNull(pendingRemovals.TryRemoveMember(member, universe));
|
||||
Assert.IsFalse(pendingRemovals.CheckPendingRemovals(new HashSet<Symbol>(), universe).Any());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Values.Count());
|
||||
Assert.AreEqual(universe, pendingRemovals.PendingRemovals.Keys.First());
|
||||
Assert.AreEqual(security, pendingRemovals.PendingRemovals.Values.First().First().Security);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontBeReturnedBecauseReSelected()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
using var universe = new TestUniverse();
|
||||
orderProvider.AddOrder(new LimitOrder(security.Symbol, 1, 1, DateTime.UtcNow));
|
||||
pendingRemovals.TryRemoveMember(member, universe);
|
||||
|
||||
Assert.IsFalse(pendingRemovals.CheckPendingRemovals(
|
||||
new HashSet<Symbol> { security.Symbol}, universe).Any());
|
||||
|
||||
// internally it was removed because it was reselected
|
||||
Assert.AreEqual(0, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(0, pendingRemovals.PendingRemovals.Values.Count());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WontRemoveBecauseUnsettledFunds()
|
||||
{
|
||||
var orderProvider = new FakeOrderProcessor();
|
||||
var pendingRemovals = new PendingRemovalsManager(orderProvider);
|
||||
using var universe = new TestUniverse();
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var member = new Universe.Member(Noon, security, false);
|
||||
|
||||
security.SetSettlementModel(new DelayedSettlementModel(1, TimeSpan.FromHours(8)));
|
||||
var securities = new SecurityManager(TimeKeeper);
|
||||
var transactions = new SecurityTransactionManager(null, securities);
|
||||
var portfolio = new SecurityPortfolioManager(securities, transactions, new AlgorithmSettings());
|
||||
security.SettlementModel.ApplyFunds(new ApplyFundsSettlementModelParameters(portfolio, security, TimeKeeper.UtcTime.Date.AddDays(1),
|
||||
new CashAmount(1000, Currencies.USD), null));
|
||||
|
||||
Assert.IsNull(pendingRemovals.TryRemoveMember(member, universe));
|
||||
Assert.IsFalse(pendingRemovals.CheckPendingRemovals(new HashSet<Symbol>(), universe).Any());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Keys.Count());
|
||||
Assert.AreEqual(1, pendingRemovals.PendingRemovals.Values.Count());
|
||||
Assert.AreEqual(universe, pendingRemovals.PendingRemovals.Keys.First());
|
||||
Assert.AreEqual(security, pendingRemovals.PendingRemovals.Values.First().First().Security);
|
||||
}
|
||||
|
||||
private class TestUniverse : Universe
|
||||
{
|
||||
public TestUniverse()
|
||||
: base(SecurityTests.CreateTradeBarConfig())
|
||||
{
|
||||
}
|
||||
public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
|
||||
private static Security CreateSecurity(Symbol symbol)
|
||||
{
|
||||
return new Security(symbol,
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Securities;
|
||||
using System;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class PrecalculatedSubscriptionDataTests
|
||||
{
|
||||
private SubscriptionDataConfig _config;
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
_config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ChangeDataNormalizationMode()
|
||||
{
|
||||
var tb = new TradeBar
|
||||
{
|
||||
Time = new DateTime(2020, 5, 21, 8, 9, 0),
|
||||
Period = TimeSpan.FromHours(1),
|
||||
Symbol = Symbols.SPY,
|
||||
Open = 100,
|
||||
High = 200,
|
||||
Low = 300,
|
||||
Close = 400
|
||||
};
|
||||
|
||||
var factor = 0.5m;
|
||||
var sumOfDividends = 100m;
|
||||
var adjustedTb = tb.Clone(tb.IsFillForward).Normalize(factor, DataNormalizationMode.Adjusted, 0);
|
||||
|
||||
var exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.Utc);
|
||||
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.Utc, new DateTime(2020, 5, 21), new DateTime(2020, 5, 22));
|
||||
|
||||
var emitTimeUtc = offsetProvider.ConvertToUtc(tb.EndTime);
|
||||
_config.SumOfDividends = sumOfDividends;
|
||||
|
||||
var subscriptionData = new PrecalculatedSubscriptionData(
|
||||
_config,
|
||||
tb,
|
||||
adjustedTb,
|
||||
DataNormalizationMode.Adjusted,
|
||||
emitTimeUtc);
|
||||
|
||||
_config.DataNormalizationMode = DataNormalizationMode.Raw;
|
||||
Assert.AreEqual(tb.Open, (subscriptionData.Data as TradeBar).Open);
|
||||
Assert.AreEqual(tb.High, (subscriptionData.Data as TradeBar).High);
|
||||
Assert.AreEqual(tb.Low, (subscriptionData.Data as TradeBar).Low);
|
||||
Assert.AreEqual(tb.Close, (subscriptionData.Data as TradeBar).Close);
|
||||
|
||||
_config.DataNormalizationMode = DataNormalizationMode.Adjusted;
|
||||
Assert.AreEqual(tb.Open * factor, (subscriptionData.Data as TradeBar).Open);
|
||||
Assert.AreEqual(tb.High * factor, (subscriptionData.Data as TradeBar).High);
|
||||
Assert.AreEqual(tb.Low * factor, (subscriptionData.Data as TradeBar).Low);
|
||||
Assert.AreEqual(tb.Close * factor, (subscriptionData.Data as TradeBar).Close);
|
||||
|
||||
_config.DataNormalizationMode = DataNormalizationMode.TotalReturn;
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
{
|
||||
var data = subscriptionData.Data;
|
||||
}
|
||||
);
|
||||
|
||||
_config.DataNormalizationMode = DataNormalizationMode.SplitAdjusted;
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
{
|
||||
var data = subscriptionData.Data;
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class PredicateTimeProviderTests
|
||||
{
|
||||
[Test]
|
||||
public void RespectsCustomStepEvaluator()
|
||||
{
|
||||
var startTime = new DateTime(2018, 1, 1);
|
||||
var manualTimeProvider = new ManualTimeProvider(startTime);
|
||||
var stepTimeProvider = new PredicateTimeProvider(manualTimeProvider,
|
||||
// only step when minute is a pair number
|
||||
time => time.Minute % 2 == 0);
|
||||
|
||||
Assert.AreEqual(manualTimeProvider.GetUtcNow(), stepTimeProvider.GetUtcNow());
|
||||
|
||||
manualTimeProvider.AdvanceSeconds(45 * 60); // advance 45 minutes, past the interval
|
||||
|
||||
// still the same because 45 minutes isn't pair
|
||||
Assert.AreEqual(startTime, stepTimeProvider.GetUtcNow());
|
||||
Assert.AreNotEqual(manualTimeProvider.GetUtcNow(), stepTimeProvider.GetUtcNow());
|
||||
|
||||
manualTimeProvider.AdvanceSeconds(60);
|
||||
Assert.AreEqual(manualTimeProvider.GetUtcNow(), stepTimeProvider.GetUtcNow());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Threading;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Explicit("Performance test")]
|
||||
public class RealTimeScheduleEventServiceTests
|
||||
{
|
||||
[Test]
|
||||
public void Accuracy()
|
||||
{
|
||||
using var scheduledEventService = new RealTimeScheduleEventService(RealTimeProvider.Instance);
|
||||
EventHandler handler = (_, __) =>
|
||||
{
|
||||
Log.Trace($"{DateTime.UtcNow:O}");
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var nextEventTime = now.RoundDown(TimeSpan.FromSeconds(1)).Add(TimeSpan.FromSeconds(1) + TimeSpan.FromMilliseconds(101));
|
||||
scheduledEventService.ScheduleEvent(nextEventTime - now, now);
|
||||
};
|
||||
scheduledEventService.NewEvent += handler;
|
||||
handler(this, null);
|
||||
Thread.Sleep(5000);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using Newtonsoft.Json;
|
||||
using NodaTime;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
/// <summary>
|
||||
/// Custom data type that causes rest api calls
|
||||
/// </summary>
|
||||
public class RestApiBaseData : TradeBar
|
||||
{
|
||||
public static int ReaderCount = 0;
|
||||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
|
||||
{
|
||||
ReaderCount++;
|
||||
//[{"symbol":"SPY","time":1444271505,"alpha":1,"beta":2}]
|
||||
var array = JsonConvert.DeserializeObject<JsonSerialization[]>(line);
|
||||
if (array.Length > 0)
|
||||
{
|
||||
return array[0].ToBaseData(config.DataTimeZone, config.Increment, config.Symbol);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var remoteFileSource = @"https://www.quantconnect.com/live-test?type=rest&symbols=" + config.Symbol.Value;
|
||||
//remoteFileSource = @"http://beta.quantconnect.com/live-test?type=rest&symbols=" + config.Symbol.Value;
|
||||
return new SubscriptionDataSource(remoteFileSource, SubscriptionTransportMedium.Rest, FileFormat.Csv);
|
||||
}
|
||||
|
||||
private class JsonSerialization
|
||||
{
|
||||
public string symbol = String.Empty;
|
||||
public double time = 0;
|
||||
public double alpha = 0;
|
||||
public double beta = 0;
|
||||
|
||||
public RestApiBaseData ToBaseData(DateTimeZone timeZone, TimeSpan period, Symbol sym)
|
||||
{
|
||||
var dateTime = QuantConnect.Time.UnixTimeStampToDateTime(time).ConvertFromUtc(timeZone).Subtract(period);
|
||||
return new RestApiBaseData
|
||||
{
|
||||
Symbol = sym,
|
||||
Time = dateTime,
|
||||
EndTime = dateTime.Add(period),
|
||||
Value = (decimal) alpha
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using NodaTime;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Securities.Equity;
|
||||
using QuantConnect.Securities.Future;
|
||||
using QuantConnect.Securities.Option;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.All)]
|
||||
public class SubscriptionCollectionTests
|
||||
{
|
||||
[Test]
|
||||
public void EnumerationWhileUpdatingDoesNotThrow()
|
||||
{
|
||||
using var cts = new CancellationTokenSource();
|
||||
var subscriptions = new SubscriptionCollection();
|
||||
var start = DateTime.UtcNow;
|
||||
var end = start.AddSeconds(10);
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Minute, DateTimeZone.Utc, DateTimeZone.Utc, true, false, false);
|
||||
var security = new Equity(
|
||||
Symbols.SPY,
|
||||
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
|
||||
new Cash(Currencies.USD, 0, 1),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, start, end);
|
||||
using var enumerator = new EnqueueableEnumerator<BaseData>();
|
||||
using var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, enumerator, false, false);
|
||||
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
|
||||
var subscription = new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider);
|
||||
|
||||
var addTask = Task.Factory.StartNew(() =>
|
||||
{
|
||||
Log.Trace("Add task started");
|
||||
|
||||
while (DateTime.UtcNow < end)
|
||||
{
|
||||
if (!subscriptions.Contains(config))
|
||||
{
|
||||
subscriptions.TryAdd(subscription);
|
||||
}
|
||||
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
|
||||
Log.Trace("Add task ended");
|
||||
}, cts.Token);
|
||||
|
||||
var removeTask = Task.Factory.StartNew(() =>
|
||||
{
|
||||
Log.Trace("Remove task started");
|
||||
|
||||
while (DateTime.UtcNow < end)
|
||||
{
|
||||
Subscription removed;
|
||||
subscriptions.TryRemove(config, out removed);
|
||||
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
|
||||
Log.Trace("Remove task ended");
|
||||
}, cts.Token);
|
||||
|
||||
var readTask = Task.Factory.StartNew(() =>
|
||||
{
|
||||
Log.Trace("Read task started");
|
||||
|
||||
while (DateTime.UtcNow < end)
|
||||
{
|
||||
foreach (var sub in subscriptions) { }
|
||||
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
|
||||
Log.Trace("Read task ended");
|
||||
}, cts.Token);
|
||||
|
||||
Task.WaitAll(addTask, removeTask, readTask);
|
||||
subscription.Dispose();
|
||||
}
|
||||
[Test]
|
||||
public void DefaultFillForwardResolution()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
var defaultFillForwardResolutio = subscriptionColletion.UpdateAndGetFillForwardResolution();
|
||||
Assert.AreEqual(defaultFillForwardResolutio.Value, new TimeSpan(0, 1, 0));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UpdatesFillForwardResolutionOverridesDefaultWhenNotAdding()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
var subscription = CreateSubscription(Resolution.Daily);
|
||||
|
||||
var fillForwardResolutio = subscriptionColletion.UpdateAndGetFillForwardResolution(subscription.Configuration);
|
||||
Assert.AreEqual(fillForwardResolutio.Value, new TimeSpan(1, 0, 0, 0));
|
||||
subscription.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UpdatesFillForwardResolutionSuccessfullyWhenNotAdding()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
var subscription = CreateSubscription(Resolution.Second);
|
||||
|
||||
var fillForwardResolutio = subscriptionColletion.UpdateAndGetFillForwardResolution(subscription.Configuration);
|
||||
Assert.AreEqual(fillForwardResolutio.Value, new TimeSpan(0, 0, 1));
|
||||
subscription.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UpdatesFillForwardResolutionSuccessfullyWhenAdding()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
using var subscription = CreateSubscription(Resolution.Second);
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 0, 1));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UpdatesFillForwardResolutionSuccessfullyOverridesDefaultWhenAdding()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
var subscription = CreateSubscription(Resolution.Daily);
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(1, 0, 0, 0));
|
||||
subscription.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DoesNotUpdateFillForwardResolutionWhenAddingBiggerResolution()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
var subscription = CreateSubscription(Resolution.Second);
|
||||
var subscription2 = CreateSubscription(Resolution.Minute);
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 0, 1));
|
||||
subscriptionColletion.TryAdd(subscription2);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 0, 1));
|
||||
subscription.Dispose();
|
||||
subscription2.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UpdatesFillForwardResolutionWhenRemoving()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
using var subscription = CreateSubscription(Resolution.Second);
|
||||
using var subscription2 = CreateSubscription(Resolution.Daily);
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
subscriptionColletion.TryAdd(subscription2);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 0, 1));
|
||||
subscriptionColletion.TryRemove(subscription.Configuration, out var outSubscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(1, 0, 0, 0));
|
||||
subscriptionColletion.TryRemove(subscription2.Configuration, out var outSubscription2);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 1, 0));
|
||||
outSubscription.Dispose();
|
||||
outSubscription2.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FillForwardResolutionIgnoresTick()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
using var subscription = CreateSubscription(Resolution.Tick);
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 1, 0));
|
||||
subscriptionColletion.TryRemove(subscription.Configuration, out var outSubscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 1, 0));
|
||||
outSubscription.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FillForwardResolutionIgnoresInternalFeed()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
using var subscription = CreateSubscription(Resolution.Second, "AAPL", true);
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 1, 0));
|
||||
subscriptionColletion.TryRemove(subscription.Configuration, out var outSubscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 1, 0));
|
||||
outSubscription.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DoesNotUpdateFillForwardResolutionWhenRemovingDuplicateResolution()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
using var subscription = CreateSubscription(Resolution.Second);
|
||||
using var subscription2 = CreateSubscription(Resolution.Second, "SPY");
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 0, 1));
|
||||
subscriptionColletion.TryAdd(subscription2);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 0, 1));
|
||||
subscriptionColletion.TryRemove(subscription.Configuration, out var outSubscription);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 0, 1));
|
||||
subscriptionColletion.TryRemove(subscription2.Configuration, out var outSubscription2);
|
||||
Assert.AreEqual(subscriptionColletion.UpdateAndGetFillForwardResolution().Value, new TimeSpan(0, 1, 0));
|
||||
outSubscription.Dispose();
|
||||
outSubscription2.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SubscriptionsAreSortedWhenAdding()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
var subscription = CreateSubscription(Resolution.Second, Futures.Metals.Gold, false, SecurityType.Future);
|
||||
var subscription2 = CreateSubscription(Resolution.Second, "SPY");
|
||||
var subscription3 = CreateSubscription(Resolution.Second, "AAPL", false, SecurityType.Option);
|
||||
var subscription4 = CreateSubscription(Resolution.Second, "EURGBP");
|
||||
var subscription5 = CreateSubscription(Resolution.Second, "AAPL", false, SecurityType.Option, TickType.OpenInterest);
|
||||
var subscription6 = CreateSubscription(Resolution.Second, "AAPL", false, SecurityType.Option, TickType.Quote);
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription });
|
||||
subscriptionColletion.TryAdd(subscription2);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription2, subscription });
|
||||
subscriptionColletion.TryAdd(subscription3);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription2, subscription3, subscription });
|
||||
subscriptionColletion.TryAdd(subscription4);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription4, subscription2, subscription3, subscription });
|
||||
subscriptionColletion.TryAdd(subscription5);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription4, subscription2, subscription3, subscription5, subscription });
|
||||
subscriptionColletion.TryAdd(subscription6);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription4, subscription2, subscription3, subscription6, subscription5, subscription });
|
||||
|
||||
|
||||
Assert.AreEqual(subscriptionColletion.Select(x => x.Configuration.SecurityType).ToList(), new[] { SecurityType.Equity, SecurityType.Equity, SecurityType.Option,
|
||||
SecurityType.Option, SecurityType.Option, SecurityType.Future });
|
||||
|
||||
subscription.Dispose();
|
||||
subscription2.Dispose();
|
||||
subscription3.Dispose();
|
||||
subscription4.Dispose();
|
||||
subscription5.Dispose();
|
||||
subscription6.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SubscriptionsAreSortedWhenAdding2()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
var subscription = CreateSubscription(Resolution.Second, Futures.Metals.Gold, false, SecurityType.Future);
|
||||
var subscription2 = CreateSubscription(Resolution.Second, "SPY");
|
||||
var subscription3 = CreateSubscription(Resolution.Second, "AAPL", false, SecurityType.Option);
|
||||
var subscription4 = CreateSubscription(Resolution.Second, "EURGBP");
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription });
|
||||
subscriptionColletion.TryAdd(subscription2);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription2, subscription });
|
||||
subscriptionColletion.TryAdd(subscription3);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription2, subscription3, subscription });
|
||||
subscriptionColletion.TryAdd(subscription4);
|
||||
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription4, subscription2, subscription3, subscription });
|
||||
Assert.AreEqual(subscriptionColletion.Select(x => x.Configuration.SecurityType).ToList(), new[] { SecurityType.Equity, SecurityType.Equity, SecurityType.Option, SecurityType.Future });
|
||||
|
||||
subscription.Dispose();
|
||||
subscription2.Dispose();
|
||||
subscription3.Dispose();
|
||||
subscription4.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SubscriptionsAreSortedWhenRemoving()
|
||||
{
|
||||
var subscriptionColletion = new SubscriptionCollection();
|
||||
var subscription = CreateSubscription(Resolution.Second, Futures.Metals.Gold, false, SecurityType.Future);
|
||||
var subscription2 = CreateSubscription(Resolution.Second, "SPY");
|
||||
var subscription3 = CreateSubscription(Resolution.Second, "AAPL", false, SecurityType.Option);
|
||||
var subscription4 = CreateSubscription(Resolution.Second, "EURGBP");
|
||||
var subscription5 = CreateSubscription(Resolution.Second, "AAPL", false, SecurityType.Option, TickType.OpenInterest);
|
||||
var subscription6 = CreateSubscription(Resolution.Second, "AAPL", false, SecurityType.Option, TickType.Quote);
|
||||
|
||||
subscriptionColletion.TryAdd(subscription);
|
||||
subscriptionColletion.TryAdd(subscription2);
|
||||
subscriptionColletion.TryAdd(subscription3);
|
||||
subscriptionColletion.TryAdd(subscription4);
|
||||
subscriptionColletion.TryAdd(subscription5);
|
||||
subscriptionColletion.TryAdd(subscription6);
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription4, subscription2, subscription3, subscription6, subscription5, subscription });
|
||||
|
||||
subscriptionColletion.TryRemove(subscription2.Configuration, out subscription2);
|
||||
Assert.AreEqual(subscriptionColletion.Select(x => x.Configuration.SecurityType).ToList(), new[] { SecurityType.Equity, SecurityType.Option,
|
||||
SecurityType.Option, SecurityType.Option, SecurityType.Future });
|
||||
|
||||
subscriptionColletion.TryRemove(subscription3.Configuration, out subscription3);
|
||||
Assert.AreEqual(subscriptionColletion.Select(x => x.Configuration.SecurityType).ToList(), new[] { SecurityType.Equity, SecurityType.Option, SecurityType.Option, SecurityType.Future });
|
||||
|
||||
subscriptionColletion.TryRemove(subscription.Configuration, out subscription);
|
||||
Assert.AreEqual(subscriptionColletion.Select(x => x.Configuration.SecurityType).ToList(), new[] { SecurityType.Equity, SecurityType.Option, SecurityType.Option });
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription4, subscription6, subscription5 });
|
||||
|
||||
subscriptionColletion.TryRemove(subscription6.Configuration, out subscription6);
|
||||
Assert.AreEqual(subscriptionColletion.Select(x => x.Configuration.SecurityType).ToList(), new[] { SecurityType.Equity, SecurityType.Option });
|
||||
Assert.AreEqual(subscriptionColletion.ToList(), new[] { subscription4, subscription5 });
|
||||
|
||||
subscriptionColletion.TryRemove(subscription5.Configuration, out subscription5);
|
||||
Assert.AreEqual(subscriptionColletion.Select(x => x.Configuration.SecurityType).ToList(), new[] {SecurityType.Equity});
|
||||
|
||||
subscriptionColletion.TryRemove(subscription4.Configuration, out subscription4);
|
||||
Assert.IsTrue(subscriptionColletion.Select(x => x.Configuration.SecurityType).ToList().IsNullOrEmpty());
|
||||
|
||||
subscription.Dispose();
|
||||
subscription2.Dispose();
|
||||
subscription3.Dispose();
|
||||
subscription4.Dispose();
|
||||
subscription5.Dispose();
|
||||
subscription6.Dispose();
|
||||
}
|
||||
|
||||
private Subscription CreateSubscription(Resolution resolution, string symbol = "AAPL", bool isInternalFeed = false,
|
||||
SecurityType type = SecurityType.Equity, TickType tickType = TickType.Trade)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
var end = start.AddSeconds(10);
|
||||
Security security;
|
||||
Symbol _symbol;
|
||||
if (type == SecurityType.Equity)
|
||||
{
|
||||
_symbol = new Symbol(SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA), symbol);
|
||||
security = new Equity(
|
||||
_symbol,
|
||||
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
|
||||
new Cash(Currencies.USD, 0, 1),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
}
|
||||
else if (type == SecurityType.Option)
|
||||
{
|
||||
_symbol = Symbol.CreateOption(
|
||||
new Symbol(SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA), symbol),
|
||||
Market.USA,
|
||||
OptionStyle.American,
|
||||
OptionRight.Call,
|
||||
0m,
|
||||
DateTime.Now
|
||||
);
|
||||
security = new Option(
|
||||
_symbol,
|
||||
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
|
||||
new Cash(Currencies.USD, 0, 1),
|
||||
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache(),
|
||||
null
|
||||
);
|
||||
}
|
||||
else if (type == SecurityType.Future)
|
||||
{
|
||||
_symbol = new Symbol(SecurityIdentifier.GenerateFuture(DateTime.Now, symbol, Market.COMEX), symbol);
|
||||
security = new Future(
|
||||
_symbol,
|
||||
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
|
||||
new Cash(Currencies.USD, 0, 1),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new RegressionTestException("SecurityType not implemented");
|
||||
}
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar), _symbol, resolution, DateTimeZone.Utc, DateTimeZone.Utc, true, false, isInternalFeed, false, tickType);
|
||||
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, start, end);
|
||||
# pragma warning disable CA2000
|
||||
var enumerator = new EnqueueableEnumerator<BaseData>();
|
||||
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, enumerator, false, false);
|
||||
# pragma warning restore CA2000
|
||||
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
|
||||
return new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Util;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class SubscriptionDataReaderTests
|
||||
{
|
||||
// this is the core of this unit test:
|
||||
// the stream will have two data points one of which does not correspond to the tradable date,
|
||||
// and it shouldn't be emitted because we have a new source for the next tradable date
|
||||
[TestCase(@"25980000,1679600,1679700,1679600,1679700,200
|
||||
99900000,1679400,1679400,1679200,1679200,600", false, Resolution.Minute)]
|
||||
// this test case has two data point which should be emitted because they correspond to the same tradable date
|
||||
[TestCase(@"25980000,1679600,1679700,1679600,1679700,200
|
||||
30980000,1679400,1679400,1679200,1679200,600", true, Resolution.Minute)]
|
||||
// even if the second data point is another tradable date we emit it because daily resolution
|
||||
// always uses the same source
|
||||
[TestCase(@"20191209 00:00,956900,959700,947200,958100,3647000
|
||||
20191211 00:00,956900,959700,947200,958100,3647000", true, Resolution.Daily)]
|
||||
public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecondDataPoint, Resolution dataResolution)
|
||||
{
|
||||
var start = new DateTime(2019, 12, 9);
|
||||
var end = new DateTime(2019, 12, 12);
|
||||
|
||||
var request = GetRequest(typeof(TradeBar), start, end, dataResolution, out var config);
|
||||
using var testDataCacheProvider = new TestDataCacheProvider() { Data = data };
|
||||
using var dataReader = new SubscriptionDataReader(config,
|
||||
request,
|
||||
TestGlobals.MapFileProvider,
|
||||
TestGlobals.FactorFileProvider,
|
||||
testDataCacheProvider,
|
||||
TestGlobals.DataProvider,
|
||||
null);
|
||||
|
||||
Assert.IsTrue(dataReader.MoveNext());
|
||||
Assert.AreEqual(shouldEmitSecondDataPoint, dataReader.MoveNext());
|
||||
|
||||
}
|
||||
|
||||
[TestCase(typeof(TradeBar))]
|
||||
[TestCase(typeof(QuoteBar))]
|
||||
[TestCase(typeof(OpenInterest))]
|
||||
public void EmitsNewTradableDateWhenDateAfterDelistingIsNonTradable(Type dataType)
|
||||
{
|
||||
var start = new DateTime(2023, 06, 30);
|
||||
var end = new DateTime(2023, 08, 01);
|
||||
|
||||
var request = GetRequest(dataType, start, end, Resolution.Minute, out var config);
|
||||
using var testDataCacheProvider = new TestDataCacheProvider();
|
||||
using var dataReader = new SubscriptionDataReader(config,
|
||||
request,
|
||||
TestGlobals.MapFileProvider,
|
||||
TestGlobals.FactorFileProvider,
|
||||
testDataCacheProvider,
|
||||
TestGlobals.DataProvider,
|
||||
null);
|
||||
|
||||
var expectedLastTradableDate = new DateTime(2023, 07, 05);
|
||||
var lastTradableDate = default(DateTime);
|
||||
|
||||
dataReader.NewTradableDate += (sender, args) =>
|
||||
{
|
||||
lastTradableDate = args.Date;
|
||||
};
|
||||
|
||||
while (dataReader.MoveNext())
|
||||
{
|
||||
}
|
||||
|
||||
Assert.AreEqual(expectedLastTradableDate, lastTradableDate);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DoesNotYieldDataWhenDelisted()
|
||||
{
|
||||
var start = new DateTime(2023, 8, 1);
|
||||
var end = new DateTime(2023, 08, 10);
|
||||
|
||||
var request = GetRequest(typeof(TradeBar), start, end, Resolution.Minute, out var config);
|
||||
using var testDataCacheProvider = new TestDataCacheProvider();
|
||||
using var dataReader = new SubscriptionDataReader(config,
|
||||
request,
|
||||
TestGlobals.MapFileProvider,
|
||||
TestGlobals.FactorFileProvider,
|
||||
testDataCacheProvider,
|
||||
TestGlobals.DataProvider,
|
||||
null);
|
||||
|
||||
var dataYielded = false;
|
||||
var newTradableDateCalled = false;
|
||||
dataReader.NewTradableDate += (sender, args) =>
|
||||
{
|
||||
newTradableDateCalled = true;
|
||||
};
|
||||
|
||||
while (dataReader.MoveNext())
|
||||
{
|
||||
dataYielded = true;
|
||||
}
|
||||
|
||||
Assert.IsFalse(dataYielded);
|
||||
Assert.IsFalse(newTradableDateCalled);
|
||||
}
|
||||
|
||||
private static BaseDataRequest GetRequest(Type dataType, DateTime start, DateTime end, Resolution resolution, out SubscriptionDataConfig config)
|
||||
{
|
||||
var symbol = Symbol.CreateOption(
|
||||
Symbols.SPX,
|
||||
"SPXW",
|
||||
Market.USA,
|
||||
OptionStyle.European,
|
||||
OptionRight.Call,
|
||||
4445m,
|
||||
// Next day is a holiday
|
||||
new DateTime(2023, 7, 3));
|
||||
|
||||
var entry = MarketHoursDatabase.FromDataFolder().GetEntry(symbol.ID.Market, symbol, symbol.SecurityType);
|
||||
config = new SubscriptionDataConfig(dataType,
|
||||
symbol,
|
||||
resolution,
|
||||
entry.DataTimeZone,
|
||||
entry.ExchangeHours.TimeZone,
|
||||
false,
|
||||
false,
|
||||
false);
|
||||
return new HistoryRequest(config, entry.ExchangeHours, start, end);
|
||||
}
|
||||
|
||||
private class TestDataCacheProvider : IDataCacheProvider
|
||||
{
|
||||
private StreamWriter _writer;
|
||||
private bool _alreadyEmitted;
|
||||
|
||||
public string Data { get; set; }
|
||||
public void Dispose()
|
||||
{
|
||||
_writer.DisposeSafely();
|
||||
}
|
||||
public List<string> GetZipEntries(string zipFile)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
public bool IsDataEphemeral => true;
|
||||
public Stream Fetch(string key)
|
||||
{
|
||||
if (_alreadyEmitted)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
_alreadyEmitted = true;
|
||||
|
||||
var stream = new MemoryStream();
|
||||
_writer = new StreamWriter(stream);
|
||||
_writer.Write(Data);
|
||||
_writer.Flush();
|
||||
stream.Position = 0;
|
||||
return stream;
|
||||
}
|
||||
public void Store(string key, byte[] data)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Securities;
|
||||
using System;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class SubscriptionDataTests
|
||||
{
|
||||
[Test]
|
||||
public void CreatedSubscriptionRoundsTimeDownForDataWithPeriod()
|
||||
{
|
||||
var tb = new TradeBar
|
||||
{
|
||||
Time = new DateTime(2020, 5, 21, 8, 9, 0),
|
||||
Period = TimeSpan.FromHours(1),
|
||||
Symbol = Symbols.SPY
|
||||
};
|
||||
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Hour,
|
||||
TimeZones.Utc,
|
||||
TimeZones.Utc,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
);
|
||||
|
||||
var exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.Utc);
|
||||
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.Utc, new DateTime(2020, 5, 21), new DateTime(2020, 5, 22));
|
||||
|
||||
var subscription = SubscriptionData.Create(false, config, exchangeHours, offsetProvider, tb, config.DataNormalizationMode);
|
||||
|
||||
Assert.AreEqual(new DateTime(2020, 5, 21, 8, 0, 0), subscription.Data.Time);
|
||||
Assert.AreEqual(new DateTime(2020, 5, 21, 9, 0, 0), subscription.Data.EndTime);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CreatedSubscriptionDoesNotRoundDownForPeriodLessData()
|
||||
{
|
||||
var data = new MyCustomData
|
||||
{
|
||||
Time = new DateTime(2020, 5, 21, 8, 9, 0),
|
||||
Symbol = Symbols.SPY
|
||||
};
|
||||
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Hour,
|
||||
TimeZones.Utc,
|
||||
TimeZones.Utc,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
);
|
||||
|
||||
var exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.Utc);
|
||||
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.Utc, new DateTime(2020, 5, 21), new DateTime(2020, 5, 22));
|
||||
|
||||
var subscription = SubscriptionData.Create(false, config, exchangeHours, offsetProvider, data, config.DataNormalizationMode);
|
||||
|
||||
Assert.AreEqual(new DateTime(2020, 5, 21, 8, 9, 0), subscription.Data.Time);
|
||||
Assert.AreEqual(new DateTime(2020, 5, 21, 8, 9, 0), subscription.Data.EndTime);
|
||||
}
|
||||
|
||||
[TestCase(1, 0)]
|
||||
[TestCase(null, 0)]
|
||||
[TestCase(null, 1000)]
|
||||
public void CreateDefaults(decimal? scale, decimal dividends)
|
||||
{
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Hour,
|
||||
TimeZones.Utc,
|
||||
TimeZones.Utc,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
);
|
||||
|
||||
config.SumOfDividends = dividends;
|
||||
|
||||
var tb = new TradeBar
|
||||
{
|
||||
Time = new DateTime(2020, 5, 21, 8, 9, 0),
|
||||
Period = TimeSpan.FromHours(1),
|
||||
Symbol = Symbols.SPY,
|
||||
Open = 100,
|
||||
High = 200,
|
||||
Low = 300,
|
||||
Close = 400
|
||||
};
|
||||
|
||||
var data = SubscriptionData.Create(false,
|
||||
config,
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
new TimeZoneOffsetProvider(TimeZones.NewYork, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1)),
|
||||
tb,
|
||||
config.DataNormalizationMode,
|
||||
scale);
|
||||
|
||||
Assert.True(data.GetType() == typeof(SubscriptionData));
|
||||
|
||||
Assert.AreEqual(tb.Open, (data.Data as TradeBar).Open);
|
||||
Assert.AreEqual(tb.High, (data.Data as TradeBar).High);
|
||||
Assert.AreEqual(tb.Low, (data.Data as TradeBar).Low);
|
||||
Assert.AreEqual(tb.Close, (data.Data as TradeBar).Close);
|
||||
}
|
||||
|
||||
[TestCase(typeof(SubscriptionData), 1)]
|
||||
[TestCase(typeof(PrecalculatedSubscriptionData), 2)]
|
||||
[TestCase(typeof(PrecalculatedSubscriptionData), 0.5)]
|
||||
public void CreateZeroDividends(Type type, decimal? scale)
|
||||
{
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Hour,
|
||||
TimeZones.Utc,
|
||||
TimeZones.Utc,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
);
|
||||
|
||||
config.SumOfDividends = 0;
|
||||
|
||||
var tb = new TradeBar
|
||||
{
|
||||
Time = new DateTime(2020, 5, 21, 8, 9, 0),
|
||||
Period = TimeSpan.FromHours(1),
|
||||
Symbol = Symbols.SPY,
|
||||
Open = 100,
|
||||
High = 200,
|
||||
Low = 300,
|
||||
Close = 400
|
||||
};
|
||||
|
||||
var data = SubscriptionData.Create(false,
|
||||
config,
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
new TimeZoneOffsetProvider(TimeZones.NewYork, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1)),
|
||||
tb,
|
||||
config.DataNormalizationMode,
|
||||
scale);
|
||||
|
||||
Assert.True(data.GetType() == type);
|
||||
|
||||
Assert.AreEqual(tb.Open * scale, (data.Data as TradeBar).Open);
|
||||
Assert.AreEqual(tb.High * scale, (data.Data as TradeBar).High);
|
||||
Assert.AreEqual(tb.Low * scale, (data.Data as TradeBar).Low);
|
||||
Assert.AreEqual(tb.Close * scale, (data.Data as TradeBar).Close);
|
||||
}
|
||||
|
||||
[TestCase(typeof(PrecalculatedSubscriptionData), 1)]
|
||||
[TestCase(typeof(PrecalculatedSubscriptionData), 2)]
|
||||
[TestCase(typeof(PrecalculatedSubscriptionData), 0.5)]
|
||||
public void CreateAdjustedNotZeroDividends(Type type, decimal? scale)
|
||||
{
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Hour,
|
||||
TimeZones.Utc,
|
||||
TimeZones.Utc,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
);
|
||||
|
||||
config.SumOfDividends = 100;
|
||||
|
||||
var tb = new TradeBar
|
||||
{
|
||||
Time = new DateTime(2020, 5, 21, 8, 9, 0),
|
||||
Period = TimeSpan.FromHours(1),
|
||||
Symbol = Symbols.SPY,
|
||||
Open = 100,
|
||||
High = 200,
|
||||
Low = 300,
|
||||
Close = 400
|
||||
};
|
||||
|
||||
var data = SubscriptionData.Create(false,
|
||||
config,
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
new TimeZoneOffsetProvider(TimeZones.NewYork, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1)),
|
||||
tb,
|
||||
config.DataNormalizationMode,
|
||||
scale);
|
||||
|
||||
Assert.True(data.GetType() == type);
|
||||
|
||||
Assert.AreEqual(tb.Open * scale, (data.Data as TradeBar).Open);
|
||||
Assert.AreEqual(tb.High * scale, (data.Data as TradeBar).High);
|
||||
Assert.AreEqual(tb.Low * scale, (data.Data as TradeBar).Low);
|
||||
Assert.AreEqual(tb.Close * scale, (data.Data as TradeBar).Close);
|
||||
}
|
||||
|
||||
[TestCase(typeof(PrecalculatedSubscriptionData), 1)]
|
||||
[TestCase(typeof(PrecalculatedSubscriptionData), 2)]
|
||||
[TestCase(typeof(PrecalculatedSubscriptionData), 0.5)]
|
||||
public void CreateTotalNotZeroDividends(Type type, decimal? scale)
|
||||
{
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Hour,
|
||||
TimeZones.Utc,
|
||||
TimeZones.Utc,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
);
|
||||
|
||||
config.SumOfDividends = 100;
|
||||
config.DataNormalizationMode = DataNormalizationMode.TotalReturn;
|
||||
|
||||
var tb = new TradeBar
|
||||
{
|
||||
Time = new DateTime(2020, 5, 21, 8, 9, 0),
|
||||
Period = TimeSpan.FromHours(1),
|
||||
Symbol = Symbols.SPY,
|
||||
Open = 100,
|
||||
High = 200,
|
||||
Low = 300,
|
||||
Close = 400
|
||||
};
|
||||
|
||||
var data = SubscriptionData.Create(false,
|
||||
config,
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
new TimeZoneOffsetProvider(TimeZones.NewYork, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1)),
|
||||
tb,
|
||||
config.DataNormalizationMode,
|
||||
scale);
|
||||
|
||||
Assert.True(data.GetType() == type);
|
||||
|
||||
Assert.AreEqual(tb.Open * scale + config.SumOfDividends, (data.Data as TradeBar).Open);
|
||||
Assert.AreEqual(tb.High * scale + config.SumOfDividends, (data.Data as TradeBar).High);
|
||||
Assert.AreEqual(tb.Low * scale + config.SumOfDividends, (data.Data as TradeBar).Low);
|
||||
Assert.AreEqual(tb.Close * scale + config.SumOfDividends, (data.Data as TradeBar).Close);
|
||||
}
|
||||
|
||||
[TestCase(true, typeof(TradeBar))]
|
||||
[TestCase(false, typeof(TradeBar))]
|
||||
[TestCase(true, typeof(QuoteBar))]
|
||||
[TestCase(false, typeof(QuoteBar))]
|
||||
[TestCase(true, typeof(Tick))]
|
||||
[TestCase(false, typeof(Tick))]
|
||||
public void FillForwardFlagIsCorrectlySet(bool isFillForward, Type type)
|
||||
{
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Hour,
|
||||
TimeZones.Utc,
|
||||
TimeZones.Utc,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
);
|
||||
|
||||
var scale = 0.5m;
|
||||
config.DataNormalizationMode = DataNormalizationMode.Adjusted;
|
||||
|
||||
var data = (BaseData)Activator.CreateInstance(type);
|
||||
if (isFillForward)
|
||||
{
|
||||
data = data.Clone(isFillForward);
|
||||
}
|
||||
|
||||
var subscriptionData = (PrecalculatedSubscriptionData) SubscriptionData.Create(false, config,
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
new TimeZoneOffsetProvider(TimeZones.NewYork, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1)),
|
||||
data,
|
||||
config.DataNormalizationMode,
|
||||
scale);
|
||||
|
||||
config.DataNormalizationMode = DataNormalizationMode.Raw;
|
||||
Assert.AreEqual(isFillForward, subscriptionData.Data.IsFillForward);
|
||||
|
||||
config.DataNormalizationMode = DataNormalizationMode.Adjusted;
|
||||
Assert.AreEqual(isFillForward, subscriptionData.Data.IsFillForward);
|
||||
}
|
||||
|
||||
internal class MyCustomData : BaseData
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Algorithm;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Category("TravisExclude"), Parallelizable(ParallelScope.All)]
|
||||
public class SubscriptionSynchronizerTests
|
||||
{
|
||||
[Test]
|
||||
[TestCase(1, Resolution.Second)]
|
||||
[TestCase(20, Resolution.Minute)]
|
||||
[TestCase(50, Resolution.Minute)]
|
||||
[TestCase(100, Resolution.Minute)]
|
||||
[TestCase(250, Resolution.Minute)]
|
||||
[TestCase(500, Resolution.Hour)]
|
||||
[TestCase(1000, Resolution.Hour)]
|
||||
public void SubscriptionSynchronizerPerformance(int securityCount, Resolution resolution)
|
||||
{
|
||||
// since data is pre-generated, it's important to use the larger resolutions with large security counts
|
||||
|
||||
var algorithm = PerformanceBenchmarkAlgorithms.CreateBenchmarkAlgorithm(securityCount, resolution);
|
||||
TestSubscriptionSynchronizerSpeed(algorithm);
|
||||
}
|
||||
|
||||
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm)
|
||||
{
|
||||
var feed = new MockDataFeed();
|
||||
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
|
||||
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
|
||||
var securityService = new SecurityService(
|
||||
algorithm.Portfolio.CashBook,
|
||||
marketHoursDatabase,
|
||||
symbolPropertiesDataBase,
|
||||
algorithm,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCacheProvider(algorithm.Portfolio),
|
||||
algorithm: algorithm);
|
||||
algorithm.Securities.SetSecurityService(securityService);
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
var dataManager = new DataManager(feed,
|
||||
new UniverseSelection(algorithm, securityService, dataPermissionManager, TestGlobals.DataProvider),
|
||||
algorithm,
|
||||
algorithm.TimeKeeper,
|
||||
marketHoursDatabase,
|
||||
false,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
dataPermissionManager);
|
||||
algorithm.SubscriptionManager.SetDataManager(dataManager);
|
||||
|
||||
algorithm.Initialize();
|
||||
algorithm.PostInitialize();
|
||||
|
||||
// set exchanges to be always open
|
||||
foreach (var kvp in algorithm.Securities)
|
||||
{
|
||||
var security = kvp.Value;
|
||||
security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone));
|
||||
}
|
||||
|
||||
var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork);
|
||||
var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork);
|
||||
var subscriptionBasedTimeProvider = new SubscriptionFrontierTimeProvider(startTimeUtc, dataManager);
|
||||
var timeSliceFactory = new TimeSliceFactory(algorithm.TimeZone);
|
||||
var synchronizer = new SubscriptionSynchronizer(dataManager.UniverseSelection, new());
|
||||
synchronizer.SetTimeProvider(subscriptionBasedTimeProvider);
|
||||
synchronizer.SetTimeSliceFactory(timeSliceFactory);
|
||||
var totalDataPoints = 0;
|
||||
var subscriptions = dataManager.DataFeedSubscriptions;
|
||||
foreach (var kvp in algorithm.Securities)
|
||||
{
|
||||
int dataPointCount;
|
||||
# pragma warning disable CA2000
|
||||
var subscription = CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount);
|
||||
# pragma warning restore CA2000
|
||||
subscriptions.TryAdd(subscription);
|
||||
totalDataPoints += dataPointCount;
|
||||
}
|
||||
|
||||
// log what we're doing
|
||||
Log.Trace($"Running {subscriptions.Count()} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}");
|
||||
|
||||
var count = 0;
|
||||
DateTime currentTime = DateTime.MaxValue;
|
||||
DateTime previousValue;
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var enumerator = synchronizer.Sync(subscriptions, CancellationToken.None).GetEnumerator();
|
||||
do
|
||||
{
|
||||
previousValue = currentTime;
|
||||
enumerator.MoveNext();
|
||||
var timeSlice = enumerator.Current;
|
||||
currentTime = timeSlice.Time;
|
||||
count += timeSlice.DataPointCount;
|
||||
}
|
||||
while (currentTime != previousValue);
|
||||
|
||||
stopwatch.Stop();
|
||||
enumerator.DisposeSafely();
|
||||
|
||||
var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds;
|
||||
Log.Trace($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}");
|
||||
Assert.GreaterOrEqual(count, 100); // this assert is for sanity purpose
|
||||
dataManager.RemoveAllSubscriptions();
|
||||
}
|
||||
|
||||
private Subscription CreateSubscription(QCAlgorithm algorithm, Security security, DateTime startTimeUtc, DateTime endTimeUtc, out int dataPointCount)
|
||||
{
|
||||
var universe = algorithm.UniverseManager.Values.OfType<UserDefinedUniverse>()
|
||||
.Single(u => u.SelectSymbols(default(DateTime), null).Contains(security.Symbol));
|
||||
|
||||
var config = security.Subscriptions.First();
|
||||
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, startTimeUtc, endTimeUtc);
|
||||
var data = LinqExtensions.Range(algorithm.StartDate, algorithm.EndDate, c => c + config.Increment).Select(time => new DataPoint
|
||||
{
|
||||
Time = time,
|
||||
EndTime = time + config.Increment
|
||||
})
|
||||
.Select(d => SubscriptionData.Create(false, config, security.Exchange.Hours, offsetProvider, d, config.DataNormalizationMode))
|
||||
.ToList();
|
||||
|
||||
dataPointCount = data.Count;
|
||||
var subscriptionRequest = new SubscriptionRequest(false, universe, security, config, startTimeUtc, endTimeUtc);
|
||||
return new Subscription(subscriptionRequest, data.GetEnumerator(), offsetProvider);
|
||||
}
|
||||
|
||||
private class DataPoint : BaseData
|
||||
{
|
||||
// bare bones base data to minimize memory footprint
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NodaTime;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Algorithm.Framework.Selection;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Tests.Common.Securities;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class SubscriptionTests
|
||||
{
|
||||
private readonly DateTime _start = DateTime.MinValue;
|
||||
private readonly DateTime _end = DateTime.MinValue.AddDays(1);
|
||||
|
||||
[Test]
|
||||
public void ConstructorNoUniverse()
|
||||
{
|
||||
var subscriptionRequest = GetSubscriptionRequest(false);
|
||||
using var subscription = new Subscription(
|
||||
subscriptionRequest,
|
||||
null,
|
||||
new TimeZoneOffsetProvider(DateTimeZone.Utc, _start, _end));
|
||||
|
||||
Assert.IsFalse(subscription.Universes.Any());
|
||||
Assert.IsFalse(subscription.EndOfStream);
|
||||
Assert.AreEqual(_start, subscription.UtcStartTime);
|
||||
Assert.AreEqual(_end, subscription.UtcEndTime);
|
||||
Assert.AreEqual(subscription.Configuration, subscriptionRequest.Configuration);
|
||||
Assert.IsFalse(subscription.IsUniverseSelectionSubscription);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Constructor()
|
||||
{
|
||||
var subscriptionRequest = GetSubscriptionRequest();
|
||||
using var subscription = new Subscription(
|
||||
subscriptionRequest,
|
||||
null,
|
||||
new TimeZoneOffsetProvider(DateTimeZone.Utc, _start, _end));
|
||||
|
||||
Assert.AreEqual(1, subscription.Universes.Count());
|
||||
Assert.AreEqual(_start, subscription.UtcStartTime);
|
||||
Assert.AreEqual(_end, subscription.UtcEndTime);
|
||||
Assert.AreEqual(subscriptionRequest.Universe, subscription.Universes.First());
|
||||
Assert.IsFalse(subscription.EndOfStream);
|
||||
Assert.AreEqual(subscription.Configuration, subscriptionRequest.Configuration);
|
||||
Assert.IsFalse(subscription.IsUniverseSelectionSubscription);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AddSubscriptionRequestOncePerUniverse()
|
||||
{
|
||||
var subscriptionRequest = GetSubscriptionRequest();
|
||||
using var subscription = new Subscription(
|
||||
subscriptionRequest,
|
||||
null,
|
||||
new TimeZoneOffsetProvider(DateTimeZone.Utc, _start, _end));
|
||||
|
||||
Assert.AreEqual(1, subscription.Universes.Count());
|
||||
Assert.AreEqual(subscriptionRequest.Universe, subscription.Universes.First());
|
||||
|
||||
subscription.AddSubscriptionRequest(subscriptionRequest);
|
||||
|
||||
Assert.AreEqual(1, subscription.Universes.Count());
|
||||
Assert.AreEqual(subscriptionRequest.Universe, subscription.Universes.First());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SubscriptionOnlyAcceptsOneUniverseSelection()
|
||||
{
|
||||
var subscriptionRequest = GetSubscriptionRequest(true, true);
|
||||
var subscriptionRequest2 = GetSubscriptionRequest();
|
||||
using var subscription = new Subscription(
|
||||
subscriptionRequest,
|
||||
null,
|
||||
new TimeZoneOffsetProvider(DateTimeZone.Utc, _start, _end));
|
||||
|
||||
try
|
||||
{
|
||||
subscription.AddSubscriptionRequest(subscriptionRequest2);
|
||||
Assert.Fail("Subscription should only accept one universe selection" +
|
||||
" subscription request.");
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
Assert.Pass();
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SubscriptionOnlyAcceptsSameConfiguration()
|
||||
{
|
||||
var subscriptionRequest = GetSubscriptionRequest();
|
||||
var subscriptionRequest2 = GetSubscriptionRequest(resolution: Resolution.Second);
|
||||
using var subscription = new Subscription(
|
||||
subscriptionRequest,
|
||||
null,
|
||||
new TimeZoneOffsetProvider(DateTimeZone.Utc, _start, _end));
|
||||
|
||||
try
|
||||
{
|
||||
subscription.AddSubscriptionRequest(subscriptionRequest2);
|
||||
Assert.Fail("Subscription should only accept the same configuration");
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
Assert.Pass();
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RemoveSubscriptionRequest()
|
||||
{
|
||||
var subscriptionRequest = GetSubscriptionRequest();
|
||||
using var subscription = new Subscription(
|
||||
subscriptionRequest,
|
||||
null,
|
||||
new TimeZoneOffsetProvider(DateTimeZone.Utc, _start, _end));
|
||||
|
||||
var emptySubscription = subscription.RemoveSubscriptionRequest(subscriptionRequest.Universe);
|
||||
|
||||
Assert.IsTrue(emptySubscription);
|
||||
Assert.IsFalse(subscription.Universes.Any());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RemoveAllSubscriptionRequest()
|
||||
{
|
||||
var subscriptionRequest = GetSubscriptionRequest();
|
||||
var subscriptionRequest2 = GetSubscriptionRequest();
|
||||
using var subscription = new Subscription(
|
||||
subscriptionRequest,
|
||||
null,
|
||||
new TimeZoneOffsetProvider(DateTimeZone.Utc, _start, _end));
|
||||
|
||||
subscription.AddSubscriptionRequest(subscriptionRequest2);
|
||||
Assert.AreEqual(2, subscription.Universes.Count());
|
||||
var emptySubscription = subscription.RemoveSubscriptionRequest();
|
||||
|
||||
Assert.IsTrue(emptySubscription);
|
||||
Assert.IsFalse(subscription.Universes.Any());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RemoveAllSubscriptionRequestNoUniverse()
|
||||
{
|
||||
var subscriptionRequest = GetSubscriptionRequest(false);
|
||||
using var subscription = new Subscription(
|
||||
subscriptionRequest,
|
||||
null,
|
||||
new TimeZoneOffsetProvider(DateTimeZone.Utc, _start, _end));
|
||||
|
||||
var emptySubscription = subscription.RemoveSubscriptionRequest();
|
||||
|
||||
Assert.IsTrue(emptySubscription);
|
||||
Assert.IsFalse(subscription.Universes.Any());
|
||||
}
|
||||
|
||||
private SubscriptionRequest GetSubscriptionRequest(
|
||||
bool useUniverse = true,
|
||||
bool isUniverseSelection = false,
|
||||
Resolution resolution = Resolution.Minute)
|
||||
{
|
||||
var security = SecurityTests.GetSecurity();
|
||||
var config = SecurityTests.CreateTradeBarConfig(resolution);
|
||||
# pragma warning disable CA2000
|
||||
var universe = new ManualUniverse(
|
||||
config,
|
||||
new UniverseSettings(Resolution.Daily, 1, true, true, TimeSpan.FromDays(1)),
|
||||
new[] {security.Symbol}
|
||||
);
|
||||
#pragma warning restore CA2000
|
||||
return new SubscriptionRequest(isUniverseSelection, useUniverse ? universe : null, security, config, _start, _end);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.All)]
|
||||
public class SubscriptionUtilsTests
|
||||
{
|
||||
private Security _security;
|
||||
private SubscriptionDataConfig _config;
|
||||
private IFactorFileProvider _factorFileProvider;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_security = new Security(Symbols.SPY,
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
new Cash(Currencies.USD, 0, 0),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
new IdentityCurrencyConverter(Currencies.USD),
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache());
|
||||
|
||||
_config = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true, true, false);
|
||||
|
||||
_factorFileProvider = TestGlobals.FactorFileProvider;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SubscriptionIsDisposed()
|
||||
{
|
||||
var dataPoints = 10;
|
||||
using var enumerator = new TestDataEnumerator { MoveNextTrueCount = dataPoints };
|
||||
#pragma warning disable CA2000
|
||||
var subscription = SubscriptionUtils.CreateAndScheduleWorker(
|
||||
new SubscriptionRequest(
|
||||
false,
|
||||
null,
|
||||
_security,
|
||||
_config,
|
||||
DateTime.UtcNow,
|
||||
Time.EndOfTime
|
||||
),
|
||||
enumerator,
|
||||
_factorFileProvider,
|
||||
false, false);
|
||||
#pragma warning restore CA2000
|
||||
|
||||
var count = 0;
|
||||
while (enumerator.MoveNextTrueCount > 8)
|
||||
{
|
||||
if (count++ > 100)
|
||||
{
|
||||
Assert.Fail($"Timeout waiting for producer. {enumerator.MoveNextTrueCount}");
|
||||
}
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
|
||||
subscription.DisposeSafely();
|
||||
Assert.IsFalse(subscription.MoveNext());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThrowingEnumeratorStackDisposesOfSubscription()
|
||||
{
|
||||
using var enumerator = new TestDataEnumerator { MoveNextTrueCount = 10, ThrowException = true};
|
||||
|
||||
using var subscription = SubscriptionUtils.CreateAndScheduleWorker(
|
||||
new SubscriptionRequest(
|
||||
false,
|
||||
null,
|
||||
_security,
|
||||
_config,
|
||||
DateTime.UtcNow,
|
||||
Time.EndOfTime
|
||||
),
|
||||
enumerator,
|
||||
_factorFileProvider,
|
||||
false, false);
|
||||
|
||||
var count = 0;
|
||||
while (enumerator.MoveNextTrueCount != 9)
|
||||
{
|
||||
if (count++ > 100)
|
||||
{
|
||||
Assert.Fail("Timeout waiting for producer");
|
||||
}
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
|
||||
Assert.IsFalse(subscription.MoveNext());
|
||||
Assert.IsTrue(subscription.EndOfStream);
|
||||
|
||||
// enumerator is disposed by the producer
|
||||
count = 0;
|
||||
while (!enumerator.Disposed)
|
||||
{
|
||||
if (count++ > 100)
|
||||
{
|
||||
Assert.Fail("Timeout waiting for producer");
|
||||
}
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
// This unit tests reproduces GH 3885 where the consumer hanged forever
|
||||
public void ConsumerDoesNotHang()
|
||||
{
|
||||
for (var i = 0; i < 10000; i++)
|
||||
{
|
||||
var dataPoints = 10;
|
||||
|
||||
using var enumerator = new TestDataEnumerator {MoveNextTrueCount = dataPoints};
|
||||
#pragma warning disable CA2000
|
||||
var subscription = SubscriptionUtils.CreateAndScheduleWorker(
|
||||
new SubscriptionRequest(
|
||||
false,
|
||||
null,
|
||||
_security,
|
||||
_config,
|
||||
DateTime.UtcNow,
|
||||
Time.EndOfTime
|
||||
),
|
||||
enumerator,
|
||||
_factorFileProvider,
|
||||
false, false);
|
||||
#pragma warning restore CA2000
|
||||
for (var j = 0; j < dataPoints; j++)
|
||||
{
|
||||
Assert.IsTrue(subscription.MoveNext());
|
||||
}
|
||||
Assert.IsFalse(subscription.MoveNext());
|
||||
subscription.DisposeSafely();
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PriceScaleFirstFillForwardBar()
|
||||
{
|
||||
var referenceTime = new DateTime(2020, 08, 06);
|
||||
var point = new Tick(referenceTime, Symbols.SPY, 1, 2);
|
||||
var point2 = point.Clone(true);
|
||||
point2.Time = referenceTime;
|
||||
var point3 = point.Clone(false);
|
||||
point3.Time = referenceTime.AddDays(1);
|
||||
;
|
||||
var enumerator = new List<BaseData> { point2, point3 }.GetEnumerator();
|
||||
var factorFileProfider = new Mock<IFactorFileProvider>();
|
||||
|
||||
var factorFile = new CorporateFactorProvider(_security.Symbol.Value, new[]
|
||||
{
|
||||
new CorporateFactorRow(referenceTime, 0.5m, 1),
|
||||
new CorporateFactorRow(referenceTime.AddDays(1), 1m, 1)
|
||||
}, referenceTime);
|
||||
|
||||
factorFileProfider.Setup(s => s.Get(It.IsAny<Symbol>())).Returns(factorFile);
|
||||
#pragma warning disable CA2000
|
||||
var subscription = SubscriptionUtils.CreateAndScheduleWorker(
|
||||
new SubscriptionRequest(
|
||||
false,
|
||||
null,
|
||||
_security,
|
||||
_config,
|
||||
referenceTime,
|
||||
Time.EndOfTime
|
||||
),
|
||||
enumerator,
|
||||
factorFileProfider.Object,
|
||||
true, false);
|
||||
#pragma warning restore CA2000
|
||||
Assert.IsTrue(subscription.MoveNext());
|
||||
// we do expect it to pick up the prev factor file scale
|
||||
Assert.AreEqual(1, (subscription.Current.Data as Tick).AskPrice);
|
||||
Assert.IsTrue((subscription.Current.Data as Tick).IsFillForward);
|
||||
|
||||
Assert.IsTrue(subscription.MoveNext());
|
||||
Assert.AreEqual(2, (subscription.Current.Data as Tick).AskPrice);
|
||||
Assert.IsFalse((subscription.Current.Data as Tick).IsFillForward);
|
||||
|
||||
subscription.DisposeSafely();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PriceScaleDoesNotUpdateForFillForwardBar()
|
||||
{
|
||||
var referenceTime = new DateTime(2020, 08, 06);
|
||||
var point = new Tick(referenceTime, Symbols.SPY, 1, 2);
|
||||
var point2 = point.Clone(true);
|
||||
point2.Time = referenceTime.AddDays(1);
|
||||
var point3 = point.Clone(false);
|
||||
point3.Time = referenceTime.AddDays(2);
|
||||
var enumerator = new List<BaseData> { point, point2, point3 }.GetEnumerator();
|
||||
var factorFileProfider = new Mock<IFactorFileProvider>();
|
||||
|
||||
var factorFile = new CorporateFactorProvider(_security.Symbol.Value, new[]
|
||||
{
|
||||
new CorporateFactorRow(referenceTime, 0.5m, 1),
|
||||
new CorporateFactorRow(referenceTime.AddDays(1), 1m, 1)
|
||||
}, referenceTime);
|
||||
|
||||
factorFileProfider.Setup(s => s.Get(It.IsAny<Symbol>())).Returns(factorFile);
|
||||
#pragma warning disable CA2000
|
||||
var subscription = SubscriptionUtils.CreateAndScheduleWorker(
|
||||
new SubscriptionRequest(
|
||||
false,
|
||||
null,
|
||||
_security,
|
||||
_config,
|
||||
referenceTime,
|
||||
Time.EndOfTime
|
||||
),
|
||||
enumerator,
|
||||
factorFileProfider.Object,
|
||||
true, false);
|
||||
#pragma warning restore CA2000
|
||||
Assert.IsTrue(subscription.MoveNext());
|
||||
Assert.AreEqual(1, (subscription.Current.Data as Tick).AskPrice);
|
||||
Assert.IsFalse((subscription.Current.Data as Tick).IsFillForward);
|
||||
|
||||
Assert.IsTrue(subscription.MoveNext());
|
||||
Assert.AreEqual(1, (subscription.Current.Data as Tick).AskPrice);
|
||||
Assert.IsTrue((subscription.Current.Data as Tick).IsFillForward);
|
||||
|
||||
Assert.IsTrue(subscription.MoveNext());
|
||||
Assert.AreEqual(2, (subscription.Current.Data as Tick).AskPrice);
|
||||
Assert.IsFalse((subscription.Current.Data as Tick).IsFillForward);
|
||||
|
||||
subscription.DisposeSafely();
|
||||
}
|
||||
|
||||
[TestCase(typeof(TradeBar), true)]
|
||||
[TestCase(typeof(OpenInterest), false)]
|
||||
[TestCase(typeof(QuoteBar), false)]
|
||||
public void SubscriptionEmitsAuxData(Type typeOfConfig, bool shouldReceiveAuxData)
|
||||
{
|
||||
var config = new SubscriptionDataConfig(typeOfConfig, _security.Symbol, Resolution.Hour, TimeZones.NewYork, TimeZones.NewYork, true, true, false);
|
||||
|
||||
var totalPoints = 8;
|
||||
var time = new DateTime(2010, 1, 1);
|
||||
var enumerator = Enumerable.Range(0, totalPoints).Select(x => new Delisting { Time = time.AddHours(x) }).GetEnumerator();
|
||||
#pragma warning disable CA2000
|
||||
var subscription = SubscriptionUtils.CreateAndScheduleWorker(
|
||||
new SubscriptionRequest(
|
||||
false,
|
||||
null,
|
||||
_security,
|
||||
config,
|
||||
DateTime.UtcNow,
|
||||
Time.EndOfTime
|
||||
),
|
||||
enumerator,
|
||||
_factorFileProvider,
|
||||
false, false);
|
||||
#pragma warning restore CA2000
|
||||
// Test our subscription stream to see if it emits the aux data it should be filtered
|
||||
// by the SubscriptionUtils produce function if the config isn't for a TradeBar
|
||||
int dataReceivedCount = 0;
|
||||
while (subscription.MoveNext())
|
||||
{
|
||||
dataReceivedCount++;
|
||||
if (subscription.Current != null && subscription.Current.Data.DataType == MarketDataType.Auxiliary)
|
||||
{
|
||||
Assert.IsTrue(shouldReceiveAuxData);
|
||||
}
|
||||
}
|
||||
|
||||
// If it should receive aux data it should have emitted all points
|
||||
// otherwise none should have been emitted
|
||||
if (shouldReceiveAuxData)
|
||||
{
|
||||
Assert.AreEqual(totalPoints, dataReceivedCount);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.AreEqual(0, dataReceivedCount);
|
||||
}
|
||||
}
|
||||
|
||||
private class TestDataEnumerator : IEnumerator<BaseData>
|
||||
{
|
||||
public bool ThrowException { get; set; }
|
||||
public bool Disposed { get; set; }
|
||||
public int MoveNextTrueCount { get; set; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Disposed = true;
|
||||
}
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
Current = new Tick(DateTime.UtcNow,Symbols.SPY, 1, 2);
|
||||
var result = --MoveNextTrueCount >= 0;
|
||||
if (ThrowException)
|
||||
{
|
||||
throw new RegressionTestException("TestDataEnumerator.MoveNext()");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public void Reset()
|
||||
{
|
||||
}
|
||||
|
||||
public BaseData Current { get; set; }
|
||||
|
||||
object IEnumerator.Current => Current;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Accord.Math.Comparers;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Logging;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.Fixtures)]
|
||||
public class TextSubscriptionDataSourceReaderTests
|
||||
{
|
||||
private SubscriptionDataConfig _config;
|
||||
private DateTime _initialDate;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_config = new SubscriptionDataConfig(
|
||||
typeof(TestTradeBarFactory),
|
||||
Symbols.SPY,
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
_initialDate = new DateTime(2018, 1, 1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CachedDataIsReturnedAsClone()
|
||||
{
|
||||
using var singleEntryDataCacheProvider = new SingleEntryDataCacheProvider(TestGlobals.DataProvider);
|
||||
var reader = new TextSubscriptionDataSourceReader(
|
||||
singleEntryDataCacheProvider,
|
||||
_config,
|
||||
_initialDate,
|
||||
false,
|
||||
null);
|
||||
var source = (new TradeBar()).GetSource(_config, _initialDate, false);
|
||||
|
||||
var dataBars = reader.Read(source).First();
|
||||
dataBars.Value = 0;
|
||||
var dataBars2 = reader.Read(source).First();
|
||||
|
||||
Assert.AreNotEqual(dataBars.Price, dataBars2.Price);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DataIsNotCachedForEphemeralDataCacheProvider()
|
||||
{
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TestTradeBarFactory),
|
||||
Symbol.Create("SymbolNonEphemeralTest1", SecurityType.Equity, Market.USA),
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
using var dataCacheProvider = new CustomEphemeralDataCacheProvider { IsDataEphemeral = true};
|
||||
var reader = new TextSubscriptionDataSourceReader(
|
||||
dataCacheProvider,
|
||||
config,
|
||||
_initialDate,
|
||||
false,
|
||||
null);
|
||||
var source = (new TradeBar()).GetSource(config, _initialDate, false);
|
||||
dataCacheProvider.Data = "20000101 00:00,1,1,1,1,1";
|
||||
var dataBars = reader.Read(source).First();
|
||||
dataCacheProvider.Data = "20000101 00:00,2,2,2,2,2";
|
||||
var dataBars2 = reader.Read(source).First();
|
||||
|
||||
Assert.AreEqual(new DateTime(2000, 1, 1), dataBars.Time);
|
||||
Assert.AreEqual(new DateTime(2000, 1, 1), dataBars2.Time);
|
||||
Assert.AreNotEqual(dataBars.Price, dataBars2.Price);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DataIsCachedForNonEphemeralDataCacheProvider()
|
||||
{
|
||||
var config = new SubscriptionDataConfig(
|
||||
typeof(TestTradeBarFactory),
|
||||
Symbol.Create("SymbolNonEphemeralTest2", SecurityType.Equity, Market.USA),
|
||||
Resolution.Daily,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
using var dataCacheProvider = new CustomEphemeralDataCacheProvider { IsDataEphemeral = false };
|
||||
var reader = new TextSubscriptionDataSourceReader(
|
||||
dataCacheProvider,
|
||||
config,
|
||||
_initialDate,
|
||||
false,
|
||||
null);
|
||||
var source = (new TradeBar()).GetSource(config, _initialDate, false);
|
||||
dataCacheProvider.Data = "20000101 00:00,1,1,1,1,1";
|
||||
var dataBars = reader.Read(source).First();
|
||||
// even if the data changes it already cached
|
||||
dataCacheProvider.Data = "20000101 00:00,2,2,2,2,2";
|
||||
var dataBars2 = reader.Read(source).First();
|
||||
|
||||
Assert.AreEqual(new DateTime(2000, 1, 1), dataBars.Time);
|
||||
Assert.AreEqual(new DateTime(2000, 1, 1), dataBars2.Time);
|
||||
Assert.AreEqual(dataBars.Price, dataBars2.Price);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DataIsCachedCorrectly()
|
||||
{
|
||||
using var singleEntryDataCacheProvider = new SingleEntryDataCacheProvider(TestGlobals.DataProvider);
|
||||
var reader = new TextSubscriptionDataSourceReader(
|
||||
singleEntryDataCacheProvider,
|
||||
_config,
|
||||
_initialDate,
|
||||
false,
|
||||
null);
|
||||
var source = (new TradeBar()).GetSource(_config, _initialDate, false);
|
||||
|
||||
var dataBars = reader.Read(source).ToList();
|
||||
var dataBars2 = reader.Read(source).ToList();
|
||||
|
||||
Assert.AreEqual(dataBars2.Count, dataBars.Count);
|
||||
Assert.IsTrue(dataBars.SequenceEqual(dataBars2, new CustomComparer<BaseData>(
|
||||
(data, baseData) =>
|
||||
{
|
||||
if (data.EndTime == baseData.EndTime
|
||||
&& data.Time == baseData.Time
|
||||
&& data.Symbol == baseData.Symbol
|
||||
&& data.Price == baseData.Price
|
||||
&& data.DataType == baseData.DataType
|
||||
&& data.Value == baseData.Value)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
})));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RespectsInitialDate()
|
||||
{
|
||||
using var singleEntryDataCacheProvider = new SingleEntryDataCacheProvider(TestGlobals.DataProvider);
|
||||
var reader = new TextSubscriptionDataSourceReader(
|
||||
singleEntryDataCacheProvider,
|
||||
_config,
|
||||
_initialDate,
|
||||
false,
|
||||
null);
|
||||
var source = (new TradeBar()).GetSource(_config, _initialDate, false);
|
||||
var dataBars = reader.Read(source).First();
|
||||
|
||||
Assert.Less(dataBars.EndTime, _initialDate);
|
||||
|
||||
// 80 days after _initialDate
|
||||
var initialDate2 = _initialDate.AddDays(80);
|
||||
using var defaultDataProvider2 = new DefaultDataProvider();
|
||||
using var singleEntryDataCacheProvider2 = new SingleEntryDataCacheProvider(defaultDataProvider2);
|
||||
var reader2 = new TextSubscriptionDataSourceReader(
|
||||
singleEntryDataCacheProvider2,
|
||||
_config,
|
||||
initialDate2,
|
||||
false,
|
||||
null);
|
||||
var source2 = (new TradeBar()).GetSource(_config, initialDate2, false);
|
||||
var dataBars2 = reader2.Read(source2).First();
|
||||
|
||||
Assert.Less(dataBars2.EndTime, initialDate2);
|
||||
|
||||
// 80 days before _initialDate
|
||||
var initialDate3 = _initialDate.AddDays(-80);
|
||||
using var defaultDataProvider3 = new DefaultDataProvider();
|
||||
using var singleEntryDataCacheProvider3 = new SingleEntryDataCacheProvider(defaultDataProvider3);
|
||||
var reader3 = new TextSubscriptionDataSourceReader(
|
||||
singleEntryDataCacheProvider3,
|
||||
_config,
|
||||
initialDate3,
|
||||
false,
|
||||
null);
|
||||
var source3 = (new TradeBar()).GetSource(_config, initialDate3, false);
|
||||
var dataBars3 = reader3.Read(source3).First();
|
||||
|
||||
Assert.Less(dataBars3.EndTime, initialDate3);
|
||||
}
|
||||
|
||||
[TestCase(Resolution.Daily, true)]
|
||||
[TestCase(Resolution.Hour, true)]
|
||||
[TestCase(Resolution.Minute, false)]
|
||||
[TestCase(Resolution.Second, false)]
|
||||
[TestCase(Resolution.Tick, false)]
|
||||
public void CacheBehaviorDifferentResolutions(Resolution resolution, bool shouldBeCached)
|
||||
{
|
||||
_config = new SubscriptionDataConfig(
|
||||
typeof(TestTradeBarFactory),
|
||||
Symbols.SPY,
|
||||
resolution,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
using var singleEntryDataCacheProvider = new SingleEntryDataCacheProvider(TestGlobals.DataProvider, isDataEphemeral: false);
|
||||
var reader = new TextSubscriptionDataSourceReader(
|
||||
singleEntryDataCacheProvider,
|
||||
_config,
|
||||
new DateTime(2013, 10, 07),
|
||||
false,
|
||||
null);
|
||||
var source = (new TradeBar()).GetSource(_config, new DateTime(2013, 10, 07), false);
|
||||
|
||||
// first call should cache
|
||||
reader.Read(source).First();
|
||||
TestTradeBarFactory.ReaderWasCalled = false;
|
||||
reader.Read(source).First();
|
||||
Assert.AreEqual(!shouldBeCached, TestTradeBarFactory.ReaderWasCalled);
|
||||
}
|
||||
|
||||
[Test, Explicit("Performance test")]
|
||||
public void CacheMissPerformance()
|
||||
{
|
||||
long counter = 0;
|
||||
var datas = new List<IEnumerable<BaseData>>();
|
||||
|
||||
var factory = new TradeBar();
|
||||
using var cacheProvider = new CustomEphemeralDataCacheProvider();
|
||||
|
||||
// we load SPY hour zip into memory and use it as the source of different fake tickers
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Hour,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
var fakeSource = factory.GetSource(config, new DateTime(2013, 10, 07), false);
|
||||
cacheProvider.Data = string.Join(Environment.NewLine, QuantConnect.Compression.ReadLines(fakeSource.Source));
|
||||
|
||||
for (var i = 0; i < 500; i++)
|
||||
{
|
||||
var ticker = $"{i}";
|
||||
var fakeConfig = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbol.Create(ticker, SecurityType.Equity, Market.USA),
|
||||
Resolution.Hour,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
var reader = new TextSubscriptionDataSourceReader(cacheProvider, fakeConfig, Time.EndOfTime, false, null);
|
||||
|
||||
var source = factory.GetSource(fakeConfig, Time.BeginningOfTime, false);
|
||||
datas.Add(reader.Read(source));
|
||||
}
|
||||
|
||||
var timer = new Stopwatch();
|
||||
timer.Start();
|
||||
Parallel.ForEach(datas, enumerable =>
|
||||
{
|
||||
// after the first call should use the cache
|
||||
foreach (var data in enumerable)
|
||||
{
|
||||
Interlocked.Increment(ref counter);
|
||||
}
|
||||
});
|
||||
timer.Stop();
|
||||
Log.Trace($"Took {timer.ElapsedMilliseconds}ms. Data count {counter}");
|
||||
|
||||
timer.Reset();
|
||||
timer.Start();
|
||||
Parallel.ForEach(datas, enumerable =>
|
||||
{
|
||||
// after the first call should use the cache
|
||||
foreach (var data in enumerable)
|
||||
{
|
||||
Interlocked.Increment(ref counter);
|
||||
}
|
||||
});
|
||||
timer.Stop();
|
||||
Log.Trace($"Took2 {timer.ElapsedMilliseconds}ms. Data count {counter}");
|
||||
}
|
||||
|
||||
[Test, Explicit("Performance test")]
|
||||
public void CacheHappyPerformance()
|
||||
{
|
||||
long counter = 0;
|
||||
var datas = new List<IEnumerable<BaseData>>();
|
||||
|
||||
var factory = new TradeBar();
|
||||
using var cacheProvider = new CustomEphemeralDataCacheProvider();
|
||||
|
||||
// we load SPY hour zip into memory and use it as the source of different fake tickers
|
||||
var config = new SubscriptionDataConfig(typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Hour,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
var fakeSource = factory.GetSource(config, new DateTime(2013, 10, 07), false);
|
||||
cacheProvider.Data = string.Join(Environment.NewLine, QuantConnect.Compression.ReadLines(fakeSource.Source));
|
||||
|
||||
for (var i = 0; i < 500; i++)
|
||||
{
|
||||
var ticker = $"{i}";
|
||||
var fakeConfig = new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbol.Create(ticker, SecurityType.Equity, Market.USA),
|
||||
Resolution.Hour,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
var reader = new TextSubscriptionDataSourceReader(cacheProvider, fakeConfig, Time.EndOfTime, false, null);
|
||||
|
||||
var source = factory.GetSource(fakeConfig, Time.BeginningOfTime, false);
|
||||
datas.Add(reader.Read(source));
|
||||
}
|
||||
|
||||
var timer = new Stopwatch();
|
||||
timer.Start();
|
||||
Parallel.ForEach(datas, enumerable =>
|
||||
{
|
||||
// after the first call should use the cache
|
||||
foreach (var data in enumerable)
|
||||
{
|
||||
Interlocked.Increment(ref counter);
|
||||
}
|
||||
foreach (var data in enumerable)
|
||||
{
|
||||
Interlocked.Increment(ref counter);
|
||||
}
|
||||
foreach (var data in enumerable)
|
||||
{
|
||||
Interlocked.Increment(ref counter);
|
||||
}
|
||||
});
|
||||
timer.Stop();
|
||||
Log.Trace($"Took {timer.ElapsedMilliseconds}ms. Data count {counter}");
|
||||
}
|
||||
|
||||
private class TestTradeBarFactory : TradeBar
|
||||
{
|
||||
/// <summary>
|
||||
/// Will be true when data is created from a parsed file line
|
||||
/// </summary>
|
||||
public static bool ReaderWasCalled { get; set; }
|
||||
|
||||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
|
||||
{
|
||||
ReaderWasCalled = true;
|
||||
return base.Reader(config, line, date, isLiveMode);
|
||||
}
|
||||
public override BaseData Reader(SubscriptionDataConfig config, StreamReader streamReader, DateTime date, bool isLiveMode)
|
||||
{
|
||||
ReaderWasCalled = true;
|
||||
return base.Reader(config, streamReader, date, isLiveMode);
|
||||
}
|
||||
}
|
||||
|
||||
private class CustomEphemeralDataCacheProvider : IDataCacheProvider
|
||||
{
|
||||
public string Data { set; get; }
|
||||
public bool IsDataEphemeral { set; get; }
|
||||
|
||||
public List<string> GetZipEntries(string zipFile)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
public Stream Fetch(string key)
|
||||
{
|
||||
var stream = new MemoryStream();
|
||||
var writer = new StreamWriter(stream, leaveOpen: true);
|
||||
writer.Write(Data);
|
||||
writer.Flush();
|
||||
stream.Position = 0;
|
||||
writer.Dispose();
|
||||
return stream;
|
||||
}
|
||||
public void Store(string key, byte[] data)
|
||||
{
|
||||
}
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Custom.IconicTypes;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Tests.Common.Data.UniverseSelection;
|
||||
using QuantConnect.Securities.Option;
|
||||
using QuantConnect.Securities.Future;
|
||||
using QuantConnect.Securities.Equity;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class TimeSliceTests
|
||||
{
|
||||
private TimeSliceFactory _timeSliceFactory;
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_timeSliceFactory = new TimeSliceFactory(TimeZones.Utc);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void HandlesTicks_ExpectInOrderWithNoDuplicates()
|
||||
{
|
||||
var subscriptionDataConfig = new SubscriptionDataConfig(
|
||||
typeof(Tick),
|
||||
Symbols.EURUSD,
|
||||
Resolution.Tick,
|
||||
TimeZones.Utc,
|
||||
TimeZones.Utc,
|
||||
true,
|
||||
true,
|
||||
false);
|
||||
|
||||
var security = new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
subscriptionDataConfig,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
|
||||
DateTime refTime = DateTime.UtcNow;
|
||||
|
||||
Tick[] rawTicks = Enumerable
|
||||
.Range(0, 10)
|
||||
.Select(i => new Tick(refTime.AddSeconds(i), Symbols.EURUSD, 1.3465m, 1.34652m))
|
||||
.ToArray();
|
||||
|
||||
IEnumerable<TimeSlice> timeSlices = rawTicks.Select(t => _timeSliceFactory.Create(
|
||||
t.Time,
|
||||
new List<DataFeedPacket> { new DataFeedPacket(security, subscriptionDataConfig, new List<BaseData>() { t }) },
|
||||
SecurityChangesTests.CreateNonInternal(Enumerable.Empty<Security>(), Enumerable.Empty<Security>()),
|
||||
new Dictionary<Universe, BaseDataCollection>()));
|
||||
|
||||
Tick[] timeSliceTicks = timeSlices.SelectMany(ts => ts.Slice.Ticks.Values.SelectMany(x => x)).ToArray();
|
||||
|
||||
Assert.AreEqual(rawTicks.Length, timeSliceTicks.Length);
|
||||
for (int i = 0; i < rawTicks.Length; i++)
|
||||
{
|
||||
Assert.IsTrue(Compare(rawTicks[i], timeSliceTicks[i]));
|
||||
}
|
||||
}
|
||||
|
||||
private bool Compare(Tick expected, Tick actual)
|
||||
{
|
||||
return expected.Time == actual.Time
|
||||
&& expected.BidPrice == actual.BidPrice
|
||||
&& expected.AskPrice == actual.AskPrice
|
||||
&& expected.Quantity == actual.Quantity;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void HandlesMultipleCustomDataOfSameTypeWithDifferentSymbols()
|
||||
{
|
||||
var symbol1 = Symbol.Create("SCF/CBOE_VX1_EW", SecurityType.Base, Market.USA);
|
||||
var symbol2 = Symbol.Create("SCF/CBOE_VX2_EW", SecurityType.Base, Market.USA);
|
||||
|
||||
var subscriptionDataConfig1 = new SubscriptionDataConfig(
|
||||
typeof(UnlinkedData), symbol1, Resolution.Daily, TimeZones.Utc, TimeZones.Utc, true, true, false, isCustom: true);
|
||||
var subscriptionDataConfig2 = new SubscriptionDataConfig(
|
||||
typeof(UnlinkedData), symbol2, Resolution.Daily, TimeZones.Utc, TimeZones.Utc, true, true, false, isCustom: true);
|
||||
|
||||
var security1 = new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
subscriptionDataConfig1,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
|
||||
var security2 = new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
subscriptionDataConfig1,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
|
||||
var timeSlice = _timeSliceFactory.Create(DateTime.UtcNow,
|
||||
new List<DataFeedPacket>
|
||||
{
|
||||
new DataFeedPacket(security1, subscriptionDataConfig1, new List<BaseData> {new UnlinkedData { Symbol = symbol1, Time = DateTime.UtcNow.Date, Value = 15 } }),
|
||||
new DataFeedPacket(security2, subscriptionDataConfig2, new List<BaseData> {new UnlinkedData { Symbol = symbol2, Time = DateTime.UtcNow.Date, Value = 20 } }),
|
||||
},
|
||||
SecurityChangesTests.CreateNonInternal(Enumerable.Empty<Security>(), Enumerable.Empty<Security>()),
|
||||
new Dictionary<Universe, BaseDataCollection>());
|
||||
|
||||
Assert.AreEqual(2, timeSlice.CustomData.Count);
|
||||
|
||||
var data1 = timeSlice.CustomData[0].Data[0];
|
||||
var data2 = timeSlice.CustomData[1].Data[0];
|
||||
|
||||
Assert.IsInstanceOf(typeof(UnlinkedData), data1);
|
||||
Assert.IsInstanceOf(typeof(UnlinkedData), data2);
|
||||
Assert.AreEqual(symbol1, data1.Symbol);
|
||||
Assert.AreEqual(symbol2, data2.Symbol);
|
||||
Assert.AreEqual(15, data1.Value);
|
||||
Assert.AreEqual(20, data2.Value);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FutureDataHasVolume()
|
||||
{
|
||||
var initialVolume = 100;
|
||||
var slices = GetSlices(Symbols.Fut_SPY_Mar19_2016, initialVolume).ToArray();
|
||||
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
var chain = slices[i].FutureChains.FirstOrDefault().Value;
|
||||
var contract = chain.FirstOrDefault();
|
||||
var expected = (i + 1) * initialVolume;
|
||||
Assert.AreEqual(expected, contract.Volume);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void OptionsDataHasVolume()
|
||||
{
|
||||
var initialVolume = 150;
|
||||
var slices = GetSlices(Symbols.SPY_C_192_Feb19_2016, initialVolume).ToArray();
|
||||
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
var chain = slices[i].OptionChains.FirstOrDefault().Value;
|
||||
var contract = chain.FirstOrDefault();
|
||||
var expected = (i + 1) * initialVolume;
|
||||
Assert.AreEqual(expected, contract.Volume);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SuspiciousTicksAreNotAddedToConsolidatorUpdateData()
|
||||
{
|
||||
var symbol = Symbols.SPY;
|
||||
|
||||
var subscriptionDataConfig = new SubscriptionDataConfig(
|
||||
typeof(Tick), symbol, Resolution.Tick, TimeZones.Utc, TimeZones.Utc, true, true, false);
|
||||
|
||||
var security = new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
subscriptionDataConfig,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
|
||||
var timeSlice = _timeSliceFactory.Create(DateTime.UtcNow,
|
||||
new List<DataFeedPacket>
|
||||
{
|
||||
new DataFeedPacket(security, subscriptionDataConfig, new List<BaseData>
|
||||
{
|
||||
new Tick(DateTime.UtcNow, symbol, 280, 0, 0),
|
||||
new Tick(DateTime.UtcNow, symbol, 500, 0, 0) { Suspicious = true },
|
||||
new Tick(DateTime.UtcNow, symbol, 281, 0, 0)
|
||||
})
|
||||
},
|
||||
SecurityChangesTests.CreateNonInternal(Enumerable.Empty<Security>(), Enumerable.Empty<Security>()),
|
||||
new Dictionary<Universe, BaseDataCollection>());
|
||||
|
||||
Assert.AreEqual(1, timeSlice.ConsolidatorUpdateData.Count);
|
||||
|
||||
var data = timeSlice.ConsolidatorUpdateData[0].Data;
|
||||
Assert.AreEqual(2, data.Count);
|
||||
Assert.AreEqual(280, data[0].Value);
|
||||
Assert.AreEqual(281, data[1].Value);
|
||||
}
|
||||
|
||||
private IEnumerable<Slice> GetSlices(Symbol symbol, int initialVolume)
|
||||
{
|
||||
var dataType = symbol.SecurityType.IsOption() ? typeof(OptionUniverse) : typeof(FutureUniverse);
|
||||
var subscriptionDataConfig = new SubscriptionDataConfig(dataType, symbol, Resolution.Second, TimeZones.Utc, TimeZones.Utc, true, true, false);
|
||||
var security = GetSecurity(subscriptionDataConfig);
|
||||
var refTime = DateTime.UtcNow;
|
||||
|
||||
return Enumerable
|
||||
.Range(0, 10)
|
||||
.Select(i =>
|
||||
{
|
||||
var time = refTime.AddSeconds(i);
|
||||
var bid = new Bar(100, 100, 100, 100);
|
||||
var ask = new Bar(110, 110, 110, 110);
|
||||
var volume = (i + 1) * initialVolume;
|
||||
|
||||
var packets = new List<DataFeedPacket>();
|
||||
var packet = new DataFeedPacket(security, subscriptionDataConfig, new List<BaseData>
|
||||
{
|
||||
new QuoteBar(time, symbol, bid, i*10, ask, (i + 1) * 11),
|
||||
new TradeBar(time, symbol, 100, 100, 110, 106, volume)
|
||||
});
|
||||
|
||||
if (symbol.SecurityType == SecurityType.Option)
|
||||
{
|
||||
var underlying = (security as Option).Underlying;
|
||||
packets.Add(new DataFeedPacket(underlying, underlying.SubscriptionDataConfig, new List<BaseData>
|
||||
{
|
||||
new QuoteBar(time, underlying.Symbol, bid, i*10, ask, (i + 1) * 11),
|
||||
new TradeBar(time, underlying.Symbol, 100, 100, 110, 106, volume)
|
||||
}));
|
||||
}
|
||||
|
||||
packets.Add(packet);
|
||||
|
||||
return _timeSliceFactory.Create(
|
||||
time,
|
||||
packets,
|
||||
SecurityChangesTests.CreateNonInternal(Enumerable.Empty<Security>(), Enumerable.Empty<Security>()),
|
||||
new Dictionary<Universe, BaseDataCollection>())
|
||||
.Slice;
|
||||
});
|
||||
}
|
||||
|
||||
private Security GetSecurity(SubscriptionDataConfig config)
|
||||
{
|
||||
if (config.Symbol.SecurityType == SecurityType.Option)
|
||||
{
|
||||
var option = new Option(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
config,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null);
|
||||
var underlyingConfig = new SubscriptionDataConfig(typeof(TradeBar), config.Symbol.Underlying, Resolution.Second,
|
||||
TimeZones.Utc, TimeZones.Utc, true, true, false);
|
||||
var equity = new Equity(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
underlyingConfig,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null);
|
||||
option.Underlying = equity;
|
||||
|
||||
return option;
|
||||
}
|
||||
|
||||
if (config.Symbol.SecurityType == SecurityType.Future)
|
||||
{
|
||||
return new Future(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
config,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null);
|
||||
}
|
||||
|
||||
return new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
|
||||
config,
|
||||
new Cash(Currencies.USD, 0, 1m),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Transport;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds.Transport
|
||||
{
|
||||
[TestFixture]
|
||||
public class RemoteFileSubscriptionStreamReaderTests
|
||||
{
|
||||
private TestDownloadProvider _api;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_api = new TestDownloadProvider();
|
||||
_api.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder);
|
||||
RemoteFileSubscriptionStreamReader.SetDownloadProvider(_api);
|
||||
TestDownloadProvider.DownloadCount = 0;
|
||||
// create cache directory if not existing
|
||||
if (!Directory.Exists(Globals.Cache))
|
||||
{
|
||||
Directory.CreateDirectory(Globals.Cache);
|
||||
}
|
||||
else
|
||||
{
|
||||
// clean old files out of the cache
|
||||
Directory.Delete(Globals.Cache, true);
|
||||
Directory.CreateDirectory(Globals.Cache);
|
||||
}
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
_api.DisposeSafely();
|
||||
}
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void SingleEntryDataCacheProviderEphemeralDataIsRespected(bool isDataEphemeral)
|
||||
{
|
||||
using var cacheProvider = new SingleEntryDataCacheProvider(TestGlobals.DataProvider, isDataEphemeral: isDataEphemeral);
|
||||
using var remoteReader = new RemoteFileSubscriptionStreamReader(
|
||||
cacheProvider,
|
||||
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
|
||||
Globals.Cache,
|
||||
null);
|
||||
|
||||
Assert.IsFalse(remoteReader.EndOfStream);
|
||||
Assert.AreEqual(1, TestDownloadProvider.DownloadCount);
|
||||
|
||||
using var cacheProvider2 = new SingleEntryDataCacheProvider(TestGlobals.DataProvider, isDataEphemeral: isDataEphemeral);
|
||||
using var remoteReader2 = new RemoteFileSubscriptionStreamReader(
|
||||
cacheProvider2,
|
||||
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
|
||||
Globals.Cache,
|
||||
null);
|
||||
|
||||
Assert.IsFalse(remoteReader.EndOfStream);
|
||||
Assert.AreEqual(isDataEphemeral ? 2 : 1, TestDownloadProvider.DownloadCount);
|
||||
}
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void ZipDataCacheProviderEphemeralDataIsRespected(bool isDataEphemeral)
|
||||
{
|
||||
using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider, isDataEphemeral: isDataEphemeral);
|
||||
using var remoteReader = new RemoteFileSubscriptionStreamReader(
|
||||
cacheProvider,
|
||||
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
|
||||
Globals.Cache,
|
||||
null);
|
||||
|
||||
Assert.IsFalse(remoteReader.EndOfStream);
|
||||
Assert.AreEqual(1, TestDownloadProvider.DownloadCount);
|
||||
|
||||
using var cacheProvider2 = new ZipDataCacheProvider(TestGlobals.DataProvider, isDataEphemeral: isDataEphemeral);
|
||||
using var remoteReader2 = new RemoteFileSubscriptionStreamReader(
|
||||
cacheProvider2,
|
||||
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
|
||||
Globals.Cache,
|
||||
null);
|
||||
|
||||
Assert.IsFalse(remoteReader.EndOfStream);
|
||||
Assert.AreEqual(isDataEphemeral ? 2 : 1, TestDownloadProvider.DownloadCount);
|
||||
}
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void ZipDataCacheProviderEphemeralDataIsRespectedForZippedData(bool isDataEphemeral)
|
||||
{
|
||||
using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider, isDataEphemeral: isDataEphemeral);
|
||||
using var remoteReader = new RemoteFileSubscriptionStreamReader(
|
||||
cacheProvider,
|
||||
@"https://cdn.quantconnect.com/uploads/multi_csv_zipped_file.zip",
|
||||
Globals.Cache,
|
||||
null);
|
||||
|
||||
Assert.IsFalse(remoteReader.EndOfStream);
|
||||
Assert.AreEqual(1, TestDownloadProvider.DownloadCount);
|
||||
|
||||
using var remoteReader2 = new RemoteFileSubscriptionStreamReader(
|
||||
cacheProvider,
|
||||
@"https://cdn.quantconnect.com/uploads/multi_csv_zipped_file.zip",
|
||||
Globals.Cache,
|
||||
null);
|
||||
|
||||
Assert.IsFalse(remoteReader.EndOfStream);
|
||||
Assert.AreEqual(isDataEphemeral ? 2 : 1, TestDownloadProvider.DownloadCount);
|
||||
}
|
||||
|
||||
[TestCase(true, "", 78)] // No fragment, will read the first entry
|
||||
[TestCase(false, "", 78)]
|
||||
[TestCase(true, "#csv_file_10.csv", 1)]
|
||||
[TestCase(false, "#csv_file_10.csv", 1)]
|
||||
public void GetsZippedDataForUrlNotEndingWithZipExtension(bool withQuery, string entryName, int expectedLines)
|
||||
{
|
||||
using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
|
||||
var url = @"https://cdn.quantconnect.com/uploads/multi_csv_zipped_file.zip" + (withQuery ? "?some=query" : "") + entryName;
|
||||
using var remoteReader = new RemoteFileSubscriptionStreamReader(cacheProvider, url, Globals.Cache, null);
|
||||
|
||||
Assert.IsFalse(remoteReader.EndOfStream);
|
||||
Assert.AreEqual(1, TestDownloadProvider.DownloadCount);
|
||||
|
||||
var count = 0;
|
||||
while (!remoteReader.EndOfStream)
|
||||
{
|
||||
var line = remoteReader.ReadLine();
|
||||
count++;
|
||||
|
||||
var csv = line.ToCsv();
|
||||
Assert.AreEqual(2, csv.Count);
|
||||
Assert.IsTrue(int.TryParse(csv[0], NumberStyles.Number, CultureInfo.InvariantCulture, out _));
|
||||
Assert.IsTrue(decimal.TryParse(csv[1], NumberStyles.Number, CultureInfo.InvariantCulture, out _));
|
||||
}
|
||||
|
||||
Assert.AreEqual(expectedLines, count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void InvalidDataSource()
|
||||
{
|
||||
using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
|
||||
using var remoteReader = new RemoteFileSubscriptionStreamReader(
|
||||
cacheProvider,
|
||||
@"http://quantconnect.com",
|
||||
Globals.Cache,
|
||||
null);
|
||||
|
||||
Assert.IsFalse(remoteReader.EndOfStream);
|
||||
|
||||
// Fails to get quantconnect.com, missing http://
|
||||
Assert.Throws<WebException>(() => new RemoteFileSubscriptionStreamReader(
|
||||
new SingleEntryDataCacheProvider(new DefaultDataProvider()),
|
||||
@"quantconnect.com",
|
||||
Globals.Cache,
|
||||
null),
|
||||
"Api.Download(): Failed to download data from quantconnect.com. Please verify the source for missing http:// or https://"
|
||||
);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RetriesTransientEmptyDownload()
|
||||
{
|
||||
// the remote endpoint answers empty twice before returning the data; the reader should retry and recover
|
||||
var provider = new TransientEmptyDownloadProvider(emptyResponses: 2, payload: "20131007 00:00,1,2,0,1,100");
|
||||
RemoteFileSubscriptionStreamReader.SetDownloadProvider(provider);
|
||||
|
||||
using var cacheProvider = new SingleEntryDataCacheProvider(TestGlobals.DataProvider, isDataEphemeral: true);
|
||||
using var remoteReader = new RemoteFileSubscriptionStreamReader(
|
||||
cacheProvider,
|
||||
"https://cdn.quantconnect.com/uploads/transient-empty.csv",
|
||||
Globals.Cache,
|
||||
null);
|
||||
|
||||
Assert.IsFalse(remoteReader.EndOfStream, "reader should have data after retrying past the empty responses");
|
||||
Assert.AreEqual("20131007 00:00,1,2,0,1,100", remoteReader.ReadLine());
|
||||
Assert.AreEqual(3, provider.DownloadCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmptyDownloadIsNotCached()
|
||||
{
|
||||
var provider = new TransientEmptyDownloadProvider(emptyResponses: int.MaxValue, payload: string.Empty);
|
||||
RemoteFileSubscriptionStreamReader.SetDownloadProvider(provider);
|
||||
|
||||
var url = "https://cdn.quantconnect.com/uploads/always-empty.csv";
|
||||
using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider, isDataEphemeral: false);
|
||||
using var remoteReader = new RemoteFileSubscriptionStreamReader(cacheProvider, url, Globals.Cache, null);
|
||||
|
||||
Assert.IsTrue(remoteReader.EndOfStream, "an empty download should yield no data");
|
||||
// the empty response must not have been persisted to the cache
|
||||
Assert.IsFalse(File.Exists(Path.Combine(Globals.Cache, url.ToMD5() + ".csv")),
|
||||
"an empty download must not be cached");
|
||||
}
|
||||
|
||||
private class TransientEmptyDownloadProvider : IDownloadProvider
|
||||
{
|
||||
private readonly int _emptyResponses;
|
||||
private readonly byte[] _payload;
|
||||
public int DownloadCount { get; private set; }
|
||||
|
||||
public TransientEmptyDownloadProvider(int emptyResponses, string payload)
|
||||
{
|
||||
_emptyResponses = emptyResponses;
|
||||
_payload = Encoding.UTF8.GetBytes(payload);
|
||||
}
|
||||
|
||||
public byte[] DownloadBytes(string address, IEnumerable<KeyValuePair<string, string>> headers, string userName, string password)
|
||||
{
|
||||
DownloadCount++;
|
||||
return DownloadCount <= _emptyResponses ? Array.Empty<byte>() : _payload;
|
||||
}
|
||||
|
||||
public string Download(string address, IEnumerable<KeyValuePair<string, string>> headers, string userName, string password)
|
||||
{
|
||||
return Encoding.UTF8.GetString(DownloadBytes(address, headers, userName, password));
|
||||
}
|
||||
}
|
||||
|
||||
private class TestDownloadProvider : QuantConnect.Api.Api
|
||||
{
|
||||
public static int DownloadCount { get; set; }
|
||||
static TestDownloadProvider()
|
||||
{
|
||||
DownloadCount = 0;
|
||||
}
|
||||
public override byte[] DownloadBytes(string address, IEnumerable<KeyValuePair<string, string>> headers, string userName, string password)
|
||||
{
|
||||
DownloadCount++;
|
||||
return base.DownloadBytes(address, headers, userName, password);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Fundamental;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.Results;
|
||||
using QuantConnect.Orders;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Securities;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class UniverseSelectionTests
|
||||
{
|
||||
[Test]
|
||||
public void CreatedEquityIsNotAddedToSymbolCache()
|
||||
{
|
||||
SymbolCache.Clear();
|
||||
var algorithm = new AlgorithmStub(new MockDataFeed());
|
||||
algorithm.SetEndDate(new DateTime(2024, 12, 13));
|
||||
algorithm.SetStartDate(algorithm.EndDate.Subtract(TimeSpan.FromDays(10)));
|
||||
algorithm.AddUniverse(CoarseSelectionFunction, FineSelectionFunction);
|
||||
// OnEndOfTimeStep will add all pending universe additions
|
||||
algorithm.OnEndOfTimeStep();
|
||||
var universe = algorithm.UniverseManager.Values.First();
|
||||
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
|
||||
universe,
|
||||
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
|
||||
new BaseDataCollection(
|
||||
DateTime.UtcNow,
|
||||
Symbols.AAPL,
|
||||
new[]
|
||||
{
|
||||
new CoarseFundamental
|
||||
{
|
||||
Symbol = Symbols.AAPL,
|
||||
Time = DateTime.UtcNow
|
||||
},
|
||||
new CoarseFundamental
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = DateTime.UtcNow
|
||||
}
|
||||
}
|
||||
)
|
||||
);
|
||||
Symbol symbol;
|
||||
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
|
||||
Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol);
|
||||
Assert.IsFalse(SymbolCache.TryGetSymbol("AAPL", out symbol));
|
||||
Assert.IsFalse(SymbolCache.TryGetSymbol("SPY", out symbol));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RemovalFromUniverseAndDataFeedMakesSecurityNotTradable()
|
||||
{
|
||||
SymbolCache.Clear();
|
||||
var algorithm = new AlgorithmStub(new MockDataFeedWithSubscription());
|
||||
var orderProcessorMock = new Mock<IOrderProcessor>();
|
||||
orderProcessorMock.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>())).Returns(new List<Order>());
|
||||
algorithm.Transactions.SetOrderProcessor(orderProcessorMock.Object);
|
||||
|
||||
algorithm.SetStartDate(2012, 3, 27);
|
||||
algorithm.SetEndDate(2012, 3, 30);
|
||||
algorithm.AddUniverse("my-custom-universe", dt => dt.Day < 30 ? new List<string> { "CPRT" } : Enumerable.Empty<string>());
|
||||
// OnEndOfTimeStep will add all pending universe additions
|
||||
algorithm.OnEndOfTimeStep();
|
||||
var universe = algorithm.UniverseManager.Values.First();
|
||||
|
||||
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
|
||||
universe,
|
||||
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(2)),
|
||||
new BaseDataCollection(
|
||||
algorithm.UtcTime,
|
||||
Symbol.Create("CPRT", SecurityType.Equity, Market.USA),
|
||||
new List<BaseData>()
|
||||
)
|
||||
);
|
||||
|
||||
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
|
||||
Assert.AreEqual(0, securityChanges.RemovedSecurities.Count);
|
||||
|
||||
var security = securityChanges.AddedSecurities.First();
|
||||
Assert.IsTrue(security.IsTradable);
|
||||
|
||||
securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
|
||||
universe,
|
||||
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone),
|
||||
new BaseDataCollection(
|
||||
algorithm.UtcTime,
|
||||
Symbol.Create("CPRT", SecurityType.Equity, Market.USA),
|
||||
new List<BaseData>()
|
||||
)
|
||||
);
|
||||
|
||||
Assert.AreEqual(0, securityChanges.AddedSecurities.Count);
|
||||
Assert.AreEqual(1, securityChanges.RemovedSecurities.Count);
|
||||
Assert.AreEqual(security.Symbol, securityChanges.RemovedSecurities.First().Symbol);
|
||||
|
||||
Assert.IsFalse(security.IsTradable);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CoarseFundamentalHasFundamentalDataFalseExcludedInFineUniverseSelection()
|
||||
{
|
||||
var algorithm = new AlgorithmStub(new MockDataFeed());
|
||||
algorithm.SetEndDate(new DateTime(2024, 12, 13));
|
||||
algorithm.SetStartDate(algorithm.EndDate.Subtract(TimeSpan.FromDays(10)));
|
||||
|
||||
algorithm.AddUniverse(
|
||||
coarse => coarse.Select(c => c.Symbol),
|
||||
fine => fine.Select(f => f.Symbol).Where(x => x.ID.Symbol == "AAPL")
|
||||
);
|
||||
// OnEndOfTimeStep will add all pending universe additions
|
||||
algorithm.OnEndOfTimeStep();
|
||||
|
||||
var universe = algorithm.UniverseManager.Values.First();
|
||||
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
|
||||
universe,
|
||||
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
|
||||
new BaseDataCollection(
|
||||
DateTime.UtcNow,
|
||||
Symbols.AAPL,
|
||||
new[]
|
||||
{
|
||||
new CoarseFundamental
|
||||
{
|
||||
Symbol = Symbols.AAPL,
|
||||
Time = DateTime.UtcNow
|
||||
},
|
||||
new CoarseFundamental
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = DateTime.UtcNow
|
||||
}
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
Assert.AreEqual(1, securityChanges.Count);
|
||||
Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DoesNotAddSelectedSecuritiesIfNoTradableDates()
|
||||
{
|
||||
var algorithm = new AlgorithmStub(new MockDataFeed());
|
||||
algorithm.SetStartDate(2023, 12, 01);
|
||||
algorithm.SetEndDate(2023, 12, 30); // Sunday
|
||||
|
||||
algorithm.AddUniverse(
|
||||
coarse => coarse.Select(c => c.Symbol),
|
||||
fine => fine.Select(f => f.Symbol));
|
||||
algorithm.OnEndOfTimeStep();
|
||||
|
||||
var universe = algorithm.UniverseManager.Values.First();
|
||||
|
||||
var getUniverseData = (DateTime dt) => new BaseDataCollection(
|
||||
dt,
|
||||
Symbols.AAPL,
|
||||
[
|
||||
new CoarseFundamental
|
||||
{
|
||||
Symbol = Symbols.AAPL,
|
||||
Time = dt
|
||||
},
|
||||
new CoarseFundamental
|
||||
{
|
||||
Symbol = Symbols.SPY,
|
||||
Time = dt
|
||||
}
|
||||
]
|
||||
);
|
||||
|
||||
// Friday, one tradeale day left before end date
|
||||
var dateTime = new DateTime(2023, 12, 29).ConvertToUtc(algorithm.TimeZone);
|
||||
var universeData = getUniverseData(dateTime);
|
||||
|
||||
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
|
||||
universe,
|
||||
dateTime,
|
||||
universeData);
|
||||
Assert.AreEqual(2, securityChanges.AddedSecurities.Count);
|
||||
CollectionAssert.AreEquivalent(universeData.Select(x => x.Symbol), securityChanges.AddedSecurities.Select(x => x.Symbol));
|
||||
|
||||
// Saturday, no tradable days left before end date
|
||||
dateTime += TimeSpan.FromDays(1);
|
||||
universeData = getUniverseData(dateTime);
|
||||
|
||||
securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
|
||||
universe,
|
||||
dateTime,
|
||||
universeData);
|
||||
Assert.AreEqual(0, securityChanges.AddedSecurities.Count);
|
||||
}
|
||||
|
||||
private IEnumerable<Symbol> CoarseSelectionFunction(IEnumerable<CoarseFundamental> coarse)
|
||||
{
|
||||
return new List<Symbol> {Symbols.AAPL, Symbols.SPY};
|
||||
}
|
||||
|
||||
private IEnumerable<Symbol> FineSelectionFunction(IEnumerable<FineFundamental> fine)
|
||||
{
|
||||
return new[] { fine.First(fundamental => fundamental.Symbol.Value == "AAPL").Symbol };
|
||||
}
|
||||
|
||||
public class MockDataFeedWithSubscription : IDataFeed
|
||||
{
|
||||
public bool IsActive { get; }
|
||||
|
||||
public void Initialize(
|
||||
IAlgorithm algorithm,
|
||||
AlgorithmNodePacket job,
|
||||
IResultHandler resultHandler,
|
||||
IMapFileProvider mapFileProvider,
|
||||
IFactorFileProvider factorFileProvider,
|
||||
IDataProvider dataProvider,
|
||||
IDataFeedSubscriptionManager subscriptionManager,
|
||||
IDataFeedTimeProvider dataFeedTimeProvider,
|
||||
IDataChannelProvider dataChannelProvider
|
||||
)
|
||||
{
|
||||
}
|
||||
|
||||
public Subscription CreateSubscription(SubscriptionRequest request)
|
||||
{
|
||||
return new Subscription(request, Enumerable.Empty<SubscriptionData>().GetEnumerator(), null);
|
||||
}
|
||||
|
||||
public void RemoveSubscription(Subscription subscription)
|
||||
{
|
||||
}
|
||||
|
||||
public void Exit()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.Engine.DataFeeds
|
||||
{
|
||||
[TestFixture]
|
||||
public class ZipEntryNameSubscriptionFactoryTests
|
||||
{
|
||||
[Test]
|
||||
public void ReadsZipEntryNames()
|
||||
{
|
||||
var time = new DateTime(2016, 03, 03, 12, 48, 15);
|
||||
var source = Path.Combine("TestData", "20151224_quote_american.zip");
|
||||
var config = new SubscriptionDataConfig(typeof (CustomData), Symbol.Create("XLRE", SecurityType.Option, Market.USA), Resolution.Tick,
|
||||
TimeZones.NewYork, TimeZones.NewYork, false, false, false);
|
||||
using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
|
||||
var factory = new ZipEntryNameSubscriptionDataSourceReader(cacheProvider, config, time, false);
|
||||
var expected = new[]
|
||||
{
|
||||
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 21m, new DateTime(2016, 08, 19)),
|
||||
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 22m, new DateTime(2016, 08, 19)),
|
||||
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Put, 37m, new DateTime(2016, 08, 19)),
|
||||
};
|
||||
|
||||
var actual = factory.Read(new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName)).ToList();
|
||||
|
||||
// we only really care about the symbols
|
||||
CollectionAssert.AreEqual(expected, actual.Select(x => x.Symbol));
|
||||
Assert.IsTrue(actual.All(x => x is CustomData));
|
||||
}
|
||||
|
||||
private class CustomData : BaseData
|
||||
{
|
||||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var symbol = LeanData.ReadSymbolFromZipEntry(config.Symbol, config.Resolution, line);
|
||||
return new CustomData { Time = date, Symbol = symbol };
|
||||
}
|
||||
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType);
|
||||
return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user