chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,577 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.ToolBox;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Data.Consolidators;
|
||||
using QuantConnect.Data.Market;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.Fixtures)]
|
||||
public class LeanDataReaderTests
|
||||
{
|
||||
string _dataDirectory = "../../../Data/";
|
||||
DateTime _fromDate = new DateTime(2013, 10, 7);
|
||||
DateTime _toDate = new DateTime(2013, 10, 11);
|
||||
|
||||
[Test, Parallelizable(ParallelScope.Self)]
|
||||
public void LoadsEquity_Daily_SingleEntryZip()
|
||||
{
|
||||
var dataPath = LeanData.GenerateZipFilePath(Globals.DataFolder, Symbols.AAPL, DateTime.UtcNow, Resolution.Daily, TickType.Trade);
|
||||
var leanDataReader = new LeanDataReader(dataPath);
|
||||
var data = leanDataReader.Parse().ToList();
|
||||
|
||||
Assert.AreEqual(5849, data.Count);
|
||||
Assert.IsTrue(data.All(baseData => baseData.Symbol == Symbols.AAPL && baseData is TradeBar));
|
||||
}
|
||||
|
||||
#region futures
|
||||
|
||||
[Test, Parallelizable(ParallelScope.Self)]
|
||||
public void ReadsEntireZipFileEntries_OpenInterest()
|
||||
{
|
||||
var baseFuture = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, SecurityIdentifier.DefaultDate);
|
||||
var filePath = LeanData.GenerateZipFilePath(Globals.DataFolder, baseFuture, new DateTime(2013, 10, 06), Resolution.Minute, TickType.OpenInterest);
|
||||
var leanDataReader = new LeanDataReader(filePath);
|
||||
|
||||
var data = leanDataReader.Parse()
|
||||
.ToList()
|
||||
.GroupBy(baseData => baseData.Symbol)
|
||||
.Select(grp => grp.ToList())
|
||||
.OrderBy(list => list[0].Symbol)
|
||||
.ToList();
|
||||
|
||||
Assert.AreEqual(5, data.Count);
|
||||
Assert.IsTrue(data.All(kvp => kvp.Count == 1));
|
||||
|
||||
foreach (var dataForSymbol in data)
|
||||
{
|
||||
Assert.IsTrue(dataForSymbol[0] is OpenInterest);
|
||||
Assert.IsFalse(dataForSymbol[0].Symbol.IsCanonical());
|
||||
Assert.AreEqual(Futures.Indices.SP500EMini, dataForSymbol[0].Symbol.ID.Symbol);
|
||||
Assert.AreNotEqual(0, dataForSymbol[0]);
|
||||
}
|
||||
}
|
||||
|
||||
[Test, Parallelizable(ParallelScope.Self)]
|
||||
public void ReadsEntireZipFileEntries_Trade()
|
||||
{
|
||||
var baseFuture = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, SecurityIdentifier.DefaultDate);
|
||||
var filePath = LeanData.GenerateZipFilePath(Globals.DataFolder, baseFuture, new DateTime(2013, 10, 06), Resolution.Minute, TickType.Trade);
|
||||
var leanDataReader = new LeanDataReader(filePath);
|
||||
|
||||
var data = leanDataReader.Parse()
|
||||
.ToList()
|
||||
.GroupBy(baseData => baseData.Symbol)
|
||||
.Select(grp => grp.ToList())
|
||||
.OrderBy(list => list[0].Symbol)
|
||||
.ToList();
|
||||
|
||||
Assert.AreEqual(2, data.Count);
|
||||
|
||||
foreach (var dataForSymbol in data)
|
||||
{
|
||||
Assert.IsTrue(dataForSymbol[0] is TradeBar);
|
||||
Assert.IsFalse(dataForSymbol[0].Symbol.IsCanonical());
|
||||
Assert.AreEqual(Futures.Indices.SP500EMini, dataForSymbol[0].Symbol.ID.Symbol);
|
||||
}
|
||||
|
||||
Assert.AreEqual(118, data[0].Count);
|
||||
Assert.AreEqual(10, data[1].Count);
|
||||
}
|
||||
|
||||
[Test, Parallelizable(ParallelScope.Self)]
|
||||
public void ReadsEntireZipFileEntries_Quote()
|
||||
{
|
||||
var baseFuture = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, SecurityIdentifier.DefaultDate);
|
||||
var filePath = LeanData.GenerateZipFilePath(Globals.DataFolder, baseFuture, new DateTime(2013, 10, 06), Resolution.Minute, TickType.Quote);
|
||||
var leanDataReader = new LeanDataReader(filePath);
|
||||
|
||||
var data = leanDataReader.Parse()
|
||||
.ToList()
|
||||
.GroupBy(baseData => baseData.Symbol)
|
||||
.Select(grp => grp.ToList())
|
||||
.OrderBy(list => list[0].Symbol)
|
||||
.ToList();
|
||||
|
||||
Assert.AreEqual(5, data.Count);
|
||||
|
||||
foreach (var dataForSymbol in data)
|
||||
{
|
||||
Assert.IsTrue(dataForSymbol[0] is QuoteBar);
|
||||
Assert.IsFalse(dataForSymbol[0].Symbol.IsCanonical());
|
||||
Assert.AreEqual(Futures.Indices.SP500EMini, dataForSymbol[0].Symbol.ID.Symbol);
|
||||
}
|
||||
|
||||
Assert.AreEqual(10, data[0].Count);
|
||||
Assert.AreEqual(13, data[1].Count);
|
||||
Assert.AreEqual(52, data[2].Count);
|
||||
Assert.AreEqual(155, data[3].Count);
|
||||
Assert.AreEqual(100, data[4].Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReadFutureChainData()
|
||||
{
|
||||
var canonicalFutures = new Dictionary<Symbol, string>()
|
||||
{
|
||||
{ Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME),
|
||||
"ES20Z13|ES21H14|ES20M14|ES19U14|ES19Z14" },
|
||||
{Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
|
||||
"GC29V13|GC26X13|GC27Z13|GC26G14|GC28J14|GC26M14|GC27Q14|GC29V14|GC29Z14|GC25G15|GC28J15|GC26M15|GC27Q15|GC29Z15|GC28M16|GC28Z16|GC28M17|GC27Z17|GC27M18|GC27Z18|GC26M19"},
|
||||
};
|
||||
|
||||
var tickTypes = new[] { TickType.Trade, TickType.Quote, TickType.OpenInterest };
|
||||
|
||||
var resolutions = new[] { Resolution.Minute };
|
||||
|
||||
|
||||
foreach (var canonical in canonicalFutures)
|
||||
{
|
||||
foreach (var res in resolutions)
|
||||
{
|
||||
foreach (var tickType in tickTypes)
|
||||
{
|
||||
var futures = LoadFutureChain(canonical.Key, _fromDate, tickType, res);
|
||||
|
||||
string chain = string.Join("|", futures.Select(f => f.Value));
|
||||
|
||||
if (tickType == TickType.Quote) //only quotes have the full chain!
|
||||
Assert.AreEqual(canonical.Value, chain);
|
||||
|
||||
foreach (var future in futures)
|
||||
{
|
||||
string csv = LoadFutureData(future, tickType, res);
|
||||
Assert.IsTrue(!string.IsNullOrEmpty(csv));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<Symbol> LoadFutureChain(Symbol baseFuture, DateTime date, TickType tickType, Resolution res)
|
||||
{
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, baseFuture, date, res, tickType);
|
||||
|
||||
//load future chain first
|
||||
var config = new SubscriptionDataConfig(typeof(ZipEntryNameData), baseFuture, res,
|
||||
TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType);
|
||||
var factory = new ZipEntryNameSubscriptionDataSourceReader(TestGlobals.DataCacheProvider, config, date, false);
|
||||
|
||||
var result = factory.Read(new SubscriptionDataSource(filePath, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName))
|
||||
.Select(s => s.Symbol).ToList();
|
||||
return result;
|
||||
}
|
||||
|
||||
private string LoadFutureData(Symbol future, TickType tickType, Resolution res)
|
||||
{
|
||||
var dataType = LeanData.GetDataType(res, tickType);
|
||||
var config = new SubscriptionDataConfig(dataType, future, res,
|
||||
TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType);
|
||||
|
||||
var date = _fromDate;
|
||||
|
||||
var sb = new StringBuilder();
|
||||
|
||||
while (date <= _toDate)
|
||||
{
|
||||
var leanDataReader = new LeanDataReader(config, future, res, date, _dataDirectory);
|
||||
|
||||
foreach (var bar in leanDataReader.Parse())
|
||||
{
|
||||
//write base data type back to string
|
||||
sb.AppendLine(LeanData.GenerateLine(bar, SecurityType.Future, res));
|
||||
}
|
||||
date = date.AddDays(1);
|
||||
}
|
||||
var csv = sb.ToString();
|
||||
return csv;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GenerateDailyAndHourlyFutureDataFromMinutes()
|
||||
{
|
||||
|
||||
var tickTypes = new[] { TickType.Trade, TickType.Quote, TickType.OpenInterest };
|
||||
|
||||
var futures = new[] { Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME),
|
||||
Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX)};
|
||||
var resolutions = new[] { Resolution.Hour, Resolution.Daily };
|
||||
|
||||
foreach (var future in futures)
|
||||
foreach (var res in resolutions)
|
||||
foreach (var tickType in tickTypes)
|
||||
ConvertMinuteFuturesData(future, tickType, res);
|
||||
}
|
||||
|
||||
private void ConvertMinuteFuturesData(Symbol canonical, TickType tickType, Resolution outputResolution, Resolution inputResolution = Resolution.Minute)
|
||||
{
|
||||
|
||||
var timeSpans = new Dictionary<Resolution, TimeSpan>()
|
||||
{
|
||||
{ Resolution.Daily, TimeSpan.FromHours(24)},
|
||||
{ Resolution.Hour, TimeSpan.FromHours(1)},
|
||||
};
|
||||
|
||||
var timeSpan = timeSpans[outputResolution];
|
||||
|
||||
var tickTypeConsolidatorMap = new Dictionary<TickType, Func<IDataConsolidator>>()
|
||||
{
|
||||
{TickType.Quote, () => new QuoteBarConsolidator(timeSpan)},
|
||||
{TickType.OpenInterest, ()=> new OpenInterestConsolidator(timeSpan)},
|
||||
{TickType.Trade, ()=> new TradeBarConsolidator(timeSpan) }
|
||||
|
||||
};
|
||||
|
||||
var consolidators = new Dictionary<string, IDataConsolidator>();
|
||||
var configs = new Dictionary<string, SubscriptionDataConfig>();
|
||||
var outputFiles = new Dictionary<string, StringBuilder>();
|
||||
var futures = new Dictionary<string, Symbol>();
|
||||
|
||||
var date = _fromDate;
|
||||
while (date <= _toDate)
|
||||
{
|
||||
var futureChain = LoadFutureChain(canonical, date, tickType, inputResolution);
|
||||
|
||||
foreach (var future in futureChain)
|
||||
{
|
||||
if (!futures.ContainsKey(future.Value))
|
||||
{
|
||||
futures[future.Value] = future;
|
||||
var config = new SubscriptionDataConfig(LeanData.GetDataType(outputResolution, tickType),
|
||||
future, inputResolution, TimeZones.NewYork, TimeZones.NewYork,
|
||||
false, false, false, false, tickType);
|
||||
configs[future.Value] = config;
|
||||
|
||||
consolidators[future.Value] = tickTypeConsolidatorMap[tickType].Invoke();
|
||||
|
||||
var sb = new StringBuilder();
|
||||
outputFiles[future.Value] = sb;
|
||||
|
||||
consolidators[future.Value].DataConsolidated += (sender, bar) =>
|
||||
{
|
||||
sb.Append(LeanData.GenerateLine(bar, SecurityType.Future, outputResolution) + Environment.NewLine);
|
||||
};
|
||||
}
|
||||
|
||||
var leanDataReader = new LeanDataReader(configs[future.Value], future, inputResolution, date, _dataDirectory);
|
||||
|
||||
var consolidator = consolidators[future.Value];
|
||||
|
||||
foreach (var bar in leanDataReader.Parse())
|
||||
{
|
||||
consolidator.Update(bar);
|
||||
}
|
||||
}
|
||||
date = date.AddDays(1);
|
||||
}
|
||||
|
||||
//write all results
|
||||
foreach (var consolidator in consolidators.Values)
|
||||
consolidator.Scan(date);
|
||||
|
||||
var zip = LeanData.GenerateRelativeZipFilePath(canonical, _fromDate, outputResolution, tickType);
|
||||
var zipPath = Path.Combine(_dataDirectory, zip);
|
||||
var fi = new FileInfo(zipPath);
|
||||
|
||||
if (!fi.Directory.Exists)
|
||||
fi.Directory.Create();
|
||||
|
||||
foreach (var future in futures.Values)
|
||||
{
|
||||
var zipEntry = LeanData.GenerateZipEntryName(future, _fromDate, outputResolution, tickType);
|
||||
var sb = outputFiles[future.Value];
|
||||
|
||||
//Uncomment to write zip files
|
||||
//QuantConnect.Compression.ZipCreateAppendData(zipPath, zipEntry, sb.ToString());
|
||||
|
||||
Assert.IsTrue(sb.Length > 0);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
[Test, TestCaseSource(nameof(OptionAndFuturesCases))]
|
||||
public void ReadLeanFutureAndOptionDataFromFilePath(string composedFilePath, Symbol symbol, int rowsInfile, double sumValue)
|
||||
{
|
||||
// Act
|
||||
var ldr = new LeanDataReader(composedFilePath);
|
||||
var data = ldr.Parse().ToList();
|
||||
// Assert
|
||||
Assert.True(symbol.Equals(data.First().Symbol));
|
||||
Assert.AreEqual(rowsInfile, data.Count);
|
||||
Assert.AreEqual(sumValue, data.Sum(c => c.Value));
|
||||
}
|
||||
|
||||
|
||||
public static object[] OptionAndFuturesCases =
|
||||
{
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/future/cme/minute/es/20131008_quote.zip#20131008_es_minute_quote_201312.csv",
|
||||
LeanData
|
||||
.ReadSymbolFromZipEntry(Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME),
|
||||
Resolution.Minute, "20131008_es_minute_quote_201312.csv"),
|
||||
1411,
|
||||
2346061.875
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/future/comex/minute/gc/20131010_trade.zip#20131010_gc_minute_trade_201312.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
|
||||
Resolution.Minute, "20131010_gc_minute_trade_201312.csv"),
|
||||
1379,
|
||||
1791800.9
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/future/comex/tick/gc/20131009_quote.zip#20131009_gc_tick_quote_201406.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
|
||||
Resolution.Tick, "20131009_gc_tick_quote_201406.csv"),
|
||||
197839,
|
||||
259245064.8
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/future/comex/tick/gc/20131009_trade.zip#20131009_gc_tick_trade_201312.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
|
||||
Resolution.Tick, "20131009_gc_tick_trade_201312.csv"),
|
||||
64712,
|
||||
84596673.8
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/future/cme/minute/es/20131010_openinterest.zip#20131010_es_minute_openinterest_201312.csv",
|
||||
LeanData
|
||||
.ReadSymbolFromZipEntry(Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME),
|
||||
Resolution.Minute, "20131010_es_minute_openinterest_201312.csv"),
|
||||
3,
|
||||
8119169
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/future/comex/tick/gc/20131009_openinterest.zip#20131009_gc_tick_openinterest_201310.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
|
||||
Resolution.Tick, "20131009_gc_tick_openinterest_201310.csv"),
|
||||
4,
|
||||
1312
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/option/usa/minute/aapl/20140606_quote_american.zip#20140606_aapl_minute_quote_american_put_7500000_20141018.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
|
||||
Resolution.Minute,
|
||||
"20140606_aapl_minute_quote_american_put_7500000_20141018.csv"),
|
||||
391,
|
||||
44210.7
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/option/usa/minute/aapl/20140606_trade_american.zip#20140606_aapl_minute_trade_american_call_6475000_20140606.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
|
||||
Resolution.Minute,
|
||||
"20140606_aapl_minute_trade_american_call_6475000_20140606.csv"),
|
||||
374,
|
||||
745.35
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/option/usa/minute/goog/20151224_openinterest_american.zip#20151224_goog_minute_openinterest_american_call_3000000_20160115.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create("GOOG", SecurityType.Option, Market.USA),
|
||||
Resolution.Minute,
|
||||
"20151224_goog_minute_openinterest_american_call_3000000_20160115.csv"),
|
||||
1,
|
||||
38
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/option/usa/daily/aapl_2014_openinterest_american.zip#aapl_openinterest_american_call_1950000_20150117.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
|
||||
Resolution.Daily,
|
||||
"aapl_openinterest_american_call_1950000_20150117.csv"),
|
||||
2,
|
||||
824
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/option/usa/daily/aapl_2014_trade_american.zip#aapl_trade_american_call_5400000_20141018.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
|
||||
Resolution.Daily,
|
||||
"aapl_trade_american_call_5400000_20141018.csv"),
|
||||
1,
|
||||
109.9
|
||||
},
|
||||
|
||||
new object[]
|
||||
{
|
||||
"../../../Data/option/usa/daily/aapl_2014_quote_american.zip#aapl_quote_american_call_307100_20150117.csv",
|
||||
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
|
||||
Resolution.Daily,
|
||||
"aapl_quote_american_call_307100_20150117.csv"),
|
||||
1,
|
||||
63.3
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
[Test, TestCaseSource(nameof(SpotMarketCases))]
|
||||
public void ReadLeanSpotMarketsSecuritiesDataFromFilePath(string securityType, string market, string resolution, string ticker, string fileName, int rowsInfile, double sumValue)
|
||||
{
|
||||
// Arrange
|
||||
var filepath = GenerateFilepathForTesting(_dataDirectory, securityType, market, resolution, ticker, fileName);
|
||||
|
||||
SecurityType securityTypeEnum;
|
||||
Enum.TryParse(securityType, true, out securityTypeEnum);
|
||||
var symbol = Symbol.Create(ticker, securityTypeEnum, market);
|
||||
|
||||
// Act
|
||||
var ldr = new LeanDataReader(filepath);
|
||||
var data = ldr.Parse().ToList();
|
||||
// Assert
|
||||
Assert.True(symbol.Equals(data.First().Symbol));
|
||||
Assert.AreEqual(rowsInfile, data.Count);
|
||||
Assert.AreEqual(sumValue, data.Sum(c => c.Value));
|
||||
}
|
||||
|
||||
public static object[] SpotMarketCases =
|
||||
{
|
||||
//TODO: generate Low resolution sample data for equities
|
||||
new object[] {"equity", "usa", "daily", "aig", "aig.zip", 5849, 340770.5801},
|
||||
new object[] {"equity", "usa", "minute", "aapl", "20140605_trade.zip", 686, 443184.58},
|
||||
new object[] {"equity", "usa", "minute", "ibm", "20131010_quote.zip", 584, 107061.125},
|
||||
new object[] {"equity", "usa", "second", "ibm", "20131010_trade.zip", 5060, 929385.34},
|
||||
new object[] {"equity", "usa", "tick", "bac", "20131011_trade.zip", 112177, 1591680.73},
|
||||
new object[] {"forex", "oanda", "minute", "eurusd", "20140502_quote.zip", 1222, 1693.578875},
|
||||
new object[] {"forex", "oanda", "second", "nzdusd", "20140514_quote.zip", 18061, 15638.724575},
|
||||
new object[] {"forex", "oanda", "tick", "eurusd", "20140507_quote.zip", 41367, 57598.54664},
|
||||
new object[] {"cfd", "oanda", "hour", "xauusd", "xauusd.zip", 76499, 90453133.772 },
|
||||
new object[] {"crypto", "coinbase", "second", "btcusd", "20161008_trade.zip", 3453, 2137057.57},
|
||||
new object[] {"crypto", "coinbase", "minute", "ethusd", "20170903_trade.zip", 1440, 510470.66},
|
||||
new object[] {"crypto", "coinbase", "daily", "btcusd", "btcusd_trade.zip", 1318, 3725052.03},
|
||||
};
|
||||
|
||||
private static IEnumerable<TestCaseData> MinuteZipEntryFileNames
|
||||
{
|
||||
get
|
||||
{
|
||||
yield return new TestCaseData(
|
||||
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Grains.Corn, Market.CBOT, new(2025, 12, 12))),
|
||||
"20251103_ozc_minute_trade_american_call_41000_20251121.csv",
|
||||
4.1m,
|
||||
OptionRight.Call,
|
||||
new DateTime(2025, 11, 21));
|
||||
|
||||
yield return new TestCaseData(
|
||||
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Currencies.JPY, Market.CME, new(2025, 12, 15))),
|
||||
"20251103_jpu_minute_trade_american_call_62.50000_20260109.csv",
|
||||
0.00625m,
|
||||
OptionRight.Call,
|
||||
new DateTime(2026, 01, 09));
|
||||
|
||||
yield return new TestCaseData(
|
||||
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Currencies.AUD, Market.CME, new(2025, 12, 15))),
|
||||
"20251103_adu_minute_openinterest_american_put_6350_20251205.csv",
|
||||
0.635m,
|
||||
OptionRight.Put,
|
||||
new DateTime(2025, 12, 05));
|
||||
|
||||
yield return new TestCaseData(
|
||||
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Currencies.AUD, Market.CME, new(2025, 12, 15))),
|
||||
"20251103_adu_minute_quote_american_call_8400_20260306.csv",
|
||||
0.84m,
|
||||
OptionRight.Call,
|
||||
new DateTime(2026, 03, 06));
|
||||
|
||||
yield return new TestCaseData(
|
||||
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Currencies.NZD, Market.CME, new(2025, 12, 15))),
|
||||
"20251103_6n_minute_quote_american_call_5600_20260403.csv",
|
||||
0.56m,
|
||||
OptionRight.Call,
|
||||
new DateTime(2026, 04, 03));
|
||||
|
||||
yield return new TestCaseData(
|
||||
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Meats.LiveCattle, Market.CME, new(2025, 12, 31))),
|
||||
"20251103_le_minute_quote_american_call_21800_20251205.csv",
|
||||
2.18m,
|
||||
OptionRight.Call,
|
||||
new DateTime(2025, 12, 05));
|
||||
}
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(MinuteZipEntryFileNames))]
|
||||
public void ReadSymbolFromZipEntryShouldParseFileNameWithFloatingNumber(Symbol rootSymbol, string fileNameCsv,
|
||||
decimal expectedStrike, OptionRight expectedOptionRight, DateTime expectedExpiry)
|
||||
{
|
||||
var actualSymbol = LeanData.ReadSymbolFromZipEntry(rootSymbol, Resolution.Minute, fileNameCsv);
|
||||
|
||||
Assert.AreEqual(expectedStrike, actualSymbol.ID.StrikePrice);
|
||||
Assert.AreEqual(expectedOptionRight, actualSymbol.ID.OptionRight);
|
||||
Assert.AreEqual(expectedExpiry, actualSymbol.ID.Date);
|
||||
}
|
||||
|
||||
public static string GenerateFilepathForTesting(string dataDirectory, string securityType, string market, string resolution, string ticker,
|
||||
string fileName)
|
||||
{
|
||||
string filepath;
|
||||
if (resolution == "daily" || resolution == "hour")
|
||||
{
|
||||
filepath = Path.Combine(dataDirectory, securityType, market, resolution, fileName);
|
||||
}
|
||||
else
|
||||
{
|
||||
filepath = Path.Combine(dataDirectory, securityType, market, resolution, ticker, fileName);
|
||||
}
|
||||
return filepath;
|
||||
}
|
||||
|
||||
private class ZipEntryNameData : BaseData
|
||||
{
|
||||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var symbol = LeanData.ReadSymbolFromZipEntry(config.Symbol, config.Resolution, line);
|
||||
return new ZipEntryNameData { Time = date, Symbol = symbol };
|
||||
}
|
||||
|
||||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
|
||||
{
|
||||
var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType);
|
||||
return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,564 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Data.Market;
|
||||
using System.Collections.Generic;
|
||||
using NodaTime;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.HistoricalData;
|
||||
using QuantConnect.Tests.Algorithm;
|
||||
using QuantConnect.ToolBox;
|
||||
using System.Globalization;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox
|
||||
{
|
||||
[TestFixture]
|
||||
public class LeanDataWriterTests
|
||||
{
|
||||
private readonly string _dataDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
|
||||
private Symbol _forex;
|
||||
private Symbol _cfd;
|
||||
private Symbol _equity;
|
||||
private Symbol _crypto;
|
||||
private DateTime _date;
|
||||
|
||||
[OneTimeSetUp]
|
||||
public void Setup()
|
||||
{
|
||||
_forex = Symbol.Create("EURUSD", SecurityType.Forex, Market.FXCM);
|
||||
_cfd = Symbol.Create("BCOUSD", SecurityType.Cfd, Market.Oanda);
|
||||
_equity = Symbol.Create("spy", SecurityType.Equity, Market.USA);
|
||||
_date = Parse.DateTime("3/16/2017 12:00:00 PM");
|
||||
_crypto = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.GDAX);
|
||||
}
|
||||
|
||||
private List<Tick> GetTicks(Symbol sym)
|
||||
{
|
||||
return new List<Tick>()
|
||||
{
|
||||
new Tick(Parse.DateTime("3/16/2017 12:00:00 PM"), sym, 1.0m, 2.0m),
|
||||
new Tick(Parse.DateTime("3/16/2017 12:00:01 PM"), sym, 3.0m, 4.0m),
|
||||
new Tick(Parse.DateTime("3/16/2017 12:00:02 PM"), sym, 5.0m, 6.0m),
|
||||
};
|
||||
}
|
||||
|
||||
private List<QuoteBar> GetQuoteBars(Symbol sym)
|
||||
{
|
||||
return new List<QuoteBar>()
|
||||
{
|
||||
new QuoteBar(Parse.DateTime("3/16/2017 12:00:00 PM"), sym, new Bar(1m, 2m, 3m, 4m), 1, new Bar(5m, 6m, 7m, 8m), 2),
|
||||
new QuoteBar(Parse.DateTime("3/16/2017 12:00:01 PM"), sym, new Bar(11m, 21m, 31m, 41m), 3, new Bar(51m, 61m, 71m, 81m), 4),
|
||||
new QuoteBar(Parse.DateTime("3/16/2017 12:00:02 PM"), sym, new Bar(10m, 20m, 30m, 40m), 5, new Bar(50m, 60m, 70m, 80m), 6),
|
||||
};
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LeanDataWriter_MultipleDays()
|
||||
{
|
||||
var leanDataWriter = new LeanDataWriter(Resolution.Second, _forex, _dataDirectory, TickType.Quote);
|
||||
var sourceData = new List<QuoteBar>
|
||||
{
|
||||
new (Parse.DateTime("3/16/2021 12:00:00 PM"), _forex, new Bar(1m, 2m, 3m, 4m), 1, new Bar(5m, 6m, 7m, 8m), 2)
|
||||
};
|
||||
|
||||
for (var i = 1; i < 100; i++)
|
||||
{
|
||||
sourceData.Add(new QuoteBar(sourceData.Last().Time.AddDays(1),
|
||||
_forex,
|
||||
new Bar(1m, 2m, 3m, 4m),
|
||||
1, new Bar(5m, 6m, 7m, 8m),
|
||||
2));
|
||||
}
|
||||
leanDataWriter.Write(sourceData);
|
||||
|
||||
foreach (var bar in sourceData)
|
||||
{
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _forex, bar.Time, Resolution.Second, TickType.Quote);
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
Assert.IsFalse(File.Exists(filePath + ".tmp"));
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath).Single();
|
||||
|
||||
Assert.AreEqual(1, data.Value.Count);
|
||||
Assert.IsTrue(data.Key.Contains(bar.Time.ToStringInvariant(DateFormat.EightCharacter)), $"Key {data.Key} BarTime: {bar.Time}");
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void Mapping(bool mapSymbol)
|
||||
{
|
||||
LeanDataWriter.MapFileProvider = new Lazy<IMapFileProvider>(TestGlobals.MapFileProvider);
|
||||
|
||||
// asset got mapped on 20080929 to SPWRA
|
||||
var symbol = Symbol.Create("SPWR", SecurityType.Equity, Market.USA);
|
||||
var leanDataWriter = new LeanDataWriter(Resolution.Daily, symbol, _dataDirectory, TickType.Trade, mapSymbol: mapSymbol);
|
||||
var sourceData = new List<TradeBar>
|
||||
{
|
||||
new (new DateTime(2008, 9, 29), symbol, 10, 11, 12, 13, 2),
|
||||
new (new DateTime(2008, 9, 30), symbol, 10, 11, 12, 13, 2),
|
||||
};
|
||||
leanDataWriter.Write(sourceData);
|
||||
|
||||
for (int i = 0; i < sourceData.Count; i++)
|
||||
{
|
||||
var bar = sourceData[i];
|
||||
var expectedTicker = (i == 0 || !mapSymbol) ? "SPWR" : "SPWRA";
|
||||
symbol = symbol.UpdateMappedSymbol(expectedTicker);
|
||||
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, symbol, bar.Time, Resolution.Daily, TickType.Trade);
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
Assert.IsFalse(File.Exists(filePath + ".tmp"));
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath).Single();
|
||||
|
||||
Assert.AreEqual(!mapSymbol ? 2 : 1, data.Value.Count);
|
||||
Assert.AreEqual($"{expectedTicker}.csv".ToLower(), data.Key, $"Key {data.Key} BarTime: {bar.Time}");
|
||||
Assert.IsTrue(data.Value.Any(point => point.StartsWith(bar.Time.ToStringInvariant(DateFormat.TwelveCharacter), StringComparison.Ordinal)), $"Key {data.Key} BarTime: {bar.Time}");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LeanDataWriter_CanWriteForex()
|
||||
{
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _forex, _date, Resolution.Second, TickType.Quote);
|
||||
|
||||
var leanDataWriter = new LeanDataWriter(Resolution.Second, _forex, _dataDirectory, TickType.Quote);
|
||||
leanDataWriter.Write(GetQuoteBars(_forex));
|
||||
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
Assert.IsFalse(File.Exists(filePath + ".tmp"));
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath);
|
||||
|
||||
Assert.AreEqual(data.First().Value.Count, 3);
|
||||
}
|
||||
|
||||
[TestCase(SecurityType.FutureOption, Resolution.Second)]
|
||||
[TestCase(SecurityType.Future, Resolution.Second)]
|
||||
[TestCase(SecurityType.Option, Resolution.Second)]
|
||||
[TestCase(SecurityType.Option, Resolution.Daily)]
|
||||
[TestCase(SecurityType.Future, Resolution.Daily)]
|
||||
[TestCase(SecurityType.FutureOption, Resolution.Daily)]
|
||||
public void LeanDataWriter_CanWriteZipWithMultipleContracts(SecurityType securityType, Resolution resolution)
|
||||
{
|
||||
Symbol contract1;
|
||||
Symbol contract2;
|
||||
if (securityType == SecurityType.Future)
|
||||
{
|
||||
contract1 = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2020, 02, 01));
|
||||
contract2 = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2020, 03, 01));
|
||||
}
|
||||
else if (securityType == SecurityType.Option)
|
||||
{
|
||||
contract1 = Symbol.CreateOption("AAPL", Market.USA, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 02, 01));
|
||||
contract2 = Symbol.CreateOption("AAPL", Market.USA, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 03, 01));
|
||||
}
|
||||
else if (securityType == SecurityType.FutureOption)
|
||||
{
|
||||
var underlying = Symbols.ES_Future_Chain;
|
||||
contract1 = Symbol.CreateOption(underlying, Market.CME, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 02, 01));
|
||||
contract2 = Symbol.CreateOption(underlying, Market.CME, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 03, 01));
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new NotImplementedException($"{securityType} not implemented!");
|
||||
}
|
||||
|
||||
var filePath1 = LeanData.GenerateZipFilePath(_dataDirectory, contract1, _date, resolution, TickType.Quote);
|
||||
var leanDataWriter1 = new LeanDataWriter(resolution, contract1, _dataDirectory, TickType.Quote);
|
||||
leanDataWriter1.Write(GetQuoteBars(contract1));
|
||||
|
||||
var filePath2 = LeanData.GenerateZipFilePath(_dataDirectory, contract2, _date, resolution, TickType.Quote);
|
||||
var leanDataWriter2 = new LeanDataWriter(resolution, contract2, _dataDirectory, TickType.Quote);
|
||||
leanDataWriter2.Write(GetQuoteBars(contract2));
|
||||
|
||||
Assert.AreEqual(filePath1, filePath2);
|
||||
Assert.IsTrue(File.Exists(filePath1));
|
||||
Assert.IsFalse(File.Exists(filePath1 + ".tmp"));
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath1).ToDictionary(x => x.Key, x => x.Value.ToList());
|
||||
Assert.AreEqual(2, data.Count);
|
||||
Assert.That(data.Values, Has.All.Count.EqualTo(3));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LeanDataWriter_CanWriteCfd()
|
||||
{
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _cfd, _date, Resolution.Minute, TickType.Quote);
|
||||
|
||||
var leanDataWriter = new LeanDataWriter(Resolution.Minute, _cfd, _dataDirectory, TickType.Quote);
|
||||
leanDataWriter.Write(GetQuoteBars(_cfd));
|
||||
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
Assert.IsFalse(File.Exists(filePath + ".tmp"));
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath);
|
||||
|
||||
Assert.AreEqual(data.First().Value.Count, 3);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LeanDataWriter_CanWriteEquity()
|
||||
{
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _equity, _date, Resolution.Tick, TickType.Trade);
|
||||
|
||||
var leanDataWriter = new LeanDataWriter(Resolution.Tick, _equity, _dataDirectory);
|
||||
leanDataWriter.Write(GetTicks(_equity));
|
||||
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
Assert.IsFalse(File.Exists(filePath + ".tmp"));
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath);
|
||||
|
||||
Assert.AreEqual(data.First().Value.Count, 3);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LeanDataWriter_CanSupportUtf8Chars()
|
||||
{
|
||||
var symbol = Symbol.Create("币安人生usdt", SecurityType.CryptoFuture, Market.Binance);
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, symbol, _date, Resolution.Tick, TickType.Trade);
|
||||
|
||||
var leanDataWriter = new LeanDataWriter(Resolution.Tick, symbol, _dataDirectory);
|
||||
leanDataWriter.Write(GetTicks(symbol));
|
||||
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
Assert.IsFalse(File.Exists(filePath + ".tmp"));
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath);
|
||||
|
||||
var entry = data.First();
|
||||
Assert.AreEqual(entry.Key, "20170316_币安人生usdt_tick_trade_perp.csv");
|
||||
Assert.AreEqual(entry.Value.Count, 3);
|
||||
}
|
||||
|
||||
[TestCase("CON")]
|
||||
[TestCase("PRN")]
|
||||
[TestCase("AUX")]
|
||||
[TestCase("NUL")]
|
||||
[TestCase("COM0")]
|
||||
[TestCase("COM1")]
|
||||
[TestCase("COM2")]
|
||||
[TestCase("COM3")]
|
||||
[TestCase("COM4")]
|
||||
[TestCase("COM5")]
|
||||
[TestCase("COM6")]
|
||||
[TestCase("COM7")]
|
||||
[TestCase("COM8")]
|
||||
[TestCase("COM9")]
|
||||
[TestCase("LPT0")]
|
||||
[TestCase("LPT1")]
|
||||
[TestCase("LPT2")]
|
||||
[TestCase("LPT3")]
|
||||
[TestCase("LPT4")]
|
||||
[TestCase("LPT5")]
|
||||
[TestCase("LPT6")]
|
||||
[TestCase("LPT7")]
|
||||
[TestCase("LPT8")]
|
||||
[TestCase("LPT9")]
|
||||
[Platform("Win", Reason = "The paths in these testcases are only forbidden in Windows OS")]
|
||||
public void LeanDataWriterHandlesWindowsInvalidNames(string ticker)
|
||||
{
|
||||
var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA);
|
||||
var filePath = FileExtension.ToNormalizedPath(LeanData.GenerateZipFilePath(_dataDirectory, symbol, _date, Resolution.Tick, TickType.Trade));
|
||||
|
||||
var leanDataWriter = new LeanDataWriter(Resolution.Tick, symbol, _dataDirectory);
|
||||
leanDataWriter.Write(GetTicks(symbol));
|
||||
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
Assert.IsFalse(File.Exists(filePath + ".tmp"));
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath);
|
||||
|
||||
Assert.AreEqual(data.First().Value.Count, 3);
|
||||
}
|
||||
|
||||
[TestCase(null, Resolution.Daily)]
|
||||
[TestCase(null, Resolution.Second)]
|
||||
[TestCase(WritePolicy.Merge, Resolution.Second)]
|
||||
[TestCase(WritePolicy.Merge, Resolution.Daily)]
|
||||
[TestCase(WritePolicy.Append, Resolution.Second)]
|
||||
[TestCase(WritePolicy.Overwrite, Resolution.Second)]
|
||||
public void RespectsWritePolicy(WritePolicy? writePolicy, Resolution resolution)
|
||||
{
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _crypto, _date, resolution, TickType.Quote);
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
File.Delete(filePath);
|
||||
}
|
||||
|
||||
var loopCount = 3;
|
||||
var dataPointsPerLoop = 2;
|
||||
for (var i = 0; i < loopCount; i++)
|
||||
{
|
||||
var leanDataWriter = new LeanDataWriter(resolution, _crypto, _dataDirectory, TickType.Quote, writePolicy: writePolicy);
|
||||
var quoteBar = new QuoteBar(Parse.DateTime("3/16/2017 12:00:00 PM").AddHours(i), _crypto, new Bar(1m, 2m, 3m, 4m), 1,
|
||||
new Bar(5m, 6m, 7m, 8m), 2);
|
||||
|
||||
// same quote twice! it has the same time, so it will be dropped when merging
|
||||
leanDataWriter.Write(Enumerable.Repeat(quoteBar, dataPointsPerLoop));
|
||||
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
Assert.IsFalse(File.Exists(filePath + ".tmp"));
|
||||
}
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath).First().Value;
|
||||
|
||||
|
||||
switch (writePolicy)
|
||||
{
|
||||
case WritePolicy.Overwrite:
|
||||
Assert.AreEqual(dataPointsPerLoop, data.Count);
|
||||
break;
|
||||
case WritePolicy.Merge:
|
||||
Assert.AreEqual(loopCount, data.Count);
|
||||
if (resolution < Resolution.Hour)
|
||||
{
|
||||
var previousMs = 0;
|
||||
Assert.IsTrue(data.All(x =>
|
||||
{
|
||||
var milliseconds = int.Parse(x.Split(',')[0], NumberStyles.Number, CultureInfo.InvariantCulture);
|
||||
var result = previousMs < milliseconds;
|
||||
previousMs = milliseconds;
|
||||
return result;
|
||||
}));
|
||||
}
|
||||
break;
|
||||
case WritePolicy.Append:
|
||||
Assert.AreEqual(dataPointsPerLoop * loopCount, data.Count);
|
||||
break;
|
||||
case null:
|
||||
if (resolution >= Resolution.Hour)
|
||||
{
|
||||
// will merge by default
|
||||
Assert.AreEqual(loopCount, data.Count);
|
||||
}
|
||||
else
|
||||
{
|
||||
// overwrite
|
||||
Assert.AreEqual(dataPointsPerLoop, data.Count);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(writePolicy), writePolicy, null);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LeanDataWriter_CanWriteCrypto()
|
||||
{
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _crypto, _date, Resolution.Second, TickType.Quote);
|
||||
|
||||
var leanDataWriter = new LeanDataWriter(Resolution.Second, _crypto, _dataDirectory, TickType.Quote);
|
||||
leanDataWriter.Write(GetQuoteBars(_crypto));
|
||||
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
Assert.IsFalse(File.Exists(filePath + ".tmp"));
|
||||
|
||||
var data = QuantConnect.Compression.Unzip(filePath);
|
||||
|
||||
Assert.AreEqual(data.First().Value.Count, 3);
|
||||
}
|
||||
|
||||
[TestCase(SecurityType.Equity, TickType.Quote, Resolution.Minute)]
|
||||
[TestCase(SecurityType.Equity, TickType.Trade, Resolution.Daily)]
|
||||
[TestCase(SecurityType.Equity, TickType.Trade, Resolution.Hour)]
|
||||
[TestCase(SecurityType.Equity, TickType.Trade, Resolution.Minute)]
|
||||
[TestCase(SecurityType.Crypto, TickType.Quote, Resolution.Minute)]
|
||||
[TestCase(SecurityType.Crypto, TickType.Trade, Resolution.Daily)]
|
||||
[TestCase(SecurityType.Crypto, TickType.Trade, Resolution.Minute)]
|
||||
[TestCase(SecurityType.Option, TickType.Quote, Resolution.Minute)]
|
||||
[TestCase(SecurityType.Option, TickType.Trade, Resolution.Minute)]
|
||||
public void CanDownloadAndSave(SecurityType securityType, TickType tickType, Resolution resolution)
|
||||
{
|
||||
var symbol = Symbols.GetBySecurityType(securityType);
|
||||
var startTimeUtc = GetRepoDataDates(securityType, resolution);
|
||||
|
||||
// Override for this case because symbol from Symbols does not have data included
|
||||
if (securityType == SecurityType.Option)
|
||||
{
|
||||
symbol = Symbols.CreateOptionSymbol("GOOG", OptionRight.Call, 770, new DateTime(2015, 12, 24));
|
||||
startTimeUtc = new DateTime(2015, 12, 23);
|
||||
}
|
||||
|
||||
// EndTime based on start, only do 1 day for anything less than hour because we compare datafiles below
|
||||
// and minute and finer resolutions store by day
|
||||
var endTimeUtc = startTimeUtc + TimeSpan.FromDays(resolution >= Resolution.Hour ? 15 : 1);
|
||||
|
||||
// Create our writer and LocalHistory brokerage to "download" from
|
||||
var writer = new LeanDataWriter(_dataDirectory, resolution, securityType, tickType);
|
||||
var brokerage = new LocalHistoryBrokerage();
|
||||
var symbols = new List<Symbol>() {symbol};
|
||||
|
||||
// "Download" and write to file
|
||||
writer.DownloadAndSave(brokerage, symbols, startTimeUtc, endTimeUtc);
|
||||
|
||||
// Verify the file exists where we expect
|
||||
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, symbol, startTimeUtc, resolution, tickType);
|
||||
Assert.IsTrue(File.Exists(filePath));
|
||||
|
||||
// Read the file and data
|
||||
var reader = new LeanDataReader(filePath);
|
||||
var dataFromFile = reader.Parse().ToList();
|
||||
|
||||
// Ensure its not empty and it is actually for this symbol
|
||||
Assert.IsNotEmpty(dataFromFile);
|
||||
Assert.IsTrue(dataFromFile.All(x => x.Symbol == symbol));
|
||||
|
||||
// Get history directly ourselves and compare with the data in the file
|
||||
var history = GetHistory(brokerage, resolution, securityType, symbol, tickType, startTimeUtc, endTimeUtc);
|
||||
CollectionAssert.AreEqual(history.Select(x => x.Time), dataFromFile.Select(x => x.Time));
|
||||
|
||||
brokerage.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper to get history for tests from a brokerage implementation
|
||||
/// </summary>
|
||||
/// <returns>List of data points from history request</returns>
|
||||
private List<BaseData> GetHistory(IBrokerage brokerage, Resolution resolution, SecurityType securityType, Symbol symbol, TickType tickType, DateTime startTimeUtc, DateTime endTimeUtc)
|
||||
{
|
||||
var dataType = LeanData.GetDataType(resolution, tickType);
|
||||
|
||||
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
|
||||
|
||||
var ticker = symbol.ID.Symbol;
|
||||
var market = symbol.ID.Market;
|
||||
|
||||
var canonicalSymbol = Symbol.Create(ticker, securityType, market);
|
||||
|
||||
var exchangeHours = marketHoursDatabase.GetExchangeHours(canonicalSymbol.ID.Market, canonicalSymbol, securityType);
|
||||
var dataTimeZone = marketHoursDatabase.GetDataTimeZone(canonicalSymbol.ID.Market, canonicalSymbol, securityType);
|
||||
|
||||
var historyRequest = new HistoryRequest(
|
||||
startTimeUtc,
|
||||
endTimeUtc,
|
||||
dataType,
|
||||
symbol,
|
||||
resolution,
|
||||
exchangeHours,
|
||||
dataTimeZone,
|
||||
resolution,
|
||||
true,
|
||||
false,
|
||||
DataNormalizationMode.Raw,
|
||||
tickType
|
||||
);
|
||||
|
||||
return brokerage.GetHistory(historyRequest)
|
||||
.Select(
|
||||
x =>
|
||||
{
|
||||
// Convert to date timezone before we write it
|
||||
x.Time = x.Time.ConvertTo(exchangeHours.TimeZone, dataTimeZone);
|
||||
return x;
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test helper method to get dates for data we have in the repo
|
||||
/// Could possibly be refactored and used in Tests.Symbols in a similar way
|
||||
/// </summary>
|
||||
/// <returns>Start time where some data included in the repo exists</returns>
|
||||
private static DateTime GetRepoDataDates(SecurityType securityType, Resolution resolution)
|
||||
{
|
||||
// Because I intend to use this with GetBySecurityType here are the symbols we expect
|
||||
// case SecurityType.Equity: return SPY;
|
||||
// case SecurityType.Option: return SPY_C_192_Feb19_2016;
|
||||
// case SecurityType.Forex: return EURUSD;
|
||||
// case SecurityType.Future: return Future_CLF19_Jan2019;
|
||||
// case SecurityType.Cfd: return XAGUSD;
|
||||
// case SecurityType.Crypto: return BTCUSD;
|
||||
// case SecurityType.Index: return SPX;
|
||||
switch (securityType)
|
||||
{
|
||||
case SecurityType.Equity: // SPY; Daily/Hourly/Minute/Second/Tick
|
||||
return new DateTime(2013, 10, 7);
|
||||
case SecurityType.Crypto: // Coinbase (deprecated: GDAX) BTCUSD Daily/Minute/Second
|
||||
if (resolution == Resolution.Hour || resolution == Resolution.Tick)
|
||||
{
|
||||
throw new ArgumentException($"GDAX BTC Crypto does not have data for this resolution {resolution}");
|
||||
}
|
||||
return new DateTime(2017, 9, 3);
|
||||
case SecurityType.Option: // No Data for the default symbol...
|
||||
return DateTime.MinValue;
|
||||
default:
|
||||
throw new NotImplementedException("This has only implemented a few security types (Equity/Crypto/Option)");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fake brokerage that just uses Local Disk Data to do history requests
|
||||
/// </summary>
|
||||
internal class LocalHistoryBrokerage : NullBrokerage
|
||||
{
|
||||
private readonly IHistoryProvider _historyProvider;
|
||||
|
||||
public LocalHistoryBrokerage()
|
||||
{
|
||||
var mapFileProvider = TestGlobals.MapFileProvider;
|
||||
var dataProvider = TestGlobals.DataProvider;
|
||||
var factorFileProvider = TestGlobals.FactorFileProvider;
|
||||
var dataPermissionManager = new DataPermissionManager();
|
||||
|
||||
mapFileProvider.Initialize(dataProvider);
|
||||
factorFileProvider.Initialize(mapFileProvider, dataProvider);
|
||||
|
||||
_historyProvider = new SubscriptionDataReaderHistoryProvider();
|
||||
_historyProvider.Initialize(
|
||||
new HistoryProviderInitializeParameters(
|
||||
null,
|
||||
null,
|
||||
dataProvider,
|
||||
TestGlobals.DataCacheProvider,
|
||||
mapFileProvider,
|
||||
factorFileProvider,
|
||||
null,
|
||||
true,
|
||||
dataPermissionManager,
|
||||
null,
|
||||
new AlgorithmSettings()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
|
||||
{
|
||||
var requests = new List<HistoryRequest> {request};
|
||||
var slices = _historyProvider.GetHistory(requests, DateTimeZone.Utc);
|
||||
|
||||
// Grab all the bar values for this
|
||||
switch (request.TickType)
|
||||
{
|
||||
case TickType.Quote:
|
||||
return slices.SelectMany(x => x.QuoteBars.Values);
|
||||
case TickType.Trade:
|
||||
return slices.SelectMany(x => x.Bars.Values);
|
||||
default:
|
||||
throw new NotImplementedException("Only support Trade & Quote bars");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
*
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox
|
||||
{
|
||||
[TestFixture]
|
||||
public class PsychSignalDataTests
|
||||
{
|
||||
[Test, Ignore("This test requires raw PsychSignal data")]
|
||||
public void FileHourMatchesDataTimeRealRawData()
|
||||
{
|
||||
var rawPath = Path.Combine("raw", "alternative", "psychsignal");
|
||||
|
||||
foreach (var file in Directory.GetFiles(rawPath, "*.csv", SearchOption.TopDirectoryOnly).ToList())
|
||||
{
|
||||
var fileSplit = file.Split('_');
|
||||
|
||||
var hour = fileSplit[fileSplit.Length - 4].ConvertInvariant<int>();
|
||||
|
||||
// Read one line of the file, and compare the hour of the day to the hour on the file
|
||||
var line = File.ReadLines(file).Last().Split(',');
|
||||
|
||||
// SOURCE[0],SYMBOL[1],TIMESTAMP_UTC[2],BULLISH_INTENSITY[3],BEARISH_INTENSITY[4],BULL_MINUS_BEAR[5],BULL_SCORED_MESSAGES[6],BEAR_SCORED_MESSAGES[7],BULL_BEAR_MSG_RATIO[8],TOTAL_SCANNED_MESSAGES[9]
|
||||
var date = Parse.DateTime(line[2]).ToUniversalTime();
|
||||
|
||||
Assert.AreEqual(hour, date.Hour);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FileHourMatchesFakeDataTime()
|
||||
{
|
||||
var line = File.ReadLines(Path.Combine("TestData", "00010101_05_example_psychsignal_testdata.csv")).Last().Split(',');
|
||||
var hour = 5;
|
||||
|
||||
var date = Parse.DateTime(line[2]).ToUniversalTime();
|
||||
|
||||
Assert.AreEqual(hour, date.Hour);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class BaseSymbolGeneratorTests
|
||||
{
|
||||
[Test]
|
||||
[TestCase(2, 5)]
|
||||
[TestCase(3, 3)]
|
||||
[TestCase(1, 4)]
|
||||
public void NextUpperCaseString_CreatesString_WithinSpecifiedMinMaxLength(int min, int max)
|
||||
{
|
||||
var symbolGenerator = new Mock<BaseSymbolGenerator>(Mock.Of<RandomDataGeneratorSettings>(), new RandomValueGenerator()).Object;
|
||||
var str = symbolGenerator.NextUpperCaseString(min, max);
|
||||
Assert.LessOrEqual(min, str.Length);
|
||||
Assert.GreaterOrEqual(max, str.Length);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextUpperCaseString_CreatesUpperCaseString()
|
||||
{
|
||||
var symbolGenerator = new Mock<BaseSymbolGenerator>(Mock.Of<RandomDataGeneratorSettings>(), new RandomValueGenerator()).Object;
|
||||
var str = symbolGenerator.NextUpperCaseString(10, 10);
|
||||
Assert.IsTrue(str.All(char.IsUpper));
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(SecurityType.Option)]
|
||||
[TestCase(SecurityType.Future)]
|
||||
public void ThrowsArgumentException_ForDerivativeSymbols(SecurityType securityType)
|
||||
{
|
||||
var symbolGenerator = new Mock<BaseSymbolGenerator>(Mock.Of<RandomDataGeneratorSettings>(), new RandomValueGenerator()).Object;
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
symbolGenerator.NextSymbol(securityType, Market.USA)
|
||||
);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThrowIsSettingsAreNull()
|
||||
{
|
||||
Assert.Throws<ArgumentNullException>(() =>
|
||||
{
|
||||
BaseSymbolGenerator.Create(null, Mock.Of<IRandomValueGenerator>());
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThrowIsRundomValueGeneratorIsNull()
|
||||
{
|
||||
Assert.Throws<ArgumentNullException>(() =>
|
||||
{
|
||||
BaseSymbolGenerator.Create(new RandomDataGeneratorSettings(), null);
|
||||
});
|
||||
}
|
||||
|
||||
internal static IEnumerable<Symbol> GenerateAsset(BaseSymbolGenerator instance)
|
||||
{
|
||||
var generateAsset = typeof(BaseSymbolGenerator).GetMethod("GenerateAsset", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
return (IEnumerable<Symbol>)generateAsset.Invoke(instance, new[] { (object)null });
|
||||
}
|
||||
|
||||
internal static IEnumerable<Symbol> GenerateAssetWithTicker(BaseSymbolGenerator instance, string ticker)
|
||||
{
|
||||
var generateAsset = typeof(BaseSymbolGenerator).GetMethod("GenerateAsset", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
return (IEnumerable<Symbol>)generateAsset.Invoke(instance, new object[] { ticker });
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class DefaultSymbolGeneratorTests
|
||||
{
|
||||
private const int Seed = 123456789;
|
||||
private static readonly IRandomValueGenerator _randomValueGenerator = new RandomValueGenerator(Seed);
|
||||
|
||||
private BaseSymbolGenerator _symbolGenerator;
|
||||
private DateTime _minExpiry = new(2000, 01, 01);
|
||||
private DateTime _maxExpiry = new(2001, 01, 01);
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
// initialize using a seed for deterministic tests
|
||||
_symbolGenerator = new DefaultSymbolGenerator(
|
||||
new RandomDataGeneratorSettings()
|
||||
{
|
||||
Market = Market.CME,
|
||||
Start = _minExpiry,
|
||||
End = _maxExpiry
|
||||
},
|
||||
_randomValueGenerator);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(SecurityType.Equity)]
|
||||
[TestCase(SecurityType.Index)]
|
||||
public void ReturnsDefaultSymbolGeneratorInstance(SecurityType securityType)
|
||||
{
|
||||
Assert.IsInstanceOf<DefaultSymbolGenerator>(BaseSymbolGenerator.Create(
|
||||
new RandomDataGeneratorSettings() { SecurityType = securityType },
|
||||
Mock.Of<IRandomValueGenerator>()
|
||||
));
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(SecurityType.Equity, Market.USA, true)]
|
||||
[TestCase(SecurityType.Cfd, Market.FXCM, false)]
|
||||
[TestCase(SecurityType.Cfd, Market.Oanda, false)]
|
||||
[TestCase(SecurityType.Forex, Market.FXCM, false)]
|
||||
[TestCase(SecurityType.Forex, Market.Oanda, false)]
|
||||
[TestCase(SecurityType.Crypto, Market.GDAX, false)]
|
||||
[TestCase(SecurityType.Crypto, Market.Bitfinex, false)]
|
||||
public void GetAvailableSymbolCount(SecurityType securityType, string market, bool expectInfinity)
|
||||
{
|
||||
var expected = expectInfinity
|
||||
? int.MaxValue
|
||||
: SymbolPropertiesDatabase.FromDataFolder().GetSymbolPropertiesList(market, securityType).Count();
|
||||
|
||||
var symbolGenerator = new DefaultSymbolGenerator(new RandomDataGeneratorSettings
|
||||
{
|
||||
SecurityType = securityType,
|
||||
Market = market
|
||||
}, Mock.Of<RandomValueGenerator>());
|
||||
|
||||
Assert.AreEqual(expected, symbolGenerator.GetAvailableSymbolCount());
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(SecurityType.Equity, Market.USA)]
|
||||
[TestCase(SecurityType.Cfd, Market.FXCM)]
|
||||
[TestCase(SecurityType.Cfd, Market.Oanda)]
|
||||
[TestCase(SecurityType.Forex, Market.FXCM)]
|
||||
[TestCase(SecurityType.Forex, Market.Oanda)]
|
||||
[TestCase(SecurityType.Crypto, Market.GDAX)]
|
||||
[TestCase(SecurityType.Crypto, Market.Bitfinex)]
|
||||
public void NextSymbol_CreatesSymbol_WithRequestedSecurityTypeAndMarket(SecurityType securityType, string market)
|
||||
{
|
||||
var symbolGenerator = new DefaultSymbolGenerator(new RandomDataGeneratorSettings
|
||||
{
|
||||
SecurityType = securityType,
|
||||
Market = market
|
||||
}, _randomValueGenerator);
|
||||
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(symbolGenerator).ToList().ToList();
|
||||
Assert.AreEqual(1, symbols.Count);
|
||||
|
||||
var symbol = symbols.First();
|
||||
Assert.AreEqual(securityType, symbol.SecurityType);
|
||||
Assert.AreEqual(market, symbol.ID.Market);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(SecurityType.Equity, Market.USA)]
|
||||
[TestCase(SecurityType.Cfd, Market.FXCM)]
|
||||
[TestCase(SecurityType.Cfd, Market.Oanda)]
|
||||
[TestCase(SecurityType.Forex, Market.FXCM)]
|
||||
[TestCase(SecurityType.Forex, Market.Oanda)]
|
||||
[TestCase(SecurityType.Crypto, Market.GDAX)]
|
||||
[TestCase(SecurityType.Crypto, Market.Bitfinex)]
|
||||
public void NextSymbol_CreatesSymbol_WithEntryInSymbolPropertiesDatabase(SecurityType securityType, string market)
|
||||
{
|
||||
var symbolGenerator = new DefaultSymbolGenerator(new RandomDataGeneratorSettings
|
||||
{
|
||||
SecurityType = securityType,
|
||||
Market = market
|
||||
}, _randomValueGenerator);
|
||||
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(symbolGenerator).ToList().ToList();
|
||||
Assert.AreEqual(1, symbols.Count);
|
||||
|
||||
var symbol = symbols.First();
|
||||
|
||||
var db = SymbolPropertiesDatabase.FromDataFolder();
|
||||
if (db.ContainsKey(market, SecurityDatabaseKey.Wildcard, securityType))
|
||||
{
|
||||
// there is a wildcard entry, so no need to check whether there is a specific entry for the symbol
|
||||
Assert.Pass();
|
||||
}
|
||||
else
|
||||
{
|
||||
// there is no wildcard entry, so there should be a specific entry for the symbol instead
|
||||
Assert.IsTrue(db.ContainsKey(market, symbol, securityType));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(SecurityType.Cfd, Market.FXCM)]
|
||||
[TestCase(SecurityType.Cfd, Market.Oanda)]
|
||||
[TestCase(SecurityType.Forex, Market.FXCM)]
|
||||
[TestCase(SecurityType.Forex, Market.Oanda)]
|
||||
[TestCase(SecurityType.Crypto, Market.GDAX)]
|
||||
[TestCase(SecurityType.Crypto, Market.Bitfinex)]
|
||||
public void NextSymbol_ThrowsNoTickersAvailableException_WhenAllSymbolsGenerated(SecurityType securityType, string market)
|
||||
{
|
||||
var db = SymbolPropertiesDatabase.FromDataFolder();
|
||||
var symbolCount = db.GetSymbolPropertiesList(market, securityType).Count();
|
||||
|
||||
var symbolGenerator = new DefaultSymbolGenerator(new RandomDataGeneratorSettings
|
||||
{
|
||||
SecurityType = securityType,
|
||||
Market = market
|
||||
}, _randomValueGenerator);
|
||||
|
||||
for (var i = 0; i < symbolCount; i++)
|
||||
{
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(symbolGenerator).ToList();
|
||||
Assert.AreEqual(1, symbols.Count);
|
||||
}
|
||||
|
||||
Assert.Throws<NoTickersAvailableException>(() =>
|
||||
BaseSymbolGeneratorTests.GenerateAsset(symbolGenerator).ToList()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using System;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class DividendSplitMapGeneratorTests
|
||||
{
|
||||
[TestCase(240, 0.9857119009006162)]
|
||||
[TestCase(120, 0.9716279515771061)]
|
||||
[TestCase(60, 0.9440608762859234)]
|
||||
[TestCase(12, 0.7498942093324559)]
|
||||
[TestCase(6, 0.5623413251903491)]
|
||||
[TestCase(3, 0.31622776601683794)]
|
||||
[TestCase(1, 0.03162277660168379)]
|
||||
public void GetsExpectedLowerBound(int months, double expectedLowerBound)
|
||||
{
|
||||
var lowerBound = (double)DividendSplitMapGenerator.GetLowerBoundForPreviousSplitFactor(months);
|
||||
Assert.AreEqual(expectedLowerBound, lowerBound, delta: 0.0000000000001);
|
||||
Assert.IsTrue(Math.Pow(lowerBound, lowerBound * 2) >= 0.0009);
|
||||
}
|
||||
|
||||
[TestCase(240)]
|
||||
[TestCase(120)]
|
||||
[TestCase(60)]
|
||||
[TestCase(12)]
|
||||
[TestCase(6)]
|
||||
[TestCase(3)]
|
||||
[TestCase(1)]
|
||||
public void GetsValidNextPreviousSplitFactor(int months)
|
||||
{
|
||||
var lowerBound = DividendSplitMapGenerator.GetLowerBoundForPreviousSplitFactor(months);
|
||||
var upperBound = 1;
|
||||
var nextPreviousSplitFactor = DividendSplitMapGenerator.GetNextPreviousSplitFactor(new Random(), lowerBound, upperBound);
|
||||
Assert.IsTrue(lowerBound <= nextPreviousSplitFactor && nextPreviousSplitFactor <= upperBound);
|
||||
Assert.IsTrue(0.001 <= Math.Pow((double)nextPreviousSplitFactor, 2 * (double)months) && Math.Pow((double)nextPreviousSplitFactor, 2 * (double)months) <= 1);
|
||||
}
|
||||
|
||||
[TestCase(240)]
|
||||
[TestCase(120)]
|
||||
[TestCase(60)]
|
||||
[TestCase(12)]
|
||||
[TestCase(6)]
|
||||
[TestCase(3)]
|
||||
[TestCase(1)]
|
||||
public void PriceScaledBySplitFactorIsBounded(int months)
|
||||
{
|
||||
var maxPossiblePrice = 1000000m;
|
||||
var minPossiblePrice = 0.0001m;
|
||||
var lowerBound = DividendSplitMapGenerator.GetLowerBoundForPreviousSplitFactor(months);
|
||||
var upperBound = 1;
|
||||
var nextPreviousSplitFactor = DividendSplitMapGenerator.GetNextPreviousSplitFactor(new Random(), lowerBound, upperBound);
|
||||
var finalSplitFactor = (decimal)Math.Pow((double)nextPreviousSplitFactor, 2 * (double)months);
|
||||
Assert.IsTrue(0.0001m <= (maxPossiblePrice / finalSplitFactor) && (maxPossiblePrice / finalSplitFactor) <= 1000000000m, (maxPossiblePrice / finalSplitFactor).ToString());
|
||||
Assert.IsTrue(0.0001m <= (minPossiblePrice / finalSplitFactor) && (minPossiblePrice / finalSplitFactor) <= 1000000000m, (minPossiblePrice / finalSplitFactor).ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Securities.Future;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class FutureSymbolGeneratorTests
|
||||
{
|
||||
private const int Seed = 123456789;
|
||||
private static readonly IRandomValueGenerator _randomValueGenerator = new RandomValueGenerator(Seed);
|
||||
|
||||
private BaseSymbolGenerator _symbolGenerator;
|
||||
private DateTime _minExpiry = new(2000, 01, 01);
|
||||
private DateTime _maxExpiry = new(2001, 01, 01);
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
// initialize using a seed for deterministic tests
|
||||
_symbolGenerator = new FutureSymbolGenerator(
|
||||
new RandomDataGeneratorSettings()
|
||||
{
|
||||
Market = Market.CME,
|
||||
Start = _minExpiry,
|
||||
End = _maxExpiry
|
||||
},
|
||||
_randomValueGenerator);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(SecurityType.Future)]
|
||||
public void ReturnsFutureSymbolGeneratorInstance(SecurityType securityType)
|
||||
{
|
||||
Assert.IsInstanceOf<FutureSymbolGenerator>(BaseSymbolGenerator.Create(
|
||||
new RandomDataGeneratorSettings { SecurityType = securityType },
|
||||
Mock.Of<IRandomValueGenerator>()
|
||||
));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetAvailableSymbolCount()
|
||||
{
|
||||
Assert.AreEqual(int.MaxValue, _symbolGenerator.GetAvailableSymbolCount());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GeneratesFutureSymbolWithCorrectExpiryDate()
|
||||
{
|
||||
var startDate = new DateTime(2020, 01, 01);
|
||||
var endDate = new DateTime(2020, 01, 31);
|
||||
var futureSymbolGenerator = new FutureSymbolGenerator(
|
||||
new RandomDataGeneratorSettings()
|
||||
{
|
||||
Market = Market.NYMEX,
|
||||
Start = startDate,
|
||||
End = endDate
|
||||
},
|
||||
_randomValueGenerator);
|
||||
|
||||
// Generate a future symbol using a specific ticker "NG"
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAssetWithTicker(futureSymbolGenerator, "NG").ToList();
|
||||
Assert.AreEqual(1, symbols.Count);
|
||||
|
||||
var symbol = symbols.First();
|
||||
var expiry = symbol.ID.Date;
|
||||
bool hasExpiryFunction = FuturesExpiryFunctions.FuturesExpiryDictionary.TryGetValue(symbol.Canonical, out var expiryFuncWithTicker);
|
||||
Assert.IsTrue(hasExpiryFunction);
|
||||
// Add one month to simulate how the expiry function takes the first day of the next month and subtracts 3 business days
|
||||
Assert.IsTrue(expiryFuncWithTicker(expiry.AddMonths(1)).Equals(expiry));
|
||||
Assert.AreEqual(expiry, new DateTime(2020, 01, 29));
|
||||
Assert.Greater(expiry, startDate);
|
||||
Assert.LessOrEqual(expiry, endDate);
|
||||
|
||||
// Generate a future symbol without specifying ticker
|
||||
symbols = BaseSymbolGeneratorTests.GenerateAsset(futureSymbolGenerator).ToList();
|
||||
Assert.AreEqual(1, symbols.Count);
|
||||
symbol = symbols.First();
|
||||
expiry = symbol.ID.Date;
|
||||
|
||||
// Ensure the expiry falls within the configured start and end range
|
||||
Assert.Greater(expiry, startDate);
|
||||
Assert.LessOrEqual(expiry, endDate);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GeneratesFutureSymbolWithCorrectExpiryDateOverWiderDateRange()
|
||||
{
|
||||
var startDate = new DateTime(2020, 01, 01);
|
||||
var endDate = new DateTime(2024, 01, 31);
|
||||
var futureSymbolGenerator = new FutureSymbolGenerator(
|
||||
new RandomDataGeneratorSettings()
|
||||
{
|
||||
Market = Market.NYMEX,
|
||||
Start = startDate,
|
||||
End = endDate
|
||||
},
|
||||
_randomValueGenerator);
|
||||
|
||||
var expiries = new HashSet<DateTime>();
|
||||
for (int i = 0; i < 500; i++)
|
||||
{
|
||||
// Generate a future symbol using a specific ticker "NG"
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAssetWithTicker(futureSymbolGenerator, "NG").ToList();
|
||||
Assert.AreEqual(1, symbols.Count);
|
||||
var symbol = symbols.First();
|
||||
var expiry = symbol.ID.Date;
|
||||
bool hasExpiryFunction = FuturesExpiryFunctions.FuturesExpiryDictionary.TryGetValue(symbol.Canonical, out var expiryFuncWithTicker);
|
||||
Assert.IsTrue(hasExpiryFunction);
|
||||
Assert.IsTrue(expiryFuncWithTicker(expiry.AddMonths(1)).Equals(expiry));
|
||||
expiries.Add(expiry);
|
||||
}
|
||||
Assert.Greater(expiries.Count, 1);
|
||||
}
|
||||
|
||||
[TestCase("TEST")]
|
||||
[TestCase("NG")]
|
||||
public void StopsExecutionAfterReturningSingleSymbolWhenNoExpiryFunctionOrValidExpiry(string ticker)
|
||||
{
|
||||
// Define a small date range that does not contain any valid expiries
|
||||
var startDate = new DateTime(2020, 01, 15);
|
||||
var endDate = new DateTime(2020, 01, 17);
|
||||
var futureSymbolGenerator = new FutureSymbolGenerator(
|
||||
new RandomDataGeneratorSettings()
|
||||
{
|
||||
Market = Market.NYMEX,
|
||||
Start = startDate,
|
||||
End = endDate
|
||||
},
|
||||
_randomValueGenerator);
|
||||
|
||||
// Generate a future symbol using a specific ticker
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAssetWithTicker(futureSymbolGenerator, ticker);
|
||||
var enumerator = symbols.GetEnumerator();
|
||||
|
||||
// At least one symbol should be produced
|
||||
Assert.IsTrue(enumerator.MoveNext());
|
||||
|
||||
// No additional symbol should be generated
|
||||
Assert.IsFalse(enumerator.MoveNext());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextFuture_CreatesSymbol_WithFutureSecurityTypeAndRequestedMarket()
|
||||
{
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
|
||||
Assert.AreEqual(1, symbols.Count);
|
||||
|
||||
var symbol = symbols.First();
|
||||
|
||||
Assert.AreEqual(Market.CME, symbol.ID.Market);
|
||||
Assert.AreEqual(SecurityType.Future, symbol.SecurityType);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextFuture_CreatesSymbol_WithFutureWithValidFridayExpiry()
|
||||
{
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
|
||||
Assert.AreEqual(1, symbols.Count);
|
||||
|
||||
var symbol = symbols.First();
|
||||
|
||||
var expiry = symbol.ID.Date;
|
||||
Assert.Greater(expiry, _minExpiry);
|
||||
Assert.LessOrEqual(expiry, _maxExpiry);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextFuture_CreatesSymbol_WithEntryInSymbolPropertiesDatabase()
|
||||
{
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
|
||||
Assert.AreEqual(1, symbols.Count);
|
||||
|
||||
var symbol = symbols.First();
|
||||
|
||||
var db = SymbolPropertiesDatabase.FromDataFolder();
|
||||
Assert.IsTrue(db.ContainsKey(Market.CME, symbol, SecurityType.Future));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using System;
|
||||
using QLNet;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Securities.Option;
|
||||
using Cash = QuantConnect.Securities.Cash;
|
||||
using Option = QuantConnect.Securities.Option.Option;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class OptionPriceModelPriceGeneratorTests
|
||||
{
|
||||
private Security _underlying;
|
||||
private Option _option;
|
||||
|
||||
public OptionPriceModelPriceGeneratorTests()
|
||||
{
|
||||
_underlying = new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
|
||||
new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Minute,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
),
|
||||
new Cash("USD", 0, 1m),
|
||||
SymbolProperties.GetDefault("USD"),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
|
||||
var optionSymbol = Symbol.CreateOption(
|
||||
_underlying.Symbol,
|
||||
_underlying.Symbol.ID.Market,
|
||||
_underlying.Symbol.SecurityType.DefaultOptionStyle(),
|
||||
OptionRight.Call,
|
||||
20,
|
||||
new DateTime(2022, 1, 1));
|
||||
|
||||
_option = new Option(
|
||||
optionSymbol,
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
|
||||
new Cash("USD", 0, 1m),
|
||||
new OptionSymbolProperties(_underlying.SymbolProperties),
|
||||
new CashBook(),
|
||||
new RegisteredSecurityDataTypesProvider(),
|
||||
new OptionCache(),
|
||||
_underlying);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThrowsIfSecurityIsNull()
|
||||
{
|
||||
Assert.Throws<ArgumentNullException>(() =>
|
||||
{
|
||||
_ = new OptionPriceModelPriceGenerator(null);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThrowsIfSecurityIsNotOption()
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
{
|
||||
_ = new OptionPriceModelPriceGenerator(_underlying);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReturnsNewPrice()
|
||||
{
|
||||
var priceModelMock = new Mock<IOptionPriceModel>();
|
||||
priceModelMock
|
||||
.Setup(s => s.Evaluate(It.IsAny<OptionPriceModelParameters>()))
|
||||
.Returns(new OptionPriceModelResult(1000, NullGreeks.Instance));
|
||||
_option.PriceModel = priceModelMock.Object;
|
||||
var randomPriceGenerator = new OptionPriceModelPriceGenerator(_option);
|
||||
|
||||
Assert.AreEqual(1000, randomPriceGenerator.NextValue(50, new DateTime(2020, 1, 1)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WarmedUpIfNotQLOptionPriceModel()
|
||||
{
|
||||
_option.PriceModel = Mock.Of<IOptionPriceModel>();
|
||||
var blackScholesModel = new OptionPriceModelPriceGenerator(_option);
|
||||
|
||||
Assert.True(blackScholesModel.WarmedUp);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void WarmedUpSameQLOptionPriceModel(bool warmUp)
|
||||
{
|
||||
var volatilityModel = new Mock<IQLUnderlyingVolatilityEstimator>();
|
||||
volatilityModel.SetupGet(s => s.IsReady).Returns(warmUp);
|
||||
_option.PriceModel = new QLOptionPriceModel(process => new AnalyticEuropeanEngine(process),
|
||||
volatilityModel.Object,
|
||||
null,
|
||||
null);
|
||||
|
||||
var blackScholesModel = new OptionPriceModelPriceGenerator(_option);
|
||||
|
||||
Assert.AreEqual(warmUp, blackScholesModel.WarmedUp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using System;
|
||||
using System.Linq;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class OptionSymbolGeneratorTests
|
||||
{
|
||||
private const int Seed = 123456789;
|
||||
private static readonly IRandomValueGenerator _randomValueGenerator = new RandomValueGenerator(Seed);
|
||||
|
||||
private BaseSymbolGenerator _symbolGenerator;
|
||||
private DateTime _minExpiry = new(2000, 01, 01);
|
||||
private DateTime _maxExpiry = new(2001, 01, 01);
|
||||
private decimal _underlyingPrice = 100m;
|
||||
private decimal _maximumStrikePriceDeviation = 50m;
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
// initialize using a seed for deterministic tests
|
||||
_symbolGenerator = new OptionSymbolGenerator(
|
||||
new RandomDataGeneratorSettings()
|
||||
{
|
||||
Market = Market.USA,
|
||||
Start = _minExpiry,
|
||||
End = _maxExpiry
|
||||
},
|
||||
_randomValueGenerator,
|
||||
_underlyingPrice,
|
||||
_maximumStrikePriceDeviation);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(SecurityType.Option)]
|
||||
public void ReturnsFutureSymbolGeneratorInstance(SecurityType securityType)
|
||||
{
|
||||
Assert.IsInstanceOf<OptionSymbolGenerator>(BaseSymbolGenerator.Create(
|
||||
new RandomDataGeneratorSettings { SecurityType = securityType },
|
||||
Mock.Of<IRandomValueGenerator>()
|
||||
));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetAvailableSymbolCount()
|
||||
{
|
||||
Assert.AreEqual(int.MaxValue,
|
||||
new OptionSymbolGenerator(Mock.Of<RandomDataGeneratorSettings>(), Mock.Of<RandomValueGenerator>(), 100m,
|
||||
75m).GetAvailableSymbolCount());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextOptionSymbol_CreatesOptionSymbol_WithCorrectSecurityTypeAndEquityUnderlying()
|
||||
{
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
|
||||
Assert.AreEqual(3, symbols.Count);
|
||||
|
||||
var underlying = symbols[0];
|
||||
var option = symbols[1];
|
||||
|
||||
Assert.AreEqual(SecurityType.Option, option.SecurityType);
|
||||
Assert.AreEqual(OptionRight.Put, symbols[1].ID.OptionRight);
|
||||
Assert.AreEqual(SecurityType.Option, symbols[2].SecurityType);
|
||||
Assert.AreEqual(OptionRight.Call, symbols[2].ID.OptionRight);
|
||||
|
||||
var underlyingOrigin = option.Underlying;
|
||||
Assert.AreEqual(underlying.Value, underlyingOrigin.Value);
|
||||
Assert.AreEqual(Market.USA, underlying.ID.Market);
|
||||
Assert.AreEqual(SecurityType.Equity, underlying.SecurityType);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextOptionSymbol_CreatesOptionSymbol_WithinSpecifiedExpiration_OnFriday()
|
||||
{
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
|
||||
Assert.AreEqual(3, symbols.Count);
|
||||
|
||||
foreach (var option in new[] { symbols[1], symbols[2] })
|
||||
{
|
||||
var expiration = option.ID.Date;
|
||||
Assert.LessOrEqual(_minExpiry, expiration);
|
||||
Assert.GreaterOrEqual(_maxExpiry, expiration);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextOptionSymbol_CreatesOptionSymbol_WithRequestedMarket()
|
||||
{
|
||||
for (int i = 0; i < 50; i++)
|
||||
{
|
||||
var price = _randomValueGenerator.NextPrice(SecurityType.Equity, Market.USA, 100m, 100m);
|
||||
var symbolGenerator = new OptionSymbolGenerator(
|
||||
new RandomDataGeneratorSettings()
|
||||
{
|
||||
Market = Market.USA,
|
||||
Start = _minExpiry,
|
||||
End = _maxExpiry
|
||||
},
|
||||
_randomValueGenerator,
|
||||
price,
|
||||
50m);
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
|
||||
Assert.AreEqual(3, symbols.Count);
|
||||
|
||||
var option = symbols[1];
|
||||
|
||||
Assert.AreEqual(Market.USA, option.ID.Market);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextOptionSymbol_CreatesOptionSymbol_WithinSpecifiedStrikePriceDeviation()
|
||||
{
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
|
||||
Assert.AreEqual(3, symbols.Count);
|
||||
|
||||
foreach (var option in new []{ symbols[1], symbols[2] })
|
||||
{
|
||||
var strikePrice = option.ID.StrikePrice;
|
||||
var maximumDeviation = _underlyingPrice * (_maximumStrikePriceDeviation / 100m);
|
||||
Assert.LessOrEqual(_underlyingPrice - maximumDeviation, strikePrice);
|
||||
Assert.GreaterOrEqual(_underlyingPrice + maximumDeviation, strikePrice);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase("2021, 6, 2 00:00:00", "2021, 6, 4 00:00:00")]
|
||||
[TestCase("2021, 6, 2 00:00:00", "2021, 6, 5 00:00:00")]
|
||||
[TestCase("2021, 6, 2 00:00:00", "2021, 7, 2 00:00:00")]
|
||||
[TestCase("2021, 6, 2 00:00:00", "2021, 6, 10 00:00:00")]
|
||||
[TestCase("2021, 6, 2 00:00:00", "2021, 6, 11 00:00:00")]
|
||||
[TestCase("2021, 6, 2 00:00:00", "2021, 8, 2 00:00:00")]
|
||||
[TestCase("2021, 6, 2 00:00:00", "2021, 6, 15 00:00:00")]
|
||||
public void OptionSymbolGeneratorCreatesOptionSymbol_WithExpirationDateAtLeastThreeDaysAfterMinExpiryDate(DateTime minExpiry, DateTime maxExpiry)
|
||||
{
|
||||
var symbolGenerator = new OptionSymbolGenerator(
|
||||
new RandomDataGeneratorSettings()
|
||||
{
|
||||
Market = Market.USA,
|
||||
Start = minExpiry,
|
||||
End = maxExpiry
|
||||
},
|
||||
new RandomValueGenerator(),
|
||||
_underlyingPrice,
|
||||
_maximumStrikePriceDeviation);
|
||||
var symbols = BaseSymbolGeneratorTests.GenerateAsset(symbolGenerator).ToList();
|
||||
Assert.AreEqual(3, symbols.Count);
|
||||
|
||||
foreach (var option in new[] { symbols[1], symbols[2] })
|
||||
{
|
||||
var expiration = option.ID.Date;
|
||||
Assert.LessOrEqual(minExpiry.AddDays(3), expiration);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Securities;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using QuantConnect.Configuration;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Securities.Option;
|
||||
using QuantConnect.Util;
|
||||
using static QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator;
|
||||
using QuantConnect.Algorithm;
|
||||
using System.Linq;
|
||||
using System.IO;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class RandomDataGeneratorTests
|
||||
{
|
||||
[Test]
|
||||
[TestCase("2020, 1, 1 00:00:00", "2020, 1, 1 00:00:00", "2020, 1, 1 00:00:00")]
|
||||
[TestCase("2020, 1, 1 00:00:00", "2020, 2, 1 00:00:00", "2020, 1, 16 12:00:00")] // (31 days / 2) = 15.5 = 16 Rounds up to 12 pm
|
||||
[TestCase("2020, 1, 1 00:00:00", "2020, 3, 1 00:00:00", "2020, 1, 31 00:00:00")] // (60 days / 2) = 30
|
||||
[TestCase("2020, 1, 1 00:00:00", "2020, 6, 1 00:00:00", "2020, 3, 17 00:00:00")] // (152 days / 2) = 76
|
||||
|
||||
public void NextRandomGeneratedData(DateTime start, DateTime end, DateTime expectedMidPoint)
|
||||
{
|
||||
var randomValueGenerator = new RandomValueGenerator();
|
||||
var midPoint = QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator.GetDateMidpoint(start, end);
|
||||
var delistDate = QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator.GetDelistingDate(start, end, randomValueGenerator);
|
||||
|
||||
// midPoint and expectedMidPoint must be the same
|
||||
Assert.AreEqual(expectedMidPoint, midPoint);
|
||||
|
||||
// start must be less than or equal to end
|
||||
Assert.LessOrEqual(start, end);
|
||||
|
||||
// delistDate must be less than or equal to end
|
||||
Assert.LessOrEqual(delistDate, end);
|
||||
Assert.GreaterOrEqual(delistDate, midPoint);
|
||||
}
|
||||
|
||||
[TestCase("20220101", "20230101")]
|
||||
public void RandomGeneratorProducesValuesBoundedForEquitiesWhenSplit(string start, string end)
|
||||
{
|
||||
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
|
||||
start,
|
||||
end,
|
||||
"1",
|
||||
"usa",
|
||||
"Equity",
|
||||
"Minute",
|
||||
"Dense",
|
||||
"true",
|
||||
"1",
|
||||
null,
|
||||
"5.0",
|
||||
"30.0",
|
||||
"100.0",
|
||||
"60.0",
|
||||
"30.0",
|
||||
"BaroneAdesiWhaleyApproximationEngine",
|
||||
"Daily",
|
||||
"1",
|
||||
new List<string>(),
|
||||
100
|
||||
);
|
||||
|
||||
var securityManager = new SecurityManager(new TimeKeeper(settings.Start, new[] { TimeZones.Utc }));
|
||||
var securityService = GetSecurityService(settings, securityManager);
|
||||
securityManager.SetSecurityService(securityService);
|
||||
|
||||
var security = securityManager.CreateSecurity(Symbols.AAPL, new List<SubscriptionDataConfig>(), underlying: null);
|
||||
var randomValueGenerator = new RandomValueGenerator();
|
||||
var tickGenerator = new TickGenerator(settings, new TickType[1] { TickType.Trade }, security, randomValueGenerator).GenerateTicks().GetEnumerator();
|
||||
using var sync = new SynchronizingBaseDataEnumerator(tickGenerator);
|
||||
var tickHistory = new List<Tick>();
|
||||
|
||||
while (sync.MoveNext())
|
||||
{
|
||||
var dataPoint = sync.Current;
|
||||
tickHistory.Add(dataPoint as Tick);
|
||||
}
|
||||
|
||||
var dividendsSplitsMaps = new DividendSplitMapGenerator(
|
||||
Symbols.AAPL,
|
||||
settings,
|
||||
randomValueGenerator,
|
||||
BaseSymbolGenerator.Create(settings, randomValueGenerator),
|
||||
new Random(),
|
||||
GetDelistingDate(settings.Start, settings.End, randomValueGenerator),
|
||||
false);
|
||||
|
||||
dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
|
||||
Assert.IsTrue(0.099m <= dividendsSplitsMaps.FinalSplitFactor && dividendsSplitsMaps.FinalSplitFactor <= 1.5m);
|
||||
|
||||
foreach (var tick in tickHistory)
|
||||
{
|
||||
tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
|
||||
Assert.IsTrue(0.001m <= tick.Value && tick.Value <= 1000000000, $"The tick value was {tick.Value} but should have been bounded by 0.001 and 1 000 000 000");
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase(Resolution.Tick, 3.0, 33)]
|
||||
[TestCase(Resolution.Second, 3.0, 33)]
|
||||
[TestCase(Resolution.Minute, 3.0, 33)]
|
||||
[TestCase(Resolution.Hour, 3.0, 33)]
|
||||
[TestCase(Resolution.Daily, 3.0, 33)]
|
||||
[TestCase(Resolution.Minute, 5.0, 20)]
|
||||
[TestCase(Resolution.Minute, 10.0, 10)]
|
||||
public void GetProgressAsPercentageShouldLogWhenProgressExceedsThreshold(Resolution resolution, double thresholdPercent, int expectedLogCount)
|
||||
{
|
||||
TimeSpan step = resolution switch
|
||||
{
|
||||
Resolution.Tick => TimeSpan.FromTicks(1),
|
||||
Resolution.Second => TimeSpan.FromSeconds(1),
|
||||
Resolution.Minute => TimeSpan.FromMinutes(1),
|
||||
Resolution.Hour => TimeSpan.FromHours(1),
|
||||
Resolution.Daily => TimeSpan.FromDays(1),
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null)
|
||||
};
|
||||
|
||||
var start = new DateTime(2024, 1, 1, 0, 0, 0);
|
||||
var end = start.AddTicks(step.Ticks * 100);
|
||||
|
||||
var current = start;
|
||||
var logs = new List<double>();
|
||||
var lastLoggedProgress = 0.0;
|
||||
|
||||
while (current <= end)
|
||||
{
|
||||
var progress = RandomDataGeneratorHelper.GetProgressAsPercentage(start, end, current);
|
||||
|
||||
if (progress - lastLoggedProgress >= thresholdPercent)
|
||||
{
|
||||
logs.Add(progress);
|
||||
lastLoggedProgress = progress;
|
||||
}
|
||||
|
||||
current = current.Add(step);
|
||||
}
|
||||
|
||||
Assert.AreEqual(expectedLogCount, logs.Count);
|
||||
Assert.IsTrue(logs.All(p => p >= 0 && p <= 100));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RandomDataGeneratorCompletesSuccessfully()
|
||||
{
|
||||
var tempFolder = Path.Combine(Path.GetTempPath(), $"LeanTest_{Guid.NewGuid()}");
|
||||
var originalDataFolder = Config.Get("data-folder");
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(tempFolder);
|
||||
Config.Set("data-folder", tempFolder);
|
||||
Globals.Reset();
|
||||
|
||||
var hourPath = Path.Combine(tempFolder, "equity", "usa", "hour");
|
||||
var dailyPath = Path.Combine(tempFolder, "equity", "usa", "daily");
|
||||
var factorFilesPath = Path.Combine(tempFolder, "equity", "usa", "factor_files");
|
||||
var mapFilesPath = Path.Combine(tempFolder, "equity", "usa", "map_files");
|
||||
|
||||
// Create the required folders
|
||||
Directory.CreateDirectory(hourPath);
|
||||
Directory.CreateDirectory(dailyPath);
|
||||
Directory.CreateDirectory(factorFilesPath);
|
||||
Directory.CreateDirectory(mapFilesPath);
|
||||
|
||||
var settings = new RandomDataGeneratorSettings
|
||||
{
|
||||
Start = new DateTime(2024, 1, 1, 9, 30, 0),
|
||||
End = new DateTime(2024, 1, 2, 16, 0, 0),
|
||||
SymbolCount = 1,
|
||||
Market = "usa",
|
||||
SecurityType = SecurityType.Equity,
|
||||
Resolution = Resolution.Hour,
|
||||
DataDensity = DataDensity.Dense,
|
||||
IncludeCoarse = false,
|
||||
QuoteTradeRatio = 1.0,
|
||||
RandomSeed = 123456,
|
||||
HasDividendsPercentage = 0,
|
||||
HasSplitsPercentage = 0,
|
||||
HasIpoPercentage = 0,
|
||||
HasRenamePercentage = 0,
|
||||
Tickers = new List<string>() { "AAPL" }
|
||||
};
|
||||
|
||||
var generator = GetGenerator(settings);
|
||||
|
||||
Assert.DoesNotThrow(() => generator.Run());
|
||||
|
||||
var allFiles = Directory.GetFiles(tempFolder, "*", SearchOption.AllDirectories);
|
||||
Assert.Greater(allFiles.Length, 0);
|
||||
|
||||
var hourFiles = Directory.GetFiles(hourPath, "*.zip");
|
||||
Assert.Greater(hourFiles.Length, 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Config.Set("data-folder", originalDataFolder);
|
||||
Globals.Reset();
|
||||
Directory.Delete(tempFolder, true);
|
||||
}
|
||||
}
|
||||
|
||||
private static QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator GetGenerator(RandomDataGeneratorSettings settings)
|
||||
{
|
||||
var securityManager = new SecurityManager(new TimeKeeper(settings.Start, new[] { TimeZones.Utc }));
|
||||
|
||||
var securityService = new SecurityService(
|
||||
new CashBook(),
|
||||
MarketHoursDatabase.FromDataFolder(),
|
||||
SymbolPropertiesDatabase.FromDataFolder(),
|
||||
new SecurityInitializerProvider(new FuncSecurityInitializer(security =>
|
||||
{
|
||||
// init price
|
||||
security.SetMarketPrice(new Tick(settings.Start, security.Symbol, 100, 100));
|
||||
security.SetMarketPrice(new OpenInterest(settings.Start, security.Symbol, 10000));
|
||||
|
||||
// from settings
|
||||
security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(settings.VolatilityModelResolution);
|
||||
|
||||
// from settings
|
||||
if (security is Option option)
|
||||
{
|
||||
option.PriceModel = OptionPriceModels.QuantLib.Create(settings.OptionPriceEngineName,
|
||||
_interestRateProvider.GetRiskFreeRate(settings.Start, settings.End));
|
||||
}
|
||||
})),
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCacheProvider(
|
||||
new SecurityPortfolioManager(securityManager, new SecurityTransactionManager(null, securityManager), new AlgorithmSettings())),
|
||||
new MapFilePrimaryExchangeProvider(Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider")))
|
||||
);
|
||||
|
||||
securityManager.SetSecurityService(securityService);
|
||||
|
||||
var generator = new QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator();
|
||||
generator.Init(settings, securityManager);
|
||||
return generator;
|
||||
}
|
||||
|
||||
private static readonly IRiskFreeInterestRateModel _interestRateProvider = new InterestRateProvider();
|
||||
|
||||
private static SecurityService GetSecurityService(RandomDataGeneratorSettings settings, SecurityManager securityManager)
|
||||
{
|
||||
var algorithm = new QCAlgorithm();
|
||||
algorithm.Securities = securityManager;
|
||||
var securityService = new SecurityService(
|
||||
new CashBook(),
|
||||
MarketHoursDatabase.FromDataFolder(),
|
||||
SymbolPropertiesDatabase.FromDataFolder(),
|
||||
new SecurityInitializerProvider(new FuncSecurityInitializer(security =>
|
||||
{
|
||||
// init price
|
||||
security.SetMarketPrice(new Tick(settings.Start, security.Symbol, 100, 100));
|
||||
security.SetMarketPrice(new OpenInterest(settings.Start, security.Symbol, 10000));
|
||||
|
||||
// from settings
|
||||
security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(settings.VolatilityModelResolution);
|
||||
|
||||
// from settings
|
||||
if (security is Option option)
|
||||
{
|
||||
option.PriceModel = OptionPriceModels.QuantLib.Create(settings.OptionPriceEngineName,
|
||||
_interestRateProvider.GetRiskFreeRate(settings.Start, settings.End));
|
||||
}
|
||||
})),
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCacheProvider(
|
||||
new SecurityPortfolioManager(securityManager, new SecurityTransactionManager(null, securityManager), new AlgorithmSettings())),
|
||||
new MapFilePrimaryExchangeProvider(Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider"))),
|
||||
algorithm: algorithm);
|
||||
securityManager.SetSecurityService(securityService);
|
||||
|
||||
return securityService;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using System;
|
||||
using System.Linq;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class RandomPriceGeneratorTests
|
||||
{
|
||||
private Security _security;
|
||||
|
||||
public RandomPriceGeneratorTests()
|
||||
{
|
||||
_security = new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
|
||||
new SubscriptionDataConfig(
|
||||
typeof(TradeBar),
|
||||
Symbols.SPY,
|
||||
Resolution.Minute,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
),
|
||||
new Cash("USD", 0, 1m),
|
||||
SymbolProperties.GetDefault("USD"),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReturnsSameAsReference()
|
||||
{
|
||||
var randomMock = new Mock<IRandomValueGenerator>();
|
||||
randomMock.Setup(s => s.NextPrice(It.IsAny<SecurityType>(), It.IsNotNull<string>(), It.IsAny<decimal>(),
|
||||
It.IsAny<decimal>()))
|
||||
.Returns(50);
|
||||
var randomPriceGenerator = new RandomPriceGenerator(_security, randomMock.Object);
|
||||
_security.SetMarketPrice(new Tick(DateTime.UtcNow, Symbols.SPY, 10, 100));
|
||||
|
||||
var actual = randomPriceGenerator.NextValue(1, DateTime.MinValue);
|
||||
randomMock.Verify(s => s.NextPrice(It.IsAny<SecurityType>(), It.IsNotNull<string>(), 55, 1), Times.Once);
|
||||
Assert.AreEqual(50, actual);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AlwaysReady()
|
||||
{
|
||||
var priceGenerator = new RandomPriceGenerator(_security, Mock.Of<IRandomValueGenerator>());
|
||||
Assert.True(priceGenerator.WarmedUp);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ComposeRandomDataGenerator()
|
||||
{
|
||||
Assert.NotNull(Composer.Instance.GetExportedValueByTypeName<QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator>("QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using System;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class RandomValueGeneratorTests
|
||||
{
|
||||
private const int Seed = 123456789;
|
||||
private RandomValueGenerator randomValueGenerator;
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
// initialize using a seed for deterministic tests
|
||||
randomValueGenerator = new RandomValueGenerator(Seed);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextDateTime_CreatesDateTime_WithinSpecifiedMinMax()
|
||||
{
|
||||
var min = new DateTime(2000, 01, 01);
|
||||
var max = new DateTime(2001, 01, 01);
|
||||
var dateTime = randomValueGenerator.NextDate(min, max, dayOfWeek: null);
|
||||
|
||||
Assert.LessOrEqual(min, dateTime);
|
||||
Assert.GreaterOrEqual(max, dateTime);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(DayOfWeek.Sunday)]
|
||||
[TestCase(DayOfWeek.Monday)]
|
||||
[TestCase(DayOfWeek.Tuesday)]
|
||||
[TestCase(DayOfWeek.Wednesday)]
|
||||
[TestCase(DayOfWeek.Thursday)]
|
||||
[TestCase(DayOfWeek.Friday)]
|
||||
[TestCase(DayOfWeek.Saturday)]
|
||||
public void NextDateTime_CreatesDateTime_OnSpecifiedDayOfWeek(DayOfWeek dayOfWeek)
|
||||
{
|
||||
var min = new DateTime(2000, 01, 01);
|
||||
var max = new DateTime(2001, 01, 01);
|
||||
var dateTime = randomValueGenerator.NextDate(min, max, dayOfWeek);
|
||||
|
||||
Assert.AreEqual(dayOfWeek, dateTime.DayOfWeek);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextDateTime_ThrowsArgumentException_WhenMaxIsLessThanMin()
|
||||
{
|
||||
var min = new DateTime(2000, 01, 01);
|
||||
var max = min.AddDays(-1);
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
randomValueGenerator.NextDate(min, max, dayOfWeek: null)
|
||||
);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextDateTime_ThrowsArgumentException_WhenRangeIsTooSmallToProduceDateTimeOnRequestedDayOfWeek()
|
||||
{
|
||||
var min = new DateTime(2019, 01, 15);
|
||||
var max = new DateTime(2019, 01, 20);
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
// no monday between these dates, so impossible to fulfill request
|
||||
randomValueGenerator.NextDate(min, max, DayOfWeek.Monday)
|
||||
);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextPrice_ReturnsZeroForOptionsZeroReferencePrice()
|
||||
{
|
||||
var price = randomValueGenerator.NextPrice(SecurityType.Option, Market.USA, 0m, 1m);
|
||||
Assert.AreEqual(0m, price);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextPrice_ThrowsIfReferencePriceIsInvalid([Values] SecurityType securityType)
|
||||
{
|
||||
// Negative reference price
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
randomValueGenerator.NextPrice(securityType, null, -1m, 1m)
|
||||
);
|
||||
|
||||
// Zero reference price
|
||||
if (securityType != SecurityType.Option)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
randomValueGenerator.NextPrice(securityType, null, 0m, 1m)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase(0)]
|
||||
[TestCase(-1)]
|
||||
public void NextPrice_ThrowsIfMaximumPercentDeviationIsInvalid(decimal maximumPercentDeviation)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
randomValueGenerator.NextPrice(SecurityType.Equity, null, 100m, maximumPercentDeviation)
|
||||
);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextPrice_ReturnsSamePriceIfFailsToGetAValidOne()
|
||||
{
|
||||
// Default min price variation for crypto is 0.01
|
||||
var maximumPercentDeviation = 0.45m;
|
||||
var referencePrice = 0.1m; // too close to the minimum price variation
|
||||
|
||||
var price = randomValueGenerator.NextPrice(SecurityType.Crypto, Market.GDAX, referencePrice, maximumPercentDeviation);
|
||||
|
||||
Assert.AreEqual(referencePrice, price);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextPrice_PricesIsUpdatedEvenIfMaxPercentageDeviationIsLessThanMinPriceVariation()
|
||||
{
|
||||
// Default min price variation for crypto is 0.01
|
||||
var maximumPercentDeviation = 0.45m;
|
||||
var referencePrice = 2m;
|
||||
|
||||
// The maximum price variation is 0.45% of 2, which is 0.009, less than the minimum price variation of 0.01.
|
||||
// The generated price will be rounded back to 2m, but this should be properly handled.
|
||||
|
||||
var price = randomValueGenerator.NextPrice(SecurityType.Crypto, Market.GDAX, referencePrice, maximumPercentDeviation);
|
||||
|
||||
Assert.AreNotEqual(referencePrice, price);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class SecurityInitializerProviderTests
|
||||
{
|
||||
[Test]
|
||||
public void NotNull()
|
||||
{
|
||||
var securityInitializerProvider = new SecurityInitializerProvider(Mock.Of<ISecurityInitializer>());
|
||||
Assert.NotNull(securityInitializerProvider.SecurityInitializer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
|
||||
{
|
||||
[TestFixture]
|
||||
public class TickGeneratorTests
|
||||
{
|
||||
private Symbol _symbol = Symbol.Create("AAPL", SecurityType.Equity, Market.USA);
|
||||
private Security _security;
|
||||
private ITickGenerator _tickGenerator;
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
var start = new DateTime(2020, 1, 6);
|
||||
var end = new DateTime(2020, 1, 10);
|
||||
|
||||
// initialize using a seed for deterministic tests
|
||||
_symbol = Symbol.Create("AAPL", SecurityType.Equity, Market.USA);
|
||||
|
||||
_security = new Security(
|
||||
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
|
||||
new SubscriptionDataConfig(typeof(TradeBar),
|
||||
_symbol,
|
||||
Resolution.Minute,
|
||||
TimeZones.NewYork,
|
||||
TimeZones.NewYork,
|
||||
true, true, false),
|
||||
new Cash(Currencies.USD, 0, 0),
|
||||
SymbolProperties.GetDefault(Currencies.USD),
|
||||
ErrorCurrencyConverter.Instance,
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCache()
|
||||
);
|
||||
_security.SetMarketPrice(new Tick(start, _security.Symbol, 100, 100));
|
||||
_security.SetMarketPrice(new OpenInterest(start, _security.Symbol, 10000));
|
||||
|
||||
_tickGenerator = new TickGenerator(
|
||||
new RandomDataGeneratorSettings()
|
||||
{
|
||||
Start = start,
|
||||
End = end
|
||||
},
|
||||
new TickType[3] { TickType.Trade, TickType.Quote, TickType.OpenInterest },
|
||||
_security,
|
||||
new RandomValueGenerator());
|
||||
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextTick_CreatesTradeTick_WithPriceAndQuantity()
|
||||
{
|
||||
var dateTime = new DateTime(2000, 01, 01);
|
||||
var tick = _tickGenerator.NextTick(dateTime, TickType.Trade, 1m);
|
||||
|
||||
Assert.AreEqual(_symbol, tick.Symbol);
|
||||
Assert.AreEqual(dateTime, tick.Time);
|
||||
Assert.AreEqual(TickType.Trade, tick.TickType);
|
||||
Assert.LessOrEqual(99m, tick.Value);
|
||||
Assert.GreaterOrEqual(101m, tick.Value);
|
||||
|
||||
Assert.Greater(tick.Quantity, 0);
|
||||
Assert.LessOrEqual(tick.Quantity, 1500);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextTick_CreatesQuoteTick_WithCommonValues()
|
||||
{
|
||||
var dateTime = new DateTime(2000, 01, 01);
|
||||
var tick = _tickGenerator.NextTick(dateTime, TickType.Quote, 1m);
|
||||
|
||||
Assert.AreEqual(_symbol, tick.Symbol);
|
||||
Assert.AreEqual(dateTime, tick.Time);
|
||||
Assert.AreEqual(TickType.Quote, tick.TickType);
|
||||
Assert.GreaterOrEqual(tick.Value, 99m);
|
||||
Assert.LessOrEqual(tick.Value, 101m);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextTick_CreatesQuoteTick_WithBidData()
|
||||
{
|
||||
var dateTime = new DateTime(2000, 01, 01);
|
||||
var tick = _tickGenerator.NextTick(dateTime, TickType.Quote, 1m);
|
||||
|
||||
Assert.Greater(tick.BidSize, 0);
|
||||
Assert.LessOrEqual(tick.BidSize, 1500);
|
||||
Assert.GreaterOrEqual(tick.BidPrice, 98.9m);
|
||||
Assert.LessOrEqual(tick.BidPrice, 100.9m);
|
||||
Assert.GreaterOrEqual(tick.Value, tick.BidPrice);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextTick_CreatesQuoteTick_WithAskData()
|
||||
{
|
||||
var dateTime = new DateTime(2000, 01, 01);
|
||||
var tick = _tickGenerator.NextTick(dateTime, TickType.Quote, 1m);
|
||||
|
||||
Assert.GreaterOrEqual(tick.AskSize, 0);
|
||||
Assert.LessOrEqual(tick.AskSize, 1500);
|
||||
Assert.GreaterOrEqual(tick.AskPrice, 99.1m);
|
||||
Assert.LessOrEqual(tick.AskPrice, 101.1m);
|
||||
Assert.LessOrEqual(tick.Value, tick.AskPrice);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NextTick_CreatesOpenInterestTick()
|
||||
{
|
||||
var dateTime = new DateTime(2000, 01, 01);
|
||||
var tick = _tickGenerator.NextTick(dateTime, TickType.OpenInterest, 10m);
|
||||
|
||||
Assert.AreEqual(dateTime, tick.Time);
|
||||
Assert.AreEqual(TickType.OpenInterest, tick.TickType);
|
||||
Assert.AreEqual(typeof(OpenInterest), tick.GetType());
|
||||
Assert.AreEqual(_symbol, tick.Symbol);
|
||||
Assert.GreaterOrEqual(tick.Quantity, 9000);
|
||||
Assert.LessOrEqual(tick.Quantity, 11000);
|
||||
Assert.AreEqual(tick.Value, tick.Quantity);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[TestCase(Resolution.Tick, DataDensity.Dense)]
|
||||
[TestCase(Resolution.Second, DataDensity.Dense)]
|
||||
[TestCase(Resolution.Minute, DataDensity.Dense)]
|
||||
[TestCase(Resolution.Hour, DataDensity.Dense)]
|
||||
[TestCase(Resolution.Daily, DataDensity.Dense)]
|
||||
[TestCase(Resolution.Tick, DataDensity.Sparse)]
|
||||
[TestCase(Resolution.Second, DataDensity.Sparse)]
|
||||
[TestCase(Resolution.Minute, DataDensity.Sparse)]
|
||||
[TestCase(Resolution.Hour, DataDensity.Sparse)]
|
||||
[TestCase(Resolution.Daily, DataDensity.Sparse)]
|
||||
[TestCase(Resolution.Tick, DataDensity.VerySparse)]
|
||||
[TestCase(Resolution.Second, DataDensity.VerySparse)]
|
||||
[TestCase(Resolution.Minute, DataDensity.VerySparse)]
|
||||
[TestCase(Resolution.Hour, DataDensity.VerySparse)]
|
||||
[TestCase(Resolution.Daily, DataDensity.VerySparse)]
|
||||
public void NextTickTime_CreatesTimes(Resolution resolution, DataDensity density)
|
||||
{
|
||||
var count = 100;
|
||||
var deltaSum = TimeSpan.Zero;
|
||||
var previous = new DateTime(2019, 01, 14, 9, 30, 0);
|
||||
var increment = resolution.ToTimeSpan();
|
||||
if (increment == TimeSpan.Zero)
|
||||
{
|
||||
increment = TimeSpan.FromMilliseconds(500);
|
||||
}
|
||||
|
||||
var marketHours = MarketHoursDatabase.FromDataFolder()
|
||||
.GetExchangeHours(_symbol.ID.Market, _symbol, _symbol.SecurityType);
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
var next = _tickGenerator.NextTickTime(previous, resolution, density);
|
||||
var barStart = next.Subtract(increment);
|
||||
Assert.Less(previous, next);
|
||||
Assert.IsTrue(marketHours.IsOpen(barStart, next, false));
|
||||
|
||||
var delta = next - previous;
|
||||
deltaSum += delta;
|
||||
|
||||
previous = next;
|
||||
}
|
||||
|
||||
var avgDelta = TimeSpan.FromTicks(deltaSum.Ticks / count);
|
||||
switch (density)
|
||||
{
|
||||
case DataDensity.Dense:
|
||||
// more frequent than once an increment
|
||||
Assert.Less(avgDelta, increment);
|
||||
break;
|
||||
|
||||
case DataDensity.Sparse:
|
||||
// less frequent that once an increment
|
||||
Assert.Greater(avgDelta, increment);
|
||||
break;
|
||||
|
||||
case DataDensity.VerySparse:
|
||||
// less frequent than one every 10 increments
|
||||
Assert.Greater(avgDelta, TimeSpan.FromTicks(increment.Ticks * 10));
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(density), density, null);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void HistoryIsNotEmpty()
|
||||
{
|
||||
var history = _tickGenerator.GenerateTicks().ToList();
|
||||
Assert.IsNotEmpty(history);
|
||||
Assert.That(history.Select(s => s.Symbol), Is.All.EqualTo(_symbol));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void HistoryIsBetweenStartAndEndDate()
|
||||
{
|
||||
var start = new DateTime(2020, 1, 6);
|
||||
var end = new DateTime(2020, 1, 10);
|
||||
var history = _tickGenerator.GenerateTicks().ToList();
|
||||
Assert.IsNotEmpty(history);
|
||||
Assert.That(history.All(s => start <= s.Time && s.Time <= end));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void HistoryGeneratesOpenInterestDataAsExpected()
|
||||
{
|
||||
var start = new DateTime(2020, 1, 6);
|
||||
var end = new DateTime(2020, 1, 10);
|
||||
var history = _tickGenerator.GenerateTicks().ToList();
|
||||
Assert.IsNotEmpty(history);
|
||||
Assert.AreEqual(3, history.Where(s => s.TickType == TickType.OpenInterest).Count());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
*
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using NUnit.Framework;
|
||||
using QuantConnect.Interfaces;
|
||||
using System;
|
||||
using System.Linq;
|
||||
using QuantConnect.Configuration;
|
||||
|
||||
namespace QuantConnect.Tests.ToolBox
|
||||
{
|
||||
[TestFixture]
|
||||
public class ToolBoxTests
|
||||
{
|
||||
[Test]
|
||||
public void ComposeDataQueueHandlerInstances()
|
||||
{
|
||||
var type = typeof(IDataQueueHandler);
|
||||
|
||||
var types = AppDomain.CurrentDomain.Load("QuantConnect.ToolBox")
|
||||
.GetTypes()
|
||||
.Where(p => type.IsAssignableFrom(p) && p.IsClass && !p.IsAbstract)
|
||||
.ToList();
|
||||
|
||||
Assert.Zero(types.Count);
|
||||
}
|
||||
|
||||
[TestCase("--app=RDG --tickers=AAPL --resolution=Daily --from-date=20200820-00:00:00 --to-date=20200830-00:00:00", 1)]
|
||||
[TestCase("--app=RDG --resolution=Daily --from-date=20200820-00:00:00 --to-date=20200830-00:00:00", 0)]
|
||||
[TestCase("--app=RDG --tickers=AAPL,SPY,TSLA --resolution=Daily --from-date=20200820-00:00:00 --to-date=20200830-00:00:00", 3)]
|
||||
[TestCase("--app=RDG --tickers=ES --security-type=Future --resolution=Minute --destination-dir=/Lean/Data", 1)]
|
||||
public void CanParseTickersCorrectly(string args, int expectedTcikerCount)
|
||||
{
|
||||
var options = ToolboxArgumentParser.ParseArguments(args.Split(' '));
|
||||
var tickers = ToolboxArgumentParser.GetTickers(options);
|
||||
|
||||
Assert.AreEqual(expectedTcikerCount, tickers.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user