wehub/quantconnect--lean
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tests/ToolBox/LeanDataReaderTests.cs
+577
View File
@@ -0,0 +1,577 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using NUnit.Framework;
using QuantConnect.Data;
using QuantConnect.Data.Auxiliary;
using QuantConnect.Securities;
using QuantConnect.ToolBox;
using QuantConnect.Util;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Data.Consolidators;
using QuantConnect.Data.Market;
namespace QuantConnect.Tests.ToolBox
{
[TestFixture, Parallelizable(ParallelScope.Fixtures)]
public class LeanDataReaderTests
{
string _dataDirectory = "../../../Data/";
DateTime _fromDate = new DateTime(2013, 10, 7);
DateTime _toDate = new DateTime(2013, 10, 11);
[Test, Parallelizable(ParallelScope.Self)]
public void LoadsEquity_Daily_SingleEntryZip()
{
var dataPath = LeanData.GenerateZipFilePath(Globals.DataFolder, Symbols.AAPL, DateTime.UtcNow, Resolution.Daily, TickType.Trade);
var leanDataReader = new LeanDataReader(dataPath);
var data = leanDataReader.Parse().ToList();
Assert.AreEqual(5849, data.Count);
Assert.IsTrue(data.All(baseData => baseData.Symbol == Symbols.AAPL && baseData is TradeBar));
}
#region futures
[Test, Parallelizable(ParallelScope.Self)]
public void ReadsEntireZipFileEntries_OpenInterest()
{
var baseFuture = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, SecurityIdentifier.DefaultDate);
var filePath = LeanData.GenerateZipFilePath(Globals.DataFolder, baseFuture, new DateTime(2013, 10, 06), Resolution.Minute, TickType.OpenInterest);
var leanDataReader = new LeanDataReader(filePath);
var data = leanDataReader.Parse()
.ToList()
.GroupBy(baseData => baseData.Symbol)
.Select(grp => grp.ToList())
.OrderBy(list => list[0].Symbol)
.ToList();
Assert.AreEqual(5, data.Count);
Assert.IsTrue(data.All(kvp => kvp.Count == 1));
foreach (var dataForSymbol in data)
{
Assert.IsTrue(dataForSymbol[0] is OpenInterest);
Assert.IsFalse(dataForSymbol[0].Symbol.IsCanonical());
Assert.AreEqual(Futures.Indices.SP500EMini, dataForSymbol[0].Symbol.ID.Symbol);
Assert.AreNotEqual(0, dataForSymbol[0]);
}
}
[Test, Parallelizable(ParallelScope.Self)]
public void ReadsEntireZipFileEntries_Trade()
{
var baseFuture = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, SecurityIdentifier.DefaultDate);
var filePath = LeanData.GenerateZipFilePath(Globals.DataFolder, baseFuture, new DateTime(2013, 10, 06), Resolution.Minute, TickType.Trade);
var leanDataReader = new LeanDataReader(filePath);
var data = leanDataReader.Parse()
.ToList()
.GroupBy(baseData => baseData.Symbol)
.Select(grp => grp.ToList())
.OrderBy(list => list[0].Symbol)
.ToList();
Assert.AreEqual(2, data.Count);
foreach (var dataForSymbol in data)
{
Assert.IsTrue(dataForSymbol[0] is TradeBar);
Assert.IsFalse(dataForSymbol[0].Symbol.IsCanonical());
Assert.AreEqual(Futures.Indices.SP500EMini, dataForSymbol[0].Symbol.ID.Symbol);
}
Assert.AreEqual(118, data[0].Count);
Assert.AreEqual(10, data[1].Count);
}
[Test, Parallelizable(ParallelScope.Self)]
public void ReadsEntireZipFileEntries_Quote()
{
var baseFuture = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, SecurityIdentifier.DefaultDate);
var filePath = LeanData.GenerateZipFilePath(Globals.DataFolder, baseFuture, new DateTime(2013, 10, 06), Resolution.Minute, TickType.Quote);
var leanDataReader = new LeanDataReader(filePath);
var data = leanDataReader.Parse()
.ToList()
.GroupBy(baseData => baseData.Symbol)
.Select(grp => grp.ToList())
.OrderBy(list => list[0].Symbol)
.ToList();
Assert.AreEqual(5, data.Count);
foreach (var dataForSymbol in data)
{
Assert.IsTrue(dataForSymbol[0] is QuoteBar);
Assert.IsFalse(dataForSymbol[0].Symbol.IsCanonical());
Assert.AreEqual(Futures.Indices.SP500EMini, dataForSymbol[0].Symbol.ID.Symbol);
}
Assert.AreEqual(10, data[0].Count);
Assert.AreEqual(13, data[1].Count);
Assert.AreEqual(52, data[2].Count);
Assert.AreEqual(155, data[3].Count);
Assert.AreEqual(100, data[4].Count);
}
[Test]
public void ReadFutureChainData()
{
var canonicalFutures = new Dictionary<Symbol, string>()
{
{ Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME),
"ES20Z13|ES21H14|ES20M14|ES19U14|ES19Z14" },
{Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
"GC29V13|GC26X13|GC27Z13|GC26G14|GC28J14|GC26M14|GC27Q14|GC29V14|GC29Z14|GC25G15|GC28J15|GC26M15|GC27Q15|GC29Z15|GC28M16|GC28Z16|GC28M17|GC27Z17|GC27M18|GC27Z18|GC26M19"},
};
var tickTypes = new[] { TickType.Trade, TickType.Quote, TickType.OpenInterest };
var resolutions = new[] { Resolution.Minute };
foreach (var canonical in canonicalFutures)
{
foreach (var res in resolutions)
{
foreach (var tickType in tickTypes)
{
var futures = LoadFutureChain(canonical.Key, _fromDate, tickType, res);
string chain = string.Join("|", futures.Select(f => f.Value));
if (tickType == TickType.Quote) //only quotes have the full chain!
Assert.AreEqual(canonical.Value, chain);
foreach (var future in futures)
{
string csv = LoadFutureData(future, tickType, res);
Assert.IsTrue(!string.IsNullOrEmpty(csv));
}
}
}
}
}
private List<Symbol> LoadFutureChain(Symbol baseFuture, DateTime date, TickType tickType, Resolution res)
{
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, baseFuture, date, res, tickType);
//load future chain first
var config = new SubscriptionDataConfig(typeof(ZipEntryNameData), baseFuture, res,
TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType);
var factory = new ZipEntryNameSubscriptionDataSourceReader(TestGlobals.DataCacheProvider, config, date, false);
var result = factory.Read(new SubscriptionDataSource(filePath, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName))
.Select(s => s.Symbol).ToList();
return result;
}
private string LoadFutureData(Symbol future, TickType tickType, Resolution res)
{
var dataType = LeanData.GetDataType(res, tickType);
var config = new SubscriptionDataConfig(dataType, future, res,
TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType);
var date = _fromDate;
var sb = new StringBuilder();
while (date <= _toDate)
{
var leanDataReader = new LeanDataReader(config, future, res, date, _dataDirectory);
foreach (var bar in leanDataReader.Parse())
{
//write base data type back to string
sb.AppendLine(LeanData.GenerateLine(bar, SecurityType.Future, res));
}
date = date.AddDays(1);
}
var csv = sb.ToString();
return csv;
}
[Test]
public void GenerateDailyAndHourlyFutureDataFromMinutes()
{
var tickTypes = new[] { TickType.Trade, TickType.Quote, TickType.OpenInterest };
var futures = new[] { Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME),
Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX)};
var resolutions = new[] { Resolution.Hour, Resolution.Daily };
foreach (var future in futures)
foreach (var res in resolutions)
foreach (var tickType in tickTypes)
ConvertMinuteFuturesData(future, tickType, res);
}
private void ConvertMinuteFuturesData(Symbol canonical, TickType tickType, Resolution outputResolution, Resolution inputResolution = Resolution.Minute)
{
var timeSpans = new Dictionary<Resolution, TimeSpan>()
{
{ Resolution.Daily, TimeSpan.FromHours(24)},
{ Resolution.Hour, TimeSpan.FromHours(1)},
};
var timeSpan = timeSpans[outputResolution];
var tickTypeConsolidatorMap = new Dictionary<TickType, Func<IDataConsolidator>>()
{
{TickType.Quote, () => new QuoteBarConsolidator(timeSpan)},
{TickType.OpenInterest, ()=> new OpenInterestConsolidator(timeSpan)},
{TickType.Trade, ()=> new TradeBarConsolidator(timeSpan) }
};
var consolidators = new Dictionary<string, IDataConsolidator>();
var configs = new Dictionary<string, SubscriptionDataConfig>();
var outputFiles = new Dictionary<string, StringBuilder>();
var futures = new Dictionary<string, Symbol>();
var date = _fromDate;
while (date <= _toDate)
{
var futureChain = LoadFutureChain(canonical, date, tickType, inputResolution);
foreach (var future in futureChain)
{
if (!futures.ContainsKey(future.Value))
{
futures[future.Value] = future;
var config = new SubscriptionDataConfig(LeanData.GetDataType(outputResolution, tickType),
future, inputResolution, TimeZones.NewYork, TimeZones.NewYork,
false, false, false, false, tickType);
configs[future.Value] = config;
consolidators[future.Value] = tickTypeConsolidatorMap[tickType].Invoke();
var sb = new StringBuilder();
outputFiles[future.Value] = sb;
consolidators[future.Value].DataConsolidated += (sender, bar) =>
{
sb.Append(LeanData.GenerateLine(bar, SecurityType.Future, outputResolution) + Environment.NewLine);
};
}
var leanDataReader = new LeanDataReader(configs[future.Value], future, inputResolution, date, _dataDirectory);
var consolidator = consolidators[future.Value];
foreach (var bar in leanDataReader.Parse())
{
consolidator.Update(bar);
}
}
date = date.AddDays(1);
}
//write all results
foreach (var consolidator in consolidators.Values)
consolidator.Scan(date);
var zip = LeanData.GenerateRelativeZipFilePath(canonical, _fromDate, outputResolution, tickType);
var zipPath = Path.Combine(_dataDirectory, zip);
var fi = new FileInfo(zipPath);
if (!fi.Directory.Exists)
fi.Directory.Create();
foreach (var future in futures.Values)
{
var zipEntry = LeanData.GenerateZipEntryName(future, _fromDate, outputResolution, tickType);
var sb = outputFiles[future.Value];
//Uncomment to write zip files
//QuantConnect.Compression.ZipCreateAppendData(zipPath, zipEntry, sb.ToString());
Assert.IsTrue(sb.Length > 0);
}
}
#endregion
[Test, TestCaseSource(nameof(OptionAndFuturesCases))]
public void ReadLeanFutureAndOptionDataFromFilePath(string composedFilePath, Symbol symbol, int rowsInfile, double sumValue)
{
// Act
var ldr = new LeanDataReader(composedFilePath);
var data = ldr.Parse().ToList();
// Assert
Assert.True(symbol.Equals(data.First().Symbol));
Assert.AreEqual(rowsInfile, data.Count);
Assert.AreEqual(sumValue, data.Sum(c => c.Value));
}
public static object[] OptionAndFuturesCases =
{
new object[]
{
"../../../Data/future/cme/minute/es/20131008_quote.zip#20131008_es_minute_quote_201312.csv",
LeanData
.ReadSymbolFromZipEntry(Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME),
Resolution.Minute, "20131008_es_minute_quote_201312.csv"),
1411,
2346061.875
},
new object[]
{
"../../../Data/future/comex/minute/gc/20131010_trade.zip#20131010_gc_minute_trade_201312.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
Resolution.Minute, "20131010_gc_minute_trade_201312.csv"),
1379,
1791800.9
},
new object[]
{
"../../../Data/future/comex/tick/gc/20131009_quote.zip#20131009_gc_tick_quote_201406.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
Resolution.Tick, "20131009_gc_tick_quote_201406.csv"),
197839,
259245064.8
},
new object[]
{
"../../../Data/future/comex/tick/gc/20131009_trade.zip#20131009_gc_tick_trade_201312.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
Resolution.Tick, "20131009_gc_tick_trade_201312.csv"),
64712,
84596673.8
},
new object[]
{
"../../../Data/future/cme/minute/es/20131010_openinterest.zip#20131010_es_minute_openinterest_201312.csv",
LeanData
.ReadSymbolFromZipEntry(Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME),
Resolution.Minute, "20131010_es_minute_openinterest_201312.csv"),
3,
8119169
},
new object[]
{
"../../../Data/future/comex/tick/gc/20131009_openinterest.zip#20131009_gc_tick_openinterest_201310.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create(Futures.Metals.Gold, SecurityType.Future, Market.COMEX),
Resolution.Tick, "20131009_gc_tick_openinterest_201310.csv"),
4,
1312
},
new object[]
{
"../../../Data/option/usa/minute/aapl/20140606_quote_american.zip#20140606_aapl_minute_quote_american_put_7500000_20141018.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
Resolution.Minute,
"20140606_aapl_minute_quote_american_put_7500000_20141018.csv"),
391,
44210.7
},
new object[]
{
"../../../Data/option/usa/minute/aapl/20140606_trade_american.zip#20140606_aapl_minute_trade_american_call_6475000_20140606.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
Resolution.Minute,
"20140606_aapl_minute_trade_american_call_6475000_20140606.csv"),
374,
745.35
},
new object[]
{
"../../../Data/option/usa/minute/goog/20151224_openinterest_american.zip#20151224_goog_minute_openinterest_american_call_3000000_20160115.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create("GOOG", SecurityType.Option, Market.USA),
Resolution.Minute,
"20151224_goog_minute_openinterest_american_call_3000000_20160115.csv"),
1,
38
},
new object[]
{
"../../../Data/option/usa/daily/aapl_2014_openinterest_american.zip#aapl_openinterest_american_call_1950000_20150117.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
Resolution.Daily,
"aapl_openinterest_american_call_1950000_20150117.csv"),
2,
824
},
new object[]
{
"../../../Data/option/usa/daily/aapl_2014_trade_american.zip#aapl_trade_american_call_5400000_20141018.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
Resolution.Daily,
"aapl_trade_american_call_5400000_20141018.csv"),
1,
109.9
},
new object[]
{
"../../../Data/option/usa/daily/aapl_2014_quote_american.zip#aapl_quote_american_call_307100_20150117.csv",
LeanData.ReadSymbolFromZipEntry(Symbol.Create("AAPL", SecurityType.Option, Market.USA),
Resolution.Daily,
"aapl_quote_american_call_307100_20150117.csv"),
1,
63.3
}
};
[Test, TestCaseSource(nameof(SpotMarketCases))]
public void ReadLeanSpotMarketsSecuritiesDataFromFilePath(string securityType, string market, string resolution, string ticker, string fileName, int rowsInfile, double sumValue)
{
// Arrange
var filepath = GenerateFilepathForTesting(_dataDirectory, securityType, market, resolution, ticker, fileName);
SecurityType securityTypeEnum;
Enum.TryParse(securityType, true, out securityTypeEnum);
var symbol = Symbol.Create(ticker, securityTypeEnum, market);
// Act
var ldr = new LeanDataReader(filepath);
var data = ldr.Parse().ToList();
// Assert
Assert.True(symbol.Equals(data.First().Symbol));
Assert.AreEqual(rowsInfile, data.Count);
Assert.AreEqual(sumValue, data.Sum(c => c.Value));
}
public static object[] SpotMarketCases =
{
//TODO: generate Low resolution sample data for equities
new object[] {"equity", "usa", "daily", "aig", "aig.zip", 5849, 340770.5801},
new object[] {"equity", "usa", "minute", "aapl", "20140605_trade.zip", 686, 443184.58},
new object[] {"equity", "usa", "minute", "ibm", "20131010_quote.zip", 584, 107061.125},
new object[] {"equity", "usa", "second", "ibm", "20131010_trade.zip", 5060, 929385.34},
new object[] {"equity", "usa", "tick", "bac", "20131011_trade.zip", 112177, 1591680.73},
new object[] {"forex", "oanda", "minute", "eurusd", "20140502_quote.zip", 1222, 1693.578875},
new object[] {"forex", "oanda", "second", "nzdusd", "20140514_quote.zip", 18061, 15638.724575},
new object[] {"forex", "oanda", "tick", "eurusd", "20140507_quote.zip", 41367, 57598.54664},
new object[] {"cfd", "oanda", "hour", "xauusd", "xauusd.zip", 76499, 90453133.772 },
new object[] {"crypto", "coinbase", "second", "btcusd", "20161008_trade.zip", 3453, 2137057.57},
new object[] {"crypto", "coinbase", "minute", "ethusd", "20170903_trade.zip", 1440, 510470.66},
new object[] {"crypto", "coinbase", "daily", "btcusd", "btcusd_trade.zip", 1318, 3725052.03},
};
private static IEnumerable<TestCaseData> MinuteZipEntryFileNames
{
get
{
yield return new TestCaseData(
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Grains.Corn, Market.CBOT, new(2025, 12, 12))),
"20251103_ozc_minute_trade_american_call_41000_20251121.csv",
4.1m,
OptionRight.Call,
new DateTime(2025, 11, 21));
yield return new TestCaseData(
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Currencies.JPY, Market.CME, new(2025, 12, 15))),
"20251103_jpu_minute_trade_american_call_62.50000_20260109.csv",
0.00625m,
OptionRight.Call,
new DateTime(2026, 01, 09));
yield return new TestCaseData(
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Currencies.AUD, Market.CME, new(2025, 12, 15))),
"20251103_adu_minute_openinterest_american_put_6350_20251205.csv",
0.635m,
OptionRight.Put,
new DateTime(2025, 12, 05));
yield return new TestCaseData(
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Currencies.AUD, Market.CME, new(2025, 12, 15))),
"20251103_adu_minute_quote_american_call_8400_20260306.csv",
0.84m,
OptionRight.Call,
new DateTime(2026, 03, 06));
yield return new TestCaseData(
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Currencies.NZD, Market.CME, new(2025, 12, 15))),
"20251103_6n_minute_quote_american_call_5600_20260403.csv",
0.56m,
OptionRight.Call,
new DateTime(2026, 04, 03));
yield return new TestCaseData(
Symbol.CreateCanonicalOption(Symbol.CreateFuture(Futures.Meats.LiveCattle, Market.CME, new(2025, 12, 31))),
"20251103_le_minute_quote_american_call_21800_20251205.csv",
2.18m,
OptionRight.Call,
new DateTime(2025, 12, 05));
}
}
[TestCaseSource(nameof(MinuteZipEntryFileNames))]
public void ReadSymbolFromZipEntryShouldParseFileNameWithFloatingNumber(Symbol rootSymbol, string fileNameCsv,
decimal expectedStrike, OptionRight expectedOptionRight, DateTime expectedExpiry)
{
var actualSymbol = LeanData.ReadSymbolFromZipEntry(rootSymbol, Resolution.Minute, fileNameCsv);
Assert.AreEqual(expectedStrike, actualSymbol.ID.StrikePrice);
Assert.AreEqual(expectedOptionRight, actualSymbol.ID.OptionRight);
Assert.AreEqual(expectedExpiry, actualSymbol.ID.Date);
}
public static string GenerateFilepathForTesting(string dataDirectory, string securityType, string market, string resolution, string ticker,
string fileName)
{
string filepath;
if (resolution == "daily" || resolution == "hour")
{
filepath = Path.Combine(dataDirectory, securityType, market, resolution, fileName);
}
else
{
filepath = Path.Combine(dataDirectory, securityType, market, resolution, ticker, fileName);
}
return filepath;
}
private class ZipEntryNameData : BaseData
{
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
var symbol = LeanData.ReadSymbolFromZipEntry(config.Symbol, config.Resolution, line);
return new ZipEntryNameData { Time = date, Symbol = symbol };
}
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType);
return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName);
}
}
}
}