wehub/quantconnect--lean
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tests/Engine/DataProviders/ApiDataProviderTests.cs
+205
View File
@@ -0,0 +1,205 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.IO;
using NUnit.Framework;
using System.Threading;
using QuantConnect.Util;
using System.Threading.Tasks;
using System.Collections.Generic;
using QuantConnect.Lean.Engine.DataFeeds;
namespace QuantConnect.Tests.Engine.DataProviders
{
[TestFixture, Explicit("Requires configured api access, and also makes calls to data endpoints which are charging transactions")]
public class ApiDataProviderTests
{
private ApiDataProvider _dataProvider;
[OneTimeSetUp]
public void Setup()
{
_dataProvider = new ApiDataProvider();
}
// Note: Most of these cases reflect real data files included in the repo.
// May be useful for other tests. Only ones that aren't included are map and factor file zips
// CFD Cases
[TestCase("cfd/oanda/daily/xauusd.zip", 2, true)] // Daily stale data
[TestCase("cfd/oanda/daily/xauusd.zip", 0, false)] // Daily fresh data
[TestCase("cfd/oanda/hour/xauusd.zip", 6, true)] // Hourly stale data
[TestCase("cfd/oanda/hour/xauusd.zip", 0, false)] // Hourly fresh data
[TestCase("cfd/oanda/minute/xauusd/20140501_quote.zip", 10, false)] // Date based minute data
[TestCase("cfd/oanda/second/xauusd/20140501_quote.zip", 10, false)] // Date based second data
[TestCase("cfd/oanda/tick/xauusd/20140501_quote.zip", 10, false)] // Date based tick data
// Crypto Cases
[TestCase("crypto/coinbase/daily/btcusd_quote.zip", 6, true)] // Daily stale data
[TestCase("crypto/coinbase/daily/btcusd_quote.zip", 0, false)] // Daily fresh data
[TestCase("crypto/coinbase/minute/btcusd/20161007_trade.zip", 6, false)] // Date based minute data
[TestCase("crypto/coinbase/second/btcusd/20161007_trade.zip", 6, false)] // Date based second data
// Equity Cases
[TestCase("equity/usa/daily/aaa.zip", 4, true)] // Daily stale data
[TestCase("equity/usa/daily/aaa.zip", 0, false)] // Daily fresh data
[TestCase("equity/usa/hour/aapl.zip", 4, true)] // Hourly stale data
[TestCase("equity/usa/hour/aapl.zip", 0, false)] // Hourly fresh data
[TestCase("equity/usa/minute/aapl/20140605_quote.zip", 0, false)] // Date Based minute data
[TestCase("equity/usa/second/aig/20131004_trade.zip", 0, false)] // Date Based second data
[TestCase("equity/usa/tick/aig/20131007_quote.zip", 0, false)] // Date Based tick data
[TestCase("equity/usa/factor_files/aaa.csv", 5, true)] // Stale factor file
[TestCase("equity/usa/factor_files/aaa.csv", 0, false)] // Fresh factor file
[TestCase("equity/usa/factor_files/factor_files_20210601.zip", 25, false)] // Date based factor files ** Not included in repo
[TestCase("equity/usa/map_files/aaa.csv", 5, true)] // Stale map file
[TestCase("equity/usa/map_files/aaa.csv", 0, false)] // Fresh map file
[TestCase("equity/usa/map_files/map_files_20210601.zip", 12, false)] // Date based map files ** Not included in repo
[TestCase("equity/usa/fundamental/coarse/20140324.csv", 5, false)] // Date based fundamental coarse files
[TestCase("equity/usa/fundamental/fine/aapl/20140228.csv", 30, false)] // Date based fundamental fine files (Always false because we don't support fine files)
[TestCase("equity/usa/shortable/testbrokerage/dates/20140325.csv", 3, false)] // Date based shortable files
[TestCase("equity/usa/shortable/testbrokerage/symbols/aapl.csv", 10, true)] // Symbol based shortable files
[TestCase("equity/usa/universes/daily/qctest/20131007.csv", 10, false)] // Date based universe file
// Equity Option Cases
[TestCase("option/usa/minute/aapl/20140606_openinterest_american.zip", 10, false)] // Date based minute data
// Forex Cases
[TestCase("forex/oanda/daily/eurgbp.zip", 0, false)] // Daily fresh Forex data
[TestCase("forex/oanda/daily/eurgbp.zip", 10, true)] // Daily stale Forex data
[TestCase("forex/oanda/hour/eurusd.zip", 0, false)] // Hourly fresh Forex data
[TestCase("forex/oanda/hour/eurusd.zip", 10, true)] // Hourly stale Forex data
[TestCase("forex/oanda/minute/eurusd/20140501_quote.zip", 10, false)] // Date based Forex minute data
[TestCase("forex/oanda/second/eurusd/20140501_quote.zip", 10, false)] // Date based Forex second data
[TestCase("forex/oanda/tick/eurusd/20140501_quote.zip", 10, false)] // Date based Forex tick data
// Price Future Cases False because Unsupported
[TestCase("future/cboe/margins/VX.csv", 0, false)] // Fresh Margins data
[TestCase("future/cboe/margins/VX.csv", 10, false)] // Stale Margins data
[TestCase("future/comex/minute/gc/20131007_openinterest.zip", 10, false)] // Date based minute data
[TestCase("future/comex/tick/gc/20131007_openinterest.zip", 10, false)] // Date based tick data
// Future Option Cases ** All False because Unsupported
[TestCase("futureoption/cme/minute/es/20200320/20200105_quote_american.zip", 10, false)] // Date based minute data
// Index Cases / Index Option Cases *Not included because not allowed to download
public void ShouldDownloadTest(string file, int days, bool expected)
{
// First create our test file and set its time
var path = Path.Combine("./ApiDataTests", file);
var time = DateTime.Now - TimeSpan.FromDays(days);
var directory = Path.GetDirectoryName(path);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
File.Create(path).Dispose();
File.SetLastWriteTime(path, time);
using var dataProvider = new ApiDataProviderTest();
var result = dataProvider.NeedToDownloadExposed(path);
Assert.AreEqual(expected, result);
// Cleanup after test
File.Delete(path);
}
// Alternative
[TestCase("alternative/sec/aapl/19980108_8K.zip")]
// CFD
[TestCase("cfd/oanda/daily/ch20hkd.zip")]
// Crypto
[TestCase("crypto/coinbase/minute/btcusd/20150114_trade.zip")]
// Equities
[TestCase("equity/usa/shortable/atreyu/dates/20180117.csv")]
[TestCase("equity/usa/factor_files/tsla.csv")]
[TestCase("equity/usa/factor_files/factor_files_20210601.zip")]
[TestCase("equity/usa/map_files/googl.csv")]
[TestCase("equity/usa/map_files/map_files_20210601.zip")]
// Equity Options
[TestCase("option/usa/minute/aapl/20100603_quote_american.zip")]
// Forex
[TestCase("forex/oanda/minute/eurusd/20020516_quote.zip")]
// Price Futures * False because unsupported
[TestCase("future/cbot/minute/zs/20090501_trade.zip", false)]
[TestCase("future/sgx/margins/IN.csv", false)]
// Auxiliary Data Future Cases true
[TestCase("future/comex/map_files/map_files_20211225.zip", true)]
[TestCase("future/cme/factor_files/factor_files_20211225.zip", true)]
// Future Options * False because unsupported
[TestCase("futureoption/comex/minute/og/20120227/20120105_quote_american.zip", false)]
// Index * False because unsupported
[TestCase("index/usa/minute/spx/19980217_trade.zip", false)]
// Index Options * False because unsupported
[TestCase("indexoption/usa/minute/spx/20100603_quote_european.zip", false)]
public void DownloadFiles(string file, bool expectedToExist = true)
{
var path = Path.Combine(Globals.DataFolder, file);
// Delete it if we already have it
if (File.Exists(path))
{
File.Delete(path);
Assert.IsFalse(File.Exists(path));
}
// Go get the file
var stream = _dataProvider.Fetch(path);
if (expectedToExist)
{
Assert.IsNotNull(stream);
stream.Dispose();
Assert.IsTrue(File.Exists(path));
}
else
{
Assert.IsNull(stream);
Assert.IsFalse(File.Exists(path));
}
}
[Test]
public void DownloadsFileOnceConcurrently()
{
# pragma warning disable CA2000
var dataProvider = new ApiDataProviderTest();
#pragma warning restore CA2000
var tasks = new List<Task>();
for (var i = 0; i < 10; i++)
{
tasks.Add(Task.Factory.StartNew(() =>
dataProvider.Fetch(Path.Combine(Globals.DataFolder, "equity", Market.USA, "hour", "f.zip"))));
}
tasks.ForEach(task => task.Wait());
dataProvider.DisposeSafely();
Assert.AreEqual(1, dataProvider.DownLoadCount);
}
private class ApiDataProviderTest : ApiDataProvider
{
public int DownLoadCount;
public bool NeedToDownloadExposed(string filePath)
{
return base.NeedToDownload(filePath);
}
protected override bool DownloadData(string filePath)
{
Interlocked.Increment(ref DownLoadCount);
// simulate download delay
Thread.Sleep(100);
return false;
}
}
}
}
Tests/Engine/DataProviders/DefaultDataProviderTests.cs
+49
View File
@@ -0,0 +1,49 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using NUnit.Framework;
using QuantConnect.Lean.Engine.DataFeeds;
namespace QuantConnect.Tests.Engine.DataProviders
{
[TestFixture]
public class DefaultDataProviderTests
{
private DefaultDataProvider _defaultDataProvider;
[OneTimeSetUp]
public void Setup()
{
_defaultDataProvider = new DefaultDataProvider();
}
[Test]
public void DefaultDataProvider_CanReadDataThatExists()
{
var stream = _defaultDataProvider.Fetch("../../../Data/equity/usa/minute/aapl/20140606_trade.zip");
Assert.IsNotNull(stream);
}
[Test]
public void DefaultDataProvider_CannotReadDataThatDoesNotExist()
{
var stream = _defaultDataProvider.Fetch("../../../Data/equity/usa/minute/aapl/19980606_trade.zip");
Assert.IsNull(stream);
}
}
}
Tests/Engine/DataProviders/ProcessedDataProviderTests.cs
+74
View File
@@ -0,0 +1,74 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using NUnit.Framework;
using QuantConnect.Configuration;
using QuantConnect.Lean.Engine.DataFeeds;
using System.IO;
namespace QuantConnect.Tests.Engine.DataProviders
{
[TestFixture]
public class ProcessedDataProviderTests
{
private ProcessedDataProvider _processedDataProvider;
private string _originalProcessedDataDirectory;
[OneTimeSetUp]
public void Setup()
{
// Set up the processed data provider
_originalProcessedDataDirectory = Config.Get("processed-data-directory");
Config.Set("processed-data-directory", "TestData");
_processedDataProvider = new ProcessedDataProvider();
}
[OneTimeTearDown]
public void TearDown()
{
_processedDataProvider.Dispose();
// Revert the configuration
Config.Set("processed-data-directory", _originalProcessedDataDirectory);
}
private static TestCaseData[] ExistingDataTestCases => new[]
{
// Exists in the processed data directory
new TestCaseData(Path.Combine(Globals.DataFolder, "spy_10_min.txt"), true),
// Does not exist in the processed data directory but exists in the main data folder
new TestCaseData(Path.Combine(Globals.DataFolder, "equity/usa/minute/aapl/20140606_trade.zip"), true),
// Neither the processed data directory nor in the main data folder
new TestCaseData(Path.Combine(Globals.DataFolder, "equity/usa/minute/somestock/19980606_trade.zip"), false)
};
[TestCaseSource(nameof(ExistingDataTestCases))]
public void ProcessedDataProvider_CanReadDataThatExists(string path, bool exists)
{
var stream = _processedDataProvider.Fetch(path);
if (exists)
{
Assert.IsNotNull(stream);
}
else
{
Assert.IsNull(stream);
}
}
}
}