chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,268 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using QuantConnect.Api;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Interfaces;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Configuration;
|
||||
|
||||
namespace QuantConnect.Lean.Engine.DataFeeds
|
||||
{
|
||||
/// <summary>
|
||||
/// An instance of the <see cref="IDataProvider"/> that will download and update data files as needed via QC's Api.
|
||||
/// </summary>
|
||||
public class ApiDataProvider : BaseDownloaderDataProvider
|
||||
{
|
||||
private decimal _purchaseLimit = Config.GetValue("data-purchase-limit", decimal.MaxValue); //QCC
|
||||
|
||||
private readonly HashSet<SecurityType> _unsupportedSecurityType;
|
||||
private readonly DataPricesList _dataPrices;
|
||||
private readonly IApi _api;
|
||||
private readonly bool _subscribedToIndiaEquityMapAndFactorFiles;
|
||||
private readonly bool _subscribedToUsaEquityMapAndFactorFiles;
|
||||
private readonly bool _subscribedToFutureMapAndFactorFiles;
|
||||
private volatile bool _invalidSecurityTypeLog;
|
||||
|
||||
/// <summary>
|
||||
/// Initialize a new instance of the <see cref="ApiDataProvider"/>
|
||||
/// </summary>
|
||||
public ApiDataProvider()
|
||||
{
|
||||
_unsupportedSecurityType = new HashSet<SecurityType> { SecurityType.Future, SecurityType.FutureOption, SecurityType.Index, SecurityType.IndexOption };
|
||||
|
||||
_api = Composer.Instance.GetPart<IApi>();
|
||||
|
||||
// If we have no value for organization get account preferred
|
||||
if (string.IsNullOrEmpty(Globals.OrganizationID))
|
||||
{
|
||||
var account = _api.ReadAccount();
|
||||
Globals.OrganizationID = account?.OrganizationId;
|
||||
Log.Trace($"ApiDataProvider(): Will use organization Id '{Globals.OrganizationID}'.");
|
||||
}
|
||||
|
||||
// Read in data prices and organization details
|
||||
_dataPrices = _api.ReadDataPrices(Globals.OrganizationID);
|
||||
var organization = _api.ReadOrganization(Globals.OrganizationID);
|
||||
|
||||
foreach (var productItem in organization.Products.Where(x => x.Type == ProductType.Data).SelectMany(product => product.Items))
|
||||
{
|
||||
if (productItem.Id == 37)
|
||||
{
|
||||
// Determine if the user is subscribed to Equity map and factor files (Data product Id 37)
|
||||
_subscribedToUsaEquityMapAndFactorFiles = true;
|
||||
}
|
||||
else if (productItem.Id == 137)
|
||||
{
|
||||
// Determine if the user is subscribed to Future map and factor files (Data product Id 137)
|
||||
_subscribedToFutureMapAndFactorFiles = true;
|
||||
}
|
||||
else if (productItem.Id == 172)
|
||||
{
|
||||
// Determine if the user is subscribed to India map and factor files (Data product Id 172)
|
||||
_subscribedToIndiaEquityMapAndFactorFiles = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify user has agreed to data provider agreements
|
||||
if (organization.DataAgreement.Signed)
|
||||
{
|
||||
//Log Agreement Highlights
|
||||
Log.Trace("ApiDataProvider(): Data Terms of Use has been signed. \r\n" +
|
||||
$" Find full agreement at: {_dataPrices.AgreementUrl} \r\n" +
|
||||
"==========================================================================\r\n" +
|
||||
$"CLI API Access Agreement: On {organization.DataAgreement.SignedTime:d} You Agreed:\r\n" +
|
||||
" - Display or distribution of data obtained through CLI API Access is not permitted. \r\n" +
|
||||
" - Data and Third Party Data obtained via CLI API Access can only be used for individual or internal employee's use.\r\n" +
|
||||
" - Data is provided in LEAN format can not be manipulated for transmission or use in other applications. \r\n" +
|
||||
" - QuantConnect is not liable for the quality of data received and is not responsible for trading losses. \r\n" +
|
||||
"==========================================================================");
|
||||
Thread.Sleep(TimeSpan.FromSeconds(3));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Log URL to go accept terms
|
||||
throw new InvalidOperationException($"ApiDataProvider(): Must agree to terms at {_dataPrices.AgreementUrl}, before using the ApiDataProvider");
|
||||
}
|
||||
|
||||
// Verify we have the balance to maintain our purchase limit, if not adjust it to meet our balance
|
||||
var balance = organization.Credit.Balance;
|
||||
if (balance < _purchaseLimit)
|
||||
{
|
||||
if (_purchaseLimit != decimal.MaxValue)
|
||||
{
|
||||
Log.Error("ApiDataProvider(): Purchase limit is greater than balance." +
|
||||
$" Setting purchase limit to balance : {balance}");
|
||||
}
|
||||
_purchaseLimit = balance;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves data to be used in an algorithm.
|
||||
/// If file does not exist, an attempt is made to download them from the api
|
||||
/// </summary>
|
||||
/// <param name="key">File path representing where the data requested</param>
|
||||
/// <returns>A <see cref="Stream"/> of the data requested</returns>
|
||||
public override Stream Fetch(string key)
|
||||
{
|
||||
return DownloadOnce(key, s =>
|
||||
{
|
||||
// Verify we have enough credit to handle this
|
||||
var pricePath = Api.Api.FormatPathForDataRequest(key);
|
||||
var price = _dataPrices.GetPrice(pricePath);
|
||||
|
||||
// No price found
|
||||
if (price == -1)
|
||||
{
|
||||
throw new ArgumentException($"ApiDataProvider.Fetch(): No price found for {pricePath}");
|
||||
}
|
||||
|
||||
if (_purchaseLimit < price)
|
||||
{
|
||||
throw new ArgumentException($"ApiDataProvider.Fetch(): Cost {price} for {pricePath} data exceeds remaining purchase limit: {_purchaseLimit}");
|
||||
}
|
||||
|
||||
if (DownloadData(key))
|
||||
{
|
||||
// Update our purchase limit.
|
||||
_purchaseLimit -= price;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Main filter to determine if this file needs to be downloaded
|
||||
/// </summary>
|
||||
/// <param name="filePath">File we are looking at</param>
|
||||
/// <returns>True if should download</returns>
|
||||
protected override bool NeedToDownload(string filePath)
|
||||
{
|
||||
// Ignore null
|
||||
if (filePath == null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Some security types can't be downloaded, lets attempt to extract that information
|
||||
if (LeanData.TryParseSecurityType(filePath, out SecurityType securityType, out var market) &&
|
||||
_unsupportedSecurityType.Contains(securityType) &&
|
||||
// We do support universe data for some security types (options and futures)
|
||||
!IsUniverseData(securityType, filePath))
|
||||
{
|
||||
// we do support future auxiliary data (map and factor files)
|
||||
if (securityType != SecurityType.Future || !IsAuxiliaryData(filePath))
|
||||
{
|
||||
if (!_invalidSecurityTypeLog)
|
||||
{
|
||||
// let's log this once. Will still use any existing data on disk
|
||||
_invalidSecurityTypeLog = true;
|
||||
Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (securityType == SecurityType.Equity && filePath.Contains("fine", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains("fundamental", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
// Ignore fine fundamental data requests
|
||||
return false;
|
||||
}
|
||||
|
||||
// Only download if it doesn't exist or is out of date.
|
||||
// Files are only "out of date" for non date based files (hour, daily, margins, etc.) because this data is stored all in one file
|
||||
var shouldDownload = !File.Exists(filePath) || filePath.IsOutOfDate();
|
||||
|
||||
if (shouldDownload)
|
||||
{
|
||||
if (securityType == SecurityType.Future)
|
||||
{
|
||||
if (!_subscribedToFutureMapAndFactorFiles)
|
||||
{
|
||||
throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
|
||||
"to download Future auxiliary data from QuantConnect. " +
|
||||
"Please visit https://www.quantconnect.com/datasets/quantconnect-us-futures-security-master for details.");
|
||||
}
|
||||
}
|
||||
// Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
|
||||
else if (!_subscribedToUsaEquityMapAndFactorFiles && market.Equals(Market.USA, StringComparison.InvariantCultureIgnoreCase)
|
||||
&& (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
|
||||
{
|
||||
throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
|
||||
"to download Equity data from QuantConnect. " +
|
||||
"Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
|
||||
}
|
||||
else if (!_subscribedToIndiaEquityMapAndFactorFiles && market.Equals(Market.India, StringComparison.InvariantCultureIgnoreCase)
|
||||
&& (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
|
||||
{
|
||||
throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
|
||||
"to download India data from QuantConnect. " +
|
||||
"Please visit https://www.quantconnect.com/datasets/truedata-india-equity-security-master for details.");
|
||||
}
|
||||
}
|
||||
|
||||
return shouldDownload;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempt to download data using the Api for and return a FileStream of that data.
|
||||
/// </summary>
|
||||
/// <param name="filePath">The path to store the file</param>
|
||||
/// <returns>A FileStream of the data</returns>
|
||||
protected virtual bool DownloadData(string filePath)
|
||||
{
|
||||
if (Log.DebuggingEnabled)
|
||||
{
|
||||
Log.Debug($"ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for {filePath}.");
|
||||
}
|
||||
|
||||
if (_api.DownloadData(filePath, Globals.OrganizationID))
|
||||
{
|
||||
Log.Trace($"ApiDataProvider.Fetch(): Successfully retrieved data for {filePath}.");
|
||||
return true;
|
||||
}
|
||||
// Failed to download; _api.DownloadData() will post error
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method to determine if this filepath is auxiliary data
|
||||
/// </summary>
|
||||
/// <param name="filepath">The target file path</param>
|
||||
/// <returns>True if this file is of auxiliary data</returns>
|
||||
private static bool IsAuxiliaryData(string filepath)
|
||||
{
|
||||
return filepath.Contains("map_files", StringComparison.InvariantCulture)
|
||||
|| filepath.Contains("factor_files", StringComparison.InvariantCulture)
|
||||
|| filepath.Contains("fundamental", StringComparison.InvariantCulture)
|
||||
|| filepath.Contains("shortable", StringComparison.InvariantCulture);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method to determine if this file path if for a universe file
|
||||
/// </summary>
|
||||
private static bool IsUniverseData(SecurityType securityType, string filepath)
|
||||
{
|
||||
return (securityType.IsOption() || securityType == SecurityType.Future) &&
|
||||
filepath.Contains("universes", StringComparison.InvariantCulture);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user