chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
*
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Configuration;
|
||||
using McMaster.Extensions.CommandLineUtils;
|
||||
using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants;
|
||||
|
||||
namespace QuantConnect.DownloaderDataProvider.Launcher;
|
||||
|
||||
public static class DownloaderDataProviderArgumentParser
|
||||
{
|
||||
private const string ApplicationName = "QuantConnect.DownloaderDataProvider.exe";
|
||||
private const string ApplicationDescription = "Welcome to Lean Downloader Data Provider! 🚀 Easily download historical data from various sources with our user-friendly application. Start exploring financial data effortlessly!";
|
||||
private const string ApplicationHelpText = "Hm...";
|
||||
|
||||
private static readonly List<CommandLineOption> Options = new List<CommandLineOption>
|
||||
{
|
||||
new CommandLineOption(DownloaderCommandArguments.CommandDownloaderDataDownloader, CommandOptionType.SingleValue),
|
||||
new CommandLineOption(DownloaderCommandArguments.CommandDataType, CommandOptionType.SingleValue),
|
||||
new CommandLineOption(DownloaderCommandArguments.CommandTickers, CommandOptionType.MultipleValue),
|
||||
new CommandLineOption(DownloaderCommandArguments.CommandSecurityType, CommandOptionType.SingleValue),
|
||||
new CommandLineOption(DownloaderCommandArguments.CommandMarketName, CommandOptionType.SingleValue),
|
||||
new CommandLineOption(DownloaderCommandArguments.CommandResolution, CommandOptionType.SingleValue),
|
||||
new CommandLineOption(DownloaderCommandArguments.CommandStartDate, CommandOptionType.SingleValue),
|
||||
new CommandLineOption(DownloaderCommandArguments.CommandEndDate, CommandOptionType.SingleValue)
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Parses the command-line arguments and returns a dictionary containing parsed values.
|
||||
/// </summary>
|
||||
/// <param name="args">An array of command-line arguments.</param>
|
||||
/// <returns>A dictionary containing parsed values from the command-line arguments.</returns>
|
||||
/// <remarks>
|
||||
/// The <paramref name="args"/> parameter should contain the command-line arguments to be parsed.
|
||||
/// The method uses the ApplicationParser class to parse the arguments based on the ApplicationName,
|
||||
/// ApplicationDescription, ApplicationHelpText, and Options properties.
|
||||
/// </remarks>
|
||||
public static Dictionary<string, object> ParseArguments(string[] args)
|
||||
{
|
||||
return ApplicationParser.Parse(ApplicationName, ApplicationDescription, ApplicationHelpText, args, Options);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using QuantConnect.Util;
|
||||
using System.Globalization;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Brokerages;
|
||||
using QuantConnect.Configuration;
|
||||
using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants;
|
||||
|
||||
namespace QuantConnect.DownloaderDataProvider.Launcher.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Abstract base class for configuring data download parameters, including common properties and initialization logic.
|
||||
/// </summary>
|
||||
public abstract class BaseDataDownloadConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the start date for the data download.
|
||||
/// </summary>
|
||||
public DateTime StartDate { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the end date for the data download.
|
||||
/// </summary>
|
||||
public DateTime EndDate { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the resolution of the downloaded data.
|
||||
/// </summary>
|
||||
public Resolution Resolution { get; protected set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the market name for which the data will be downloaded.
|
||||
/// </summary>
|
||||
public string MarketName { get; protected set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the type of security for which the data is being downloaded.
|
||||
/// </summary>
|
||||
public SecurityType SecurityType { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the type of tick data to be downloaded.
|
||||
/// </summary>
|
||||
public TickType TickType { get; protected set; }
|
||||
|
||||
/// <summary>
|
||||
/// The type of data based on <see cref="TickTypes"/>
|
||||
/// </summary>
|
||||
public abstract Type DataType { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the list of symbols for which the data will be downloaded.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<Symbol> Symbols { get; protected set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BaseDataDownloadConfig"/> class.
|
||||
/// </summary>
|
||||
protected BaseDataDownloadConfig()
|
||||
{
|
||||
StartDate = ParseDate(Config.Get(DownloaderCommandArguments.CommandStartDate).ToString());
|
||||
EndDate = ParseDate(Config.Get(DownloaderCommandArguments.CommandEndDate).ToString());
|
||||
|
||||
SecurityType = ParseEnum<SecurityType>(Config.Get(DownloaderCommandArguments.CommandSecurityType).ToString());
|
||||
|
||||
MarketName = Config.Get(DownloaderCommandArguments.CommandMarketName).ToString().ToLower(CultureInfo.InvariantCulture);
|
||||
|
||||
if (string.IsNullOrEmpty(MarketName))
|
||||
{
|
||||
MarketName = DefaultBrokerageModel.DefaultMarketMap[SecurityType];
|
||||
Log.Trace($"{nameof(BaseDataDownloadConfig)}: Default market '{MarketName}' applied for SecurityType '{SecurityType}'");
|
||||
}
|
||||
|
||||
if (!Market.SupportedMarkets().Contains(MarketName))
|
||||
{
|
||||
throw new ArgumentException($"The specified market '{MarketName}' is not supported. Supported markets are: {string.Join(", ", Market.SupportedMarkets())}.");
|
||||
}
|
||||
|
||||
Symbols = LoadSymbols(Config.GetValue<Dictionary<string, string>>(DownloaderCommandArguments.CommandTickers), SecurityType, MarketName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="DataDownloadConfig"/> class with the specified parameters.
|
||||
/// </summary>
|
||||
/// <param name="tickType">The type of tick data to be downloaded.</param>
|
||||
/// <param name="securityType">The type of security for which data is being downloaded.</param>
|
||||
/// <param name="resolution">The resolution of the data being downloaded.</param>
|
||||
/// <param name="startDate">The start date for the data download range.</param>
|
||||
/// <param name="endDate">The end date for the data download range.</param>
|
||||
/// <param name="marketName">The name of the market from which the data is being downloaded.</param>
|
||||
/// <param name="symbols">A list of symbols for which data is being downloaded.</param>
|
||||
protected BaseDataDownloadConfig(TickType tickType, SecurityType securityType, Resolution resolution, DateTime startDate, DateTime endDate, string marketName, List<Symbol> symbols)
|
||||
{
|
||||
StartDate = startDate;
|
||||
EndDate = endDate;
|
||||
Resolution = resolution;
|
||||
MarketName = marketName;
|
||||
SecurityType = securityType;
|
||||
TickType = tickType;
|
||||
Symbols = symbols;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads the symbols for which data will be downloaded.
|
||||
/// </summary>
|
||||
/// <param name="tickers">A dictionary of tickers to load symbols for.</param>
|
||||
/// <param name="securityType">The type of security to download data for.</param>
|
||||
/// <param name="market">The market for which the symbols are valid.</param>
|
||||
/// <returns>A collection of symbols for the specified market and security type.</returns>
|
||||
/// <summary>
|
||||
private static IReadOnlyCollection<Symbol> LoadSymbols(Dictionary<string, string> tickers, SecurityType securityType, string market)
|
||||
{
|
||||
if (tickers == null || tickers.Count == 0)
|
||||
{
|
||||
throw new ArgumentException($"{nameof(BaseDataDownloadConfig)}.{nameof(LoadSymbols)}: The tickers dictionary cannot be null or empty.");
|
||||
}
|
||||
|
||||
return tickers.Keys.ToList((ticker) => ParseTicker(ticker, securityType, market));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse input 'ticker' to a Symbol or Canonical Symbol based on the provided security type and market.
|
||||
/// </summary>
|
||||
/// <param name="ticker">The ticker string input by the user.</param>
|
||||
/// <param name="securityType">The security type.</param>
|
||||
/// <param name="market">The market name.</param>
|
||||
/// <returns>A <see cref="Symbol"/> representing the specified security.</returns>
|
||||
private static Symbol ParseTicker(string ticker, SecurityType securityType, string market)
|
||||
{
|
||||
var symbol = default(Symbol);
|
||||
try
|
||||
{
|
||||
symbol = SymbolRepresentation.ParseTickerFromUserInput(ticker, securityType, market);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Debug($"{nameof(BaseDataDownloadConfig)}.{nameof(ParseTicker)}: Failed to parse symbol. Exception: {ex.Message}");
|
||||
}
|
||||
|
||||
return symbol ?? Symbol.Create(ticker, securityType, market);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses a string to a <see cref="DateTime"/> using a specific date format.
|
||||
/// </summary>
|
||||
/// <param name="date">The date string to parse.</param>
|
||||
/// <returns>The parsed <see cref="DateTime"/> value.</returns>
|
||||
protected static DateTime ParseDate(string date) => DateTime.ParseExact(date, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Parses a string value into an enum of the specified type.
|
||||
/// </summary>
|
||||
/// <typeparam name="TEnum">The enum type to parse the value into.</typeparam>
|
||||
/// <param name="value">The string value to parse.</param>
|
||||
/// <returns>The parsed enum value.</returns>
|
||||
/// <exception cref="ArgumentException">Thrown if the value cannot be parsed or is not a valid enum value.</exception>
|
||||
protected static TEnum ParseEnum<TEnum>(string value) where TEnum : struct, Enum
|
||||
{
|
||||
if (!Enum.TryParse(value, true, out TEnum result) || !Enum.IsDefined(result))
|
||||
{
|
||||
throw new ArgumentException($"Invalid {typeof(TEnum).Name} specified: '{value}'. Please provide a valid {typeof(TEnum).Name}. " +
|
||||
$"Valid values are: {string.Join(", ", Enum.GetNames<TEnum>())}.");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Packets;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Configuration;
|
||||
|
||||
namespace QuantConnect.DownloaderDataProvider.Launcher.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Class for downloading data from a brokerage.
|
||||
/// </summary>
|
||||
public class BrokerageDataDownloader : IDataDownloader, IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents the Brokerage implementation.
|
||||
/// </summary>
|
||||
private IBrokerage _brokerage;
|
||||
|
||||
/// <summary>
|
||||
/// Provides access to exchange hours and raw data times zones in various markets
|
||||
/// </summary>
|
||||
private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BrokerageDataDownloader"/> class.
|
||||
/// </summary>
|
||||
public BrokerageDataDownloader()
|
||||
{
|
||||
var liveNodeConfiguration = new LiveNodePacket()
|
||||
{
|
||||
Brokerage = Config.Get("data-downloader-brokerage"),
|
||||
UserToken = Globals.UserToken,
|
||||
UserId = Globals.UserId,
|
||||
ProjectId = Globals.ProjectId,
|
||||
OrganizationId = Globals.OrganizationID,
|
||||
Version = Globals.Version,
|
||||
DeploymentTarget = DeploymentTarget.LocalPlatform
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
// import the brokerage data for the configured brokerage
|
||||
var brokerageFactory = Composer.Instance.Single<IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveNodeConfiguration.Brokerage));
|
||||
liveNodeConfiguration.BrokerageData = brokerageFactory.BrokerageData;
|
||||
}
|
||||
catch (InvalidOperationException error)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(BrokerageDataDownloader)}.An error occurred while resolving brokerage data for a live job. Brokerage: {liveNodeConfiguration.Brokerage}.", error);
|
||||
}
|
||||
|
||||
_brokerage = Composer.Instance.GetExportedValueByTypeName<IBrokerage>(liveNodeConfiguration.Brokerage);
|
||||
|
||||
_brokerage.Message += (object _, Brokerages.BrokerageMessageEvent e) =>
|
||||
{
|
||||
if (e.Type == Brokerages.BrokerageMessageType.Error)
|
||||
{
|
||||
Logging.Log.Error(e.Message);
|
||||
}
|
||||
else
|
||||
{
|
||||
Logging.Log.Trace(e.Message);
|
||||
}
|
||||
};
|
||||
|
||||
((IDataQueueHandler)_brokerage).SetJob(liveNodeConfiguration);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
|
||||
/// </summary>
|
||||
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
|
||||
/// <returns>Enumerable of base data for this symbol</returns>
|
||||
public IEnumerable<BaseData>? Get(DataDownloaderGetParameters dataDownloaderGetParameters)
|
||||
{
|
||||
var symbol = dataDownloaderGetParameters.Symbol;
|
||||
var resolution = dataDownloaderGetParameters.Resolution;
|
||||
var startUtc = dataDownloaderGetParameters.StartUtc;
|
||||
var endUtc = dataDownloaderGetParameters.EndUtc;
|
||||
var tickType = dataDownloaderGetParameters.TickType;
|
||||
|
||||
var dataType = LeanData.GetDataType(resolution, tickType);
|
||||
var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
|
||||
var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
|
||||
|
||||
var symbols = new List<Symbol> { symbol };
|
||||
if (symbol.IsCanonical())
|
||||
{
|
||||
symbols = GetChainSymbols(symbol, true).ToList();
|
||||
}
|
||||
|
||||
return symbols
|
||||
.Select(symbol =>
|
||||
{
|
||||
var request = new Data.HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours: exchangeHours, dataTimeZone: dataTimeZone, resolution,
|
||||
// let's not ask for extended market hours for hour and daily resolutions to match lean
|
||||
includeExtendedMarketHours: resolution != Resolution.Hour && resolution != Resolution.Daily, false, DataNormalizationMode.Raw, tickType);
|
||||
|
||||
var history = _brokerage.GetHistory(request);
|
||||
|
||||
if (history == null)
|
||||
{
|
||||
Logging.Log.Trace($"{nameof(BrokerageDataDownloader)}.{nameof(Get)}: Ignoring history request for unsupported symbol {symbol}");
|
||||
}
|
||||
|
||||
return history;
|
||||
})
|
||||
.Where(history => history != null)
|
||||
.SelectMany(history => history);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns an IEnumerable of Future/Option contract symbols for the given root ticker
|
||||
/// </summary>
|
||||
/// <param name="symbol">The Symbol to get futures/options chain for</param>
|
||||
/// <param name="includeExpired">Include expired contracts</param>
|
||||
private IEnumerable<Symbol> GetChainSymbols(Symbol symbol, bool includeExpired)
|
||||
{
|
||||
if (_brokerage is IDataQueueUniverseProvider universeProvider)
|
||||
{
|
||||
return universeProvider.LookupSymbols(symbol, includeExpired);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(BrokerageDataDownloader)}.{nameof(GetChainSymbols)}: The current brokerage does not support fetching canonical symbols. Please ensure your brokerage instance supports this feature.");
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_brokerage.DisposeSafely();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
*
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
namespace QuantConnect.DownloaderDataProvider.Launcher.Models.Constants
|
||||
{
|
||||
public sealed class DownloaderCommandArguments
|
||||
{
|
||||
public const string CommandDownloaderDataDownloader = "data-downloader";
|
||||
|
||||
public const string CommandDataType = "data-type";
|
||||
|
||||
public const string CommandTickers = "tickers";
|
||||
|
||||
public const string CommandSecurityType = "security-type";
|
||||
|
||||
public const string CommandMarketName = "market";
|
||||
|
||||
public const string CommandResolution = "resolution";
|
||||
|
||||
public const string CommandStartDate = "start-date";
|
||||
|
||||
public const string CommandEndDate = "end-date";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Configuration;
|
||||
using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants;
|
||||
|
||||
namespace QuantConnect.DownloaderDataProvider.Launcher.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the configuration for downloading data.
|
||||
/// </summary>
|
||||
public sealed class DataDownloadConfig : BaseDataDownloadConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the type of data download.
|
||||
/// </summary>
|
||||
public override Type DataType { get => LeanData.GetDataType(Resolution, TickType); }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="DataDownloadConfig"/> class.
|
||||
/// </summary>s
|
||||
public DataDownloadConfig()
|
||||
{
|
||||
TickType = ParseEnum<TickType>(Config.Get(DownloaderCommandArguments.CommandDataType));
|
||||
Resolution = ParseEnum<Resolution>(Config.Get(DownloaderCommandArguments.CommandResolution));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="DataDownloadConfig"/> class with the specified parameters.
|
||||
/// </summary>
|
||||
/// <param name="tickType">The type of tick data to be downloaded.</param>
|
||||
/// <param name="securityType">The type of security for which data is being downloaded.</param>
|
||||
/// <param name="resolution">The resolution of the data being downloaded.</param>
|
||||
/// <param name="startDate">The start date for the data download range.</param>
|
||||
/// <param name="endDate">The end date for the data download range.</param>
|
||||
/// <param name="marketName">The name of the market from which the data is being downloaded.</param>
|
||||
/// <param name="symbols">A list of symbols for which data is being downloaded.</param>
|
||||
public DataDownloadConfig(TickType tickType, SecurityType securityType, Resolution resolution, DateTime startDate, DateTime endDate, string marketName, List<Symbol> symbols)
|
||||
: base(tickType, securityType, resolution, startDate, endDate, marketName, symbols)
|
||||
{ }
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
|
||||
namespace QuantConnect.DownloaderDataProvider.Launcher.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the configuration for downloading data for a universe of securities.
|
||||
/// </summary>
|
||||
public sealed class DataUniverseDownloadConfig : BaseDataDownloadConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the type of data universe download.
|
||||
/// </summary>
|
||||
public override Type DataType { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="DataUniverseDownloadConfig"/> class using configuration settings.
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">Thrown when an unsupported security type is specified.</exception>
|
||||
public DataUniverseDownloadConfig()
|
||||
{
|
||||
Resolution = Resolution.Daily;
|
||||
DataType = GetDataUniverseType(SecurityType);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves the corresponding data universe type based on the specified security type.
|
||||
/// </summary>
|
||||
/// <param name="securityType">The security type for which the data universe type is determined.</param>
|
||||
/// <returns>The corresponding <see cref="Type"/> of the data universe.</returns>
|
||||
/// <exception cref="NotImplementedException">
|
||||
/// Thrown when the specified <paramref name="securityType"/> is not supported.
|
||||
/// </exception>
|
||||
private static Type GetDataUniverseType(SecurityType securityType)
|
||||
{
|
||||
switch (securityType)
|
||||
{
|
||||
case SecurityType.Option:
|
||||
case SecurityType.IndexOption:
|
||||
return typeof(OptionUniverse);
|
||||
default:
|
||||
throw new NotImplementedException($"DataUniverseDownloadConfig.GetDataUniverseType(): The data universe type for SecurityType '{securityType}' is not implemented.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using NodaTime;
|
||||
using QuantConnect.Util;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Configuration;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using DataFeeds = QuantConnect.Lean.Engine.DataFeeds;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.DataDownloader;
|
||||
using QuantConnect.DownloaderDataProvider.Launcher.Models;
|
||||
using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants;
|
||||
|
||||
namespace QuantConnect.DownloaderDataProvider.Launcher;
|
||||
|
||||
public static class Program
|
||||
{
|
||||
/// <summary>
|
||||
/// Synchronizer in charge of guaranteeing a single operation per file path
|
||||
/// </summary>
|
||||
private readonly static KeyStringSynchronizer DiskSynchronizer = new();
|
||||
|
||||
/// <summary>
|
||||
/// The provider used to cache history data files
|
||||
/// </summary>
|
||||
private static readonly IDataCacheProvider _dataCacheProvider = new DiskDataCacheProvider(DiskSynchronizer);
|
||||
|
||||
/// <summary>
|
||||
/// Represents the time interval of 5 seconds.
|
||||
/// </summary>
|
||||
private static TimeSpan _logDisplayInterval = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Provides access to exchange hours and raw data times zones in various markets
|
||||
/// </summary>
|
||||
private static readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
|
||||
|
||||
/// <summary>
|
||||
/// The main entry point for the application.
|
||||
/// </summary>
|
||||
/// <param name="args">Command-line arguments passed to the application.</param>
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
// Parse report arguments and merge with config to use in the optimizer
|
||||
if (args.Length > 0)
|
||||
{
|
||||
Config.MergeCommandLineArgumentsWithConfiguration(DownloaderDataProviderArgumentParser.ParseArguments(args));
|
||||
}
|
||||
|
||||
var dataDownloaderSelector = InitializeConfigurations();
|
||||
|
||||
var commandDataType = Config.Get(DownloaderCommandArguments.CommandDataType).ToUpperInvariant();
|
||||
|
||||
var dataDownloadConfig = default(BaseDataDownloadConfig);
|
||||
switch (commandDataType)
|
||||
{
|
||||
case "UNIVERSE":
|
||||
dataDownloadConfig = new DataUniverseDownloadConfig();
|
||||
RunUniverseDownloader(dataDownloaderSelector.GetDataDownloader(dataDownloadConfig.DataType), dataDownloadConfig);
|
||||
break;
|
||||
case "TRADE":
|
||||
case "QUOTE":
|
||||
case "OPENINTEREST":
|
||||
dataDownloadConfig = new DataDownloadConfig();
|
||||
RunDownload(dataDownloaderSelector.GetDataDownloader(dataDownloadConfig.DataType), dataDownloadConfig, Globals.DataFolder, _dataCacheProvider);
|
||||
break;
|
||||
default:
|
||||
Log.Error($"QuantConnect.DownloaderDataProvider.Launcher: Unsupported command data type '{commandDataType}'. Valid options: UNIVERSE, TRADE, QUOTE, OPENINTEREST.");
|
||||
break;
|
||||
}
|
||||
|
||||
dataDownloaderSelector.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes a data download operation using the specified data downloader.
|
||||
/// </summary>
|
||||
/// <param name="dataDownloader">An instance of an object implementing the <see cref="IDataDownloader"/> interface, responsible for downloading data.</param>
|
||||
/// <param name="dataDownloadConfig">Configuration settings for the data download operation.</param>
|
||||
/// <param name="dataDirectory">The directory where the downloaded data will be stored.</param>
|
||||
/// <param name="dataCacheProvider">The provider used to cache history data files</param>
|
||||
/// <param name="mapSymbol">True if the symbol should be mapped while writing the data</param>
|
||||
/// <exception cref="ArgumentNullException">Thrown when <paramref name="dataDownloader"/> is null.</exception>
|
||||
public static void RunDownload(IDataDownloader dataDownloader, BaseDataDownloadConfig dataDownloadConfig, string dataDirectory, IDataCacheProvider dataCacheProvider, bool mapSymbol = true)
|
||||
{
|
||||
if (dataDownloader == null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(dataDownloader), "The data downloader instance cannot be null. Please ensure that a valid instance of data downloader is provided.");
|
||||
}
|
||||
|
||||
var totalDownloadSymbols = dataDownloadConfig.Symbols.Count;
|
||||
var completeSymbolCount = 0;
|
||||
var startDownloadUtcTime = DateTime.UtcNow;
|
||||
|
||||
foreach (var symbol in dataDownloadConfig.Symbols)
|
||||
{
|
||||
var downloadParameters = new DataDownloaderGetParameters(symbol, dataDownloadConfig.Resolution, dataDownloadConfig.StartDate, dataDownloadConfig.EndDate, dataDownloadConfig.TickType);
|
||||
|
||||
Log.Trace($"DownloaderDataProvider.Main(): Starting download {downloadParameters}");
|
||||
var downloadedData = dataDownloader.Get(downloadParameters);
|
||||
|
||||
if (downloadedData == null)
|
||||
{
|
||||
completeSymbolCount++;
|
||||
Log.Trace($"DownloaderDataProvider.RunDownload(): No data returned for {downloadParameters.Symbol}. The brokerage does not support this request (unsupported resolution, tick type, or asset type).");
|
||||
continue;
|
||||
}
|
||||
|
||||
var (dataTimeZone, exchangeTimeZone) = GetDataAndExchangeTimeZoneBySymbol(symbol);
|
||||
|
||||
var writer = new LeanDataWriter(dataDownloadConfig.Resolution, symbol, dataDirectory, dataDownloadConfig.TickType, dataCacheProvider, mapSymbol: mapSymbol);
|
||||
|
||||
var groupedData = DataFeeds.DownloaderDataProvider.FilterAndGroupDownloadDataBySymbol(
|
||||
downloadedData,
|
||||
symbol,
|
||||
dataDownloadConfig.DataType,
|
||||
exchangeTimeZone,
|
||||
dataTimeZone,
|
||||
downloadParameters.StartUtc,
|
||||
downloadParameters.EndUtc);
|
||||
|
||||
var lastLogStatusTime = DateTime.UtcNow;
|
||||
var dataWasWritten = false;
|
||||
|
||||
foreach (var data in groupedData)
|
||||
{
|
||||
dataWasWritten = true;
|
||||
writer.Write(data.Select(data =>
|
||||
{
|
||||
var utcNow = DateTime.UtcNow;
|
||||
if (utcNow - lastLogStatusTime >= _logDisplayInterval)
|
||||
{
|
||||
lastLogStatusTime = utcNow;
|
||||
Log.Trace($"Downloading data for {downloadParameters.Symbol}. Please hold on...");
|
||||
}
|
||||
return data;
|
||||
}));
|
||||
}
|
||||
|
||||
completeSymbolCount++;
|
||||
var symbolPercentComplete = (double)completeSymbolCount / totalDownloadSymbols * 100;
|
||||
Log.Trace($"DownloaderDataProvider.RunDownload(): {symbolPercentComplete:F2}% complete ({completeSymbolCount} out of {totalDownloadSymbols} symbols)");
|
||||
|
||||
if (!dataWasWritten)
|
||||
{
|
||||
Log.Trace($"DownloaderDataProvider.RunDownload(): No data found for {downloadParameters.Symbol} between " +
|
||||
$"{dataDownloadConfig.StartDate:yyyy-MM-dd} and {dataDownloadConfig.EndDate:yyyy-MM-dd}.");
|
||||
}
|
||||
else
|
||||
{
|
||||
Log.Trace($"DownloaderDataProvider.RunDownload(): Download completed for {downloadParameters.Symbol} at {downloadParameters.Resolution} resolution, " +
|
||||
$"covering the period from {dataDownloadConfig.StartDate} to {dataDownloadConfig.EndDate}.");
|
||||
}
|
||||
}
|
||||
Log.Trace($"All downloads completed in {(DateTime.UtcNow - startDownloadUtcTime).TotalSeconds:F2} seconds.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initiates the universe downloader using the provided configuration.
|
||||
/// </summary>
|
||||
/// <param name="dataDownloader">The data downloader instance.</param>
|
||||
/// <param name="dataUniverseDownloadConfig">The universe download configuration.</param>
|
||||
private static void RunUniverseDownloader(IDataDownloader dataDownloader, BaseDataDownloadConfig dataUniverseDownloadConfig)
|
||||
{
|
||||
foreach (var symbol in dataUniverseDownloadConfig.Symbols)
|
||||
{
|
||||
var universeDownloadParameters = new DataUniverseDownloaderGetParameters(symbol, dataUniverseDownloadConfig.StartDate, dataUniverseDownloadConfig.EndDate);
|
||||
UniverseExtensions.RunUniverseDownloader(dataDownloader, universeDownloadParameters);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves the data time zone and exchange time zone associated with the specified symbol.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol for which to retrieve time zones.</param>
|
||||
/// <returns>
|
||||
/// A tuple containing the data time zone and exchange time zone.
|
||||
/// The data time zone represents the time zone for data related to the symbol.
|
||||
/// The exchange time zone represents the time zone for trading activities related to the symbol.
|
||||
/// </returns>
|
||||
private static (DateTimeZone dataTimeZone, DateTimeZone exchangeTimeZone) GetDataAndExchangeTimeZoneBySymbol(Symbol symbol)
|
||||
{
|
||||
var entry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.SecurityType);
|
||||
return (entry.DataTimeZone, entry.ExchangeHours.TimeZone);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes various configurations for the application.
|
||||
/// This method sets up logging, data providers, map file providers, and factor file providers.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The method reads configuration values to determine whether debugging is enabled,
|
||||
/// which log handler to use, and which data, map file, and factor file providers to initialize.
|
||||
/// </remarks>
|
||||
/// <seealso cref="Log"/>
|
||||
/// <seealso cref="Config"/>
|
||||
/// <seealso cref="Composer"/>
|
||||
/// <seealso cref="ILogHandler"/>
|
||||
/// <seealso cref="IDataProvider"/>
|
||||
/// <seealso cref="IMapFileProvider"/>
|
||||
/// <seealso cref="IFactorFileProvider"/>
|
||||
public static DataDownloaderSelector InitializeConfigurations()
|
||||
{
|
||||
Log.DebuggingEnabled = Config.GetBool("debug-mode", false);
|
||||
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName<ILogHandler>(Config.Get("log-handler", "ConsoleLogHandler"));
|
||||
|
||||
var dataProvider = Composer.Instance.GetExportedValueByTypeName<IDataProvider>("DefaultDataProvider");
|
||||
var mapFileProvider = Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider"));
|
||||
var factorFileProvider = Composer.Instance.GetExportedValueByTypeName<IFactorFileProvider>(Config.Get("factor-file-provider", "LocalDiskFactorFileProvider"));
|
||||
|
||||
mapFileProvider.Initialize(dataProvider);
|
||||
factorFileProvider.Initialize(mapFileProvider, dataProvider);
|
||||
|
||||
var dataDownloader = Composer.Instance.GetExportedValueByTypeName<IDataDownloader>(Config.Get(DownloaderCommandArguments.CommandDownloaderDataDownloader));
|
||||
return new DataDownloaderSelector(dataDownloader, mapFileProvider, dataProvider, factorFileProvider);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<OutputType>Exe</OutputType>
|
||||
<RootNamespace>QuantConnect.DownloaderDataProvider.Launcher</RootNamespace>
|
||||
<AssemblyName>QuantConnect.DownloaderDataProvider.Launcher</AssemblyName>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AnalysisMode>AllEnabledByDefault</AnalysisMode>
|
||||
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
|
||||
<OutputPath>bin\$(Configuration)\</OutputPath>
|
||||
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<Description>QuantConnect LEAN Downloader Data Provider: Project - Main startup executable for download data from various sources with our Lean-friendly application.</Description>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>$(SelectedOptimization)</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="..\LICENSE">
|
||||
<Pack>True</Pack>
|
||||
<PackagePath></PackagePath>
|
||||
</None>
|
||||
<None Include="config.example.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Common\QuantConnect.csproj" />
|
||||
<ProjectReference Include="..\Configuration\QuantConnect.Configuration.csproj" />
|
||||
<ProjectReference Include="..\Engine\QuantConnect.Lean.Engine.csproj" />
|
||||
<ProjectReference Include="..\Logging\QuantConnect.Logging.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"map-file-provider": "LocalDiskMapFileProvider",
|
||||
"data-provider": "DefaultDataProvider",
|
||||
"log-handler": "ConsoleLogHandler",
|
||||
"factor-file-provider": "LocalDiskFactorFileProvider",
|
||||
|
||||
// The root directory of the data folder for this application
|
||||
"data-folder": "../../../Data/",
|
||||
|
||||
// To get your api access token go to quantconnect.com/account
|
||||
"job-user-id": "0",
|
||||
"api-access-token": "",
|
||||
"job-organization-id": "",
|
||||
|
||||
// Data downloader provider
|
||||
"data-downloader": "",
|
||||
|
||||
// Specifies the name of the brokerage service used for data downloading (optional).
|
||||
"data-downloader-brokerage": ""
|
||||
}
|
||||
Reference in New Issue
Block a user