chore: import upstream snapshot with attribution

2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
@@ -0,0 +1,149 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Util;
+using QuantConnect.Data;
+using QuantConnect.Packets;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Configuration;
+
+namespace QuantConnect.DownloaderDataProvider.Launcher.Models
+{
+    /// <summary>
+    /// Class for downloading data from a brokerage.
+    /// </summary>
+    public class BrokerageDataDownloader : IDataDownloader, IDisposable
+    {
+        /// <summary>
+        /// Represents the Brokerage implementation.
+        /// </summary>
+        private IBrokerage _brokerage;
+
+        /// <summary>
+        /// Provides access to exchange hours and raw data times zones in various markets
+        /// </summary>
+        private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="BrokerageDataDownloader"/> class.
+        /// </summary>
+        public BrokerageDataDownloader()
+        {
+            var liveNodeConfiguration = new LiveNodePacket()
+            {
+                Brokerage = Config.Get("data-downloader-brokerage"),
+                UserToken = Globals.UserToken,
+                UserId = Globals.UserId,
+                ProjectId = Globals.ProjectId,
+                OrganizationId = Globals.OrganizationID,
+                Version = Globals.Version,
+                DeploymentTarget = DeploymentTarget.LocalPlatform
+            };
+
+            try
+            {
+                // import the brokerage data for the configured brokerage
+                var brokerageFactory = Composer.Instance.Single<IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveNodeConfiguration.Brokerage));
+                liveNodeConfiguration.BrokerageData = brokerageFactory.BrokerageData;
+            }
+            catch (InvalidOperationException error)
+            {
+                throw new InvalidOperationException($"{nameof(BrokerageDataDownloader)}.An error occurred while resolving brokerage data for a live job. Brokerage: {liveNodeConfiguration.Brokerage}.", error);
+            }
+
+            _brokerage = Composer.Instance.GetExportedValueByTypeName<IBrokerage>(liveNodeConfiguration.Brokerage);
+
+            _brokerage.Message += (object _, Brokerages.BrokerageMessageEvent e) =>
+            {
+                if (e.Type == Brokerages.BrokerageMessageType.Error)
+                {
+                    Logging.Log.Error(e.Message);
+                }
+                else
+                {
+                    Logging.Log.Trace(e.Message);
+                }
+            };
+
+            ((IDataQueueHandler)_brokerage).SetJob(liveNodeConfiguration);
+        }
+
+        /// <summary>
+        /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
+        /// </summary>
+        /// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
+        /// <returns>Enumerable of base data for this symbol</returns>
+        public IEnumerable<BaseData>? Get(DataDownloaderGetParameters dataDownloaderGetParameters)
+        {
+            var symbol = dataDownloaderGetParameters.Symbol;
+            var resolution = dataDownloaderGetParameters.Resolution;
+            var startUtc = dataDownloaderGetParameters.StartUtc;
+            var endUtc = dataDownloaderGetParameters.EndUtc;
+            var tickType = dataDownloaderGetParameters.TickType;
+
+            var dataType = LeanData.GetDataType(resolution, tickType);
+            var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
+            var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
+
+            var symbols = new List<Symbol> { symbol };
+            if (symbol.IsCanonical())
+            {
+                symbols = GetChainSymbols(symbol, true).ToList();
+            }
+
+            return symbols
+                .Select(symbol =>
+                {
+                    var request = new Data.HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours: exchangeHours, dataTimeZone: dataTimeZone, resolution,
+                        // let's not ask for extended market hours for hour and daily resolutions to match lean
+                        includeExtendedMarketHours: resolution != Resolution.Hour && resolution != Resolution.Daily, false, DataNormalizationMode.Raw, tickType);
+
+                    var history = _brokerage.GetHistory(request);
+
+                    if (history == null)
+                    {
+                        Logging.Log.Trace($"{nameof(BrokerageDataDownloader)}.{nameof(Get)}: Ignoring history request for unsupported symbol {symbol}");
+                    }
+
+                    return history;
+                })
+                .Where(history => history != null)
+                .SelectMany(history => history);
+        }
+
+        /// <summary>
+        /// Returns an IEnumerable of Future/Option contract symbols for the given root ticker
+        /// </summary>
+        /// <param name="symbol">The Symbol to get futures/options chain for</param>
+        /// <param name="includeExpired">Include expired contracts</param>
+        private IEnumerable<Symbol> GetChainSymbols(Symbol symbol, bool includeExpired)
+        {
+            if (_brokerage is IDataQueueUniverseProvider universeProvider)
+            {
+                return universeProvider.LookupSymbols(symbol, includeExpired);
+            }
+            else
+            {
+                throw new InvalidOperationException($"{nameof(BrokerageDataDownloader)}.{nameof(GetChainSymbols)}: The current brokerage does not support fetching canonical symbols. Please ensure your brokerage instance supports this feature.");
+            }
+        }
+
+        public void Dispose()
+        {
+            _brokerage.DisposeSafely();
+        }
+    }
+}