chore: import upstream snapshot with attribution

2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
@@ -0,0 +1,147 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.IO;
+using Ionic.Zip;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using System.Linq;
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Transport
+{
+    /// <summary>
+    /// Represents a stream reader capable of reading lines from disk
+    /// </summary>
+    public class LocalFileSubscriptionStreamReader : IStreamReader
+    {
+        private readonly ZipFile _zipFile;
+
+        /// <summary>
+        /// Gets whether or not this stream reader should be rate limited
+        /// </summary>
+        public bool ShouldBeRateLimited => false;
+
+        /// <summary>
+        /// Direct access to the StreamReader instance
+        /// </summary>
+        public StreamReader StreamReader { get; private set; }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LocalFileSubscriptionStreamReader"/> class.
+        /// </summary>
+        /// <param name="dataCacheProvider">The <see cref="IDataCacheProvider"/> used to retrieve a stream of data</param>
+        /// <param name="source">The local file to be read</param>
+        /// <param name="entryName">Specifies the zip entry to be opened. Leave null if not applicable,
+        /// or to open the first zip entry found regardless of name</param>
+        public LocalFileSubscriptionStreamReader(IDataCacheProvider dataCacheProvider, string source, string entryName = null)
+        {
+            var stream = dataCacheProvider.Fetch(source);
+
+            if (stream != null)
+            {
+                StreamReader = new StreamReader(stream);
+            }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LocalFileSubscriptionStreamReader"/> class.
+        /// </summary>
+        /// <param name="dataCacheProvider">The <see cref="IDataCacheProvider"/> used to retrieve a stream of data</param>
+        /// <param name="source">The local file to be read</param>
+        /// <param name="startingPosition">The position in the stream from which to start reading</param>
+        public LocalFileSubscriptionStreamReader(IDataCacheProvider dataCacheProvider, string source, long startingPosition)
+        {
+            var stream = dataCacheProvider.Fetch(source);
+
+            if (stream != null)
+            {
+                StreamReader = new StreamReader(stream);
+
+                if (startingPosition != 0)
+                {
+                    StreamReader.BaseStream.Seek(startingPosition, SeekOrigin.Begin);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LocalFileSubscriptionStreamReader"/> class.
+        /// </summary>
+        /// <param name="zipFile">The local zip archive to be read</param>
+        /// <param name="entryName">Specifies the zip entry to be opened. Leave null if not applicable,
+        /// or to open the first zip entry found regardless of name</param>
+        public LocalFileSubscriptionStreamReader(ZipFile zipFile, string entryName = null)
+        {
+            _zipFile = zipFile;
+            var entry = _zipFile.Entries.FirstOrDefault(x => entryName == null || string.Compare(x.FileName, entryName, StringComparison.OrdinalIgnoreCase) == 0);
+            if (entry != null)
+            {
+                var stream = new MemoryStream();
+                entry.OpenReader().CopyTo(stream);
+                stream.Position = 0;
+                StreamReader = new StreamReader(stream);
+            }
+        }
+
+        /// <summary>
+        /// Returns the list of zip entries if local file stream reader is reading zip archive
+        /// </summary>
+        public IEnumerable<string> EntryFileNames
+        {
+            get
+            {
+                return _zipFile != null ? _zipFile.Entries.Select(x => x.FileName).ToList() : Enumerable.Empty<string>();
+            }
+        }
+
+        /// <summary>
+        /// Gets <see cref="SubscriptionTransportMedium.LocalFile"/>
+        /// </summary>
+        public SubscriptionTransportMedium TransportMedium
+        {
+            get { return SubscriptionTransportMedium.LocalFile; }
+        }
+
+        /// <summary>
+        /// Gets whether or not there's more data to be read in the stream
+        /// </summary>
+        public bool EndOfStream
+        {
+            get { return StreamReader == null || StreamReader.EndOfStream; }
+        }
+
+        /// <summary>
+        /// Gets the next line/batch of content from the stream
+        /// </summary>
+        public string ReadLine()
+        {
+            return StreamReader.ReadLine();
+        }
+
+        /// <summary>
+        /// Disposes of the stream
+        /// </summary>
+        public void Dispose()
+        {
+            if (StreamReader != null)
+            {
+                StreamReader.Dispose();
+                StreamReader = null;
+            }
+        }
+    }
+}
@@ -0,0 +1,120 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.IO;
+using Ionic.Zip;
+using QuantConnect.Interfaces;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Transport
+{
+    /// <summary>
+    /// Represents a stream reader capable of reading lines from the object store
+    /// </summary>
+    public class ObjectStoreSubscriptionStreamReader : IStreamReader
+    {
+        private IObjectStore _objectStore;
+        private string _key;
+        private StreamReader _streamReader;
+
+        /// <summary>
+        /// Gets whether or not this stream reader should be rate limited
+        /// </summary>
+        public bool ShouldBeRateLimited => false;
+
+        /// <summary>
+        /// Direct access to the StreamReader instance
+        /// </summary>
+        public StreamReader StreamReader
+        {
+            get
+            {
+                if (_streamReader == null && !string.IsNullOrEmpty(_key) && _objectStore.ContainsKey(_key))
+                {
+                    var data = _objectStore.ReadBytes(_key);
+                    var stream = new MemoryStream(data);
+
+                    if (_key.EndsWith(".zip", StringComparison.InvariantCulture))
+                    {
+                        using var zipFile = ZipFile.Read(stream);
+                        // we only support single file zip files for now
+                        var zipEntry = zipFile[0];
+                        var tempStream = new MemoryStream();
+                        zipEntry.Extract(tempStream);
+                        tempStream.Position = 0;
+                        _streamReader = new StreamReader(tempStream);
+
+                        stream.DisposeSafely();
+                    }
+                    else
+                    {
+                        _streamReader = new StreamReader(stream);
+                    }
+                }
+
+                return _streamReader;
+            }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ObjectStoreSubscriptionStreamReader"/> class.
+        /// </summary>
+        /// <param name="objectStore">The <see cref="IObjectStore"/> used to retrieve a stream of data</param>
+        /// <param name="key">The object store key the data should be fetched from</param>
+        public ObjectStoreSubscriptionStreamReader(IObjectStore objectStore, string key)
+        {
+            _objectStore = objectStore;
+            _key = key;
+        }
+
+        /// <summary>
+        /// Gets <see cref="SubscriptionTransportMedium.LocalFile"/>
+        /// </summary>
+        public SubscriptionTransportMedium TransportMedium
+        {
+            get { return SubscriptionTransportMedium.ObjectStore; }
+        }
+
+        /// <summary>
+        /// Gets whether or not there's more data to be read in the stream
+        /// </summary>
+        public bool EndOfStream
+        {
+            get { return StreamReader == null || StreamReader.EndOfStream; }
+        }
+
+        /// <summary>
+        /// Gets the next line/batch of content from the stream
+        /// </summary>
+        public string ReadLine()
+        {
+            return StreamReader.ReadLine();
+        }
+
+        /// <summary>
+        /// Disposes of the stream
+        /// </summary>
+        public void Dispose()
+        {
+            if (_streamReader != null)
+            {
+                _streamReader.Dispose();
+                _streamReader = null;
+            }
+        }
+    }
+}
@@ -0,0 +1,209 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using QuantConnect.Interfaces;
+using QuantConnect.Logging;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Transport
+{
+    /// <summary>
+    /// Represents a stream reader capabable of downloading a remote file and then
+    /// reading it from disk
+    /// </summary>
+    public class RemoteFileSubscriptionStreamReader : IStreamReader
+    {
+        private readonly IStreamReader _streamReader;
+        private static IDownloadProvider _downloader;
+        // lock for multi thread scenarios where we are sharing the same cached file
+        private static readonly object _fileSystemLock = new object();
+
+        /// <summary>
+        /// Gets whether or not this stream reader should be rate limited
+        /// </summary>
+        public bool ShouldBeRateLimited => false;
+
+        /// <summary>
+        /// Direct access to the StreamReader instance
+        /// </summary>
+        public StreamReader StreamReader => _streamReader.StreamReader;
+
+        /// <summary>
+        /// The local file name of the downloaded file
+        /// </summary>
+        public string LocalFileName { get; }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RemoteFileSubscriptionStreamReader"/> class.
+        /// </summary>
+        /// <param name="dataCacheProvider">The <see cref="IDataCacheProvider"/> used to retrieve a stream of data</param>
+        /// <param name="source">The remote url to be downloaded via web client</param>
+        /// <param name="downloadDirectory">The local directory and destination of the download</param>
+        /// <param name="headers">Defines header values to add to the request</param>
+        public RemoteFileSubscriptionStreamReader(IDataCacheProvider dataCacheProvider, string source, string downloadDirectory, IEnumerable<KeyValuePair<string, string>> headers)
+        {
+            // don't use cache if data is ephemeral
+            // will be false for live history requests and live subscriptions
+            var useCache = !dataCacheProvider.IsDataEphemeral;
+
+            // create a hash for a new filename
+            string baseFileName = string.Empty;
+            string extension = string.Empty;
+            string entryName = string.Empty;
+            try
+            {
+                var uri = new Uri(source);
+                baseFileName = uri.OriginalString;
+                if (!string.IsNullOrEmpty(uri.Fragment))
+                {
+                    baseFileName = baseFileName.Replace(uri.Fragment, "", StringComparison.InvariantCulture);
+                }
+                extension = uri.AbsolutePath.GetExtension();
+                entryName = uri.Fragment;
+            }
+            catch
+            {
+                LeanData.ParseKey(source, out baseFileName, out entryName);
+                extension = Path.GetExtension(baseFileName);
+            }
+
+            var cacheFileName = (useCache ? baseFileName.ToMD5() : Guid.NewGuid().ToString()) + extension;
+            LocalFileName = Path.Combine(downloadDirectory, cacheFileName);
+
+            byte[] bytes = null;
+            if (useCache)
+            {
+                lock (_fileSystemLock)
+                {
+                    if (!File.Exists(LocalFileName))
+                    {
+                        bytes = DownloadBytesWithRetry(source, headers);
+                    }
+                }
+            }
+            else
+            {
+                bytes = DownloadBytesWithRetry(source, headers);
+            }
+
+            // Only persist a non-empty download. The remote source can intermittently answer with an empty body
+            // (e.g. a transient HTTP 200 with no content); writing/caching that empty response would make the
+            // whole subscription silently yield no data, and with caching enabled the empty file would be reused.
+            if (bytes != null && bytes.Length > 0)
+            {
+                File.WriteAllBytes(LocalFileName, bytes);
+
+                // Send the file to the dataCacheProvider so it is available when the streamReader asks for it
+                dataCacheProvider.Store(LocalFileName, bytes);
+            }
+
+            // now we can just use the local file reader.
+            // add the entry name to the local file name so the correct entry is read
+            var fileNameWithEntry = LocalFileName;
+            if (!string.IsNullOrEmpty(entryName))
+            {
+                fileNameWithEntry += entryName;
+            }
+            _streamReader = new LocalFileSubscriptionStreamReader(dataCacheProvider, fileNameWithEntry);
+        }
+
+        /// <summary>
+        /// Gets <see cref="SubscriptionTransportMedium.RemoteFile"/>
+        /// </summary>
+        public SubscriptionTransportMedium TransportMedium
+        {
+            get { return SubscriptionTransportMedium.RemoteFile; }
+        }
+
+        /// <summary>
+        /// Gets whether or not there's more data to be read in the stream
+        /// </summary>
+        public bool EndOfStream
+        {
+            get { return _streamReader.EndOfStream; }
+        }
+
+        /// <summary>
+        /// Gets the next line/batch of content from the stream
+        /// </summary>
+        public string ReadLine()
+        {
+            return _streamReader.ReadLine();
+        }
+
+        /// <summary>
+        /// Disposes of the stream
+        /// </summary>
+        public void Dispose()
+        {
+            _streamReader.Dispose();
+        }
+
+        /// <summary>
+        /// Save reference to the download system.
+        /// </summary>
+        /// <param name="downloader">Downloader provider for the remote file fetching.</param>
+        public static void SetDownloadProvider(IDownloadProvider downloader)
+        {
+            _downloader = downloader;
+        }
+
+        /// <summary>
+        /// Downloads the given source, retrying a few times on a transient failure. The remote endpoint can
+        /// intermittently throw or answer with an empty body; a bounded retry lets a single hiccup recover
+        /// instead of failing the whole subscription with no data.
+        /// </summary>
+        private static byte[] DownloadBytesWithRetry(string source, IEnumerable<KeyValuePair<string, string>> headers)
+        {
+            const int maxAttempts = 3;
+            for (var attempt = 1; attempt <= maxAttempts; attempt++)
+            {
+                try
+                {
+                    var bytes = _downloader.DownloadBytes(source, headers, null, null);
+                    if (bytes != null && bytes.Length > 0)
+                    {
+                        return bytes;
+                    }
+
+                    // a successful but empty response is transient for these sources; retry before giving up
+                    Log.Trace($"RemoteFileSubscriptionStreamReader.DownloadBytesWithRetry(): empty response for {source} " +
+                        $"(attempt {attempt}/{maxAttempts})");
+                }
+                catch (Exception exception)
+                {
+                    if (attempt == maxAttempts)
+                    {
+                        throw;
+                    }
+                    Log.Trace($"RemoteFileSubscriptionStreamReader.DownloadBytesWithRetry(): failed to download {source} " +
+                        $"(attempt {attempt}/{maxAttempts}): {exception.Message}");
+                }
+
+                if (attempt < maxAttempts)
+                {
+                    Thread.Sleep(2000 + 1000 * attempt);
+                }
+            }
+
+            return null;
+        }
+    }
+}
@@ -0,0 +1,110 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Net.Http;
+using QuantConnect.Interfaces;
+using QuantConnect.Logging;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Transport
+{
+    /// <summary>
+    /// Represents a stream reader capable of polling a rest client
+    /// </summary>
+    public class RestSubscriptionStreamReader : IStreamReader
+    {
+        private static readonly HttpClient _client = new HttpClient();
+
+        private readonly string _baseUrl;
+        private readonly Dictionary<string, string> _headers;
+        private readonly bool _isLiveMode;
+        private bool _delivered;
+
+        /// <summary>
+        /// Gets whether or not this stream reader should be rate limited
+        /// </summary>
+        public bool ShouldBeRateLimited => _isLiveMode;
+
+        /// <summary>
+        /// Direct access to the StreamReader instance
+        /// </summary>
+        public StreamReader StreamReader => null;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RestSubscriptionStreamReader"/> class.
+        /// </summary>
+        /// <param name="source">The source url to poll with a GET</param>
+        /// <param name="headers">Defines header values to add to the request</param>
+        /// <param name="isLiveMode">True for live mode, false otherwise</param>
+        public RestSubscriptionStreamReader(string source, IEnumerable<KeyValuePair<string, string>> headers, bool isLiveMode)
+        {
+            _baseUrl = source;
+            if (headers != null)
+            {
+                _headers = new Dictionary<string, string>(headers);
+            }
+            _isLiveMode = isLiveMode;
+            _delivered = false;
+        }
+
+        /// <summary>
+        /// Gets <see cref="SubscriptionTransportMedium.Rest"/>
+        /// </summary>
+        public SubscriptionTransportMedium TransportMedium
+        {
+            get { return SubscriptionTransportMedium.Rest; }
+        }
+
+        /// <summary>
+        /// Gets whether or not there's more data to be read in the stream
+        /// </summary>
+        public bool EndOfStream
+        {
+            get { return !_isLiveMode && _delivered; }
+        }
+
+        /// <summary>
+        /// Gets the next line/batch of content from the stream
+        /// </summary>
+        public string ReadLine()
+        {
+            try
+            {
+                if (_client.TryDownloadData(_baseUrl, out string data, out _, _headers))
+                {
+                    _delivered = true;
+                    return data;
+                }
+            }
+            catch (Exception err)
+            {
+                Log.Error(err);
+            }
+
+            return string.Empty;
+        }
+
+        /// <summary>
+        /// This stream reader doesn't require disposal
+        /// </summary>
+        public void Dispose()
+        {
+            _client.Dispose();
+        }
+    }
+}