chore: import upstream snapshot with attribution

2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
@@ -0,0 +1,80 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*/
+
+using System.IO;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using QuantConnect.Configuration;
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// A data provider that will check the processed data folder first
+    /// </summary>
+    public class ProcessedDataProvider : IDataProvider, IDisposable
+    {
+        private readonly DefaultDataProvider _defaultDataProvider;
+        private readonly string _processedDataDirectory;
+
+        /// <summary>
+        /// Ignored
+        /// </summary>
+        public event EventHandler<DataProviderNewDataRequestEventArgs> NewDataRequest;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public ProcessedDataProvider()
+        {
+            _defaultDataProvider = new();
+            _processedDataDirectory = Config.Get("processed-data-directory") ?? string.Empty;
+            Log.Trace($"ProcessedDataProvider(): processed data directory to use {_processedDataDirectory}, exists: {Directory.Exists(_processedDataDirectory)}");
+        }
+
+        /// <summary>
+        /// Retrieves data from disc to be used in an algorithm
+        /// </summary>
+        /// <param name="key">A string representing where the data is stored</param>
+        /// <returns>A <see cref="Stream"/> of the data requested</returns>
+        public Stream Fetch(string key)
+        {
+            Stream result = null;
+
+            // we will try the processed data folder first
+            if (_processedDataDirectory.Length != 0 && key.StartsWith(Globals.DataFolder, StringComparison.OrdinalIgnoreCase))
+            {
+                result = _defaultDataProvider.Fetch(Path.Combine(_processedDataDirectory, key.Remove(0, Globals.DataFolder.Length).TrimStart('/', '\\')));
+                if (result != null)
+                {
+                    Log.Trace($"ProcessedDataProvider.Fetch({key}): fetched from processed data directory");
+                }
+            }
+
+            // fall back to existing data folder path
+            return result ?? _defaultDataProvider.Fetch(key);
+        }
+
+        /// <summary>
+        /// Disposes of resources
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Disposes of the internal data provider
+        /// </summary>
+        protected virtual void Dispose(bool disposing)
+        {
+            if (disposing)
+            {
+                _defaultDataProvider.Dispose();
+            }
+        }
+    }
+}