chore: import upstream snapshot with attribution

2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
@@ -0,0 +1,148 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Configuration;
+using QuantConnect.Util;
+using QuantConnect.Logging;
+using System.Globalization;
+using System.Threading.Tasks;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="IMapFileProvider"/> that reads from a local zip file
+    /// </summary>
+    public class LocalZipMapFileProvider : IMapFileProvider
+    {
+        private readonly DateTime? _lookupDate;
+        private Dictionary<AuxiliaryDataKey, MapFileResolver> _cache;
+        private IDataProvider _dataProvider;
+        private object _lock;
+
+        /// <summary>
+        /// The cached refresh period for the map files
+        /// </summary>
+        /// <remarks>Exposed for testing</remarks>
+        protected virtual TimeSpan CacheRefreshPeriod
+        {
+            get
+            {
+                var dueTime = Time.GetNextLiveAuxiliaryDataDueTime();
+                if (dueTime > TimeSpan.FromMinutes(10))
+                {
+                    // Clear the cache before the auxiliary due time to avoid race conditions with consumers
+                    return dueTime - TimeSpan.FromMinutes(10);
+                }
+                return dueTime;
+            }
+        }
+
+        /// <summary>
+        /// Creates a new instance of the <see cref="LocalZipMapFileProvider"/>
+        /// </summary>
+        public LocalZipMapFileProvider()
+        {
+            _lock = new object();
+            _cache = new Dictionary<AuxiliaryDataKey, MapFileResolver>();
+            
+            var lookupDateConfig = Config.Get("map-file-provider-lookup-date");
+            if (DateTime.TryParseExact(lookupDateConfig, "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var lookupDate))
+            {
+                _lookupDate = lookupDate;
+            }
+        }
+
+        /// <summary>
+        /// Initializes our MapFileProvider by supplying our dataProvider
+        /// </summary>
+        /// <param name="dataProvider">DataProvider to use</param>
+        public void Initialize(IDataProvider dataProvider)
+        {
+            if (_dataProvider != null)
+            {
+                return;
+            }
+            
+            _dataProvider = dataProvider;
+            StartExpirationTask();
+        }
+
+        /// <summary>
+        /// Gets a <see cref="MapFileResolver"/> representing all the map files for the specified market
+        /// </summary>
+        /// <param name="auxiliaryDataKey">Key used to fetch a map file resolver. Specifying market and security type</param>
+        /// <returns>A <see cref="MapFileResolver"/> containing all map files for the specified market</returns>
+        public MapFileResolver Get(AuxiliaryDataKey auxiliaryDataKey)
+        {
+            MapFileResolver result;
+            // we use a lock so that only 1 thread loads the map file resolver while the rest wait
+            // else we could have multiple threads loading the map file resolver at the same time!
+            lock (_lock)
+            {
+                if (!_cache.TryGetValue(auxiliaryDataKey, out result))
+                {
+                    _cache[auxiliaryDataKey] = result = GetMapFileResolver(auxiliaryDataKey);
+                }
+            }
+            return result;
+        }
+
+        /// <summary>
+        /// Helper method that will clear any cached factor files in a daily basis, this is useful for live trading
+        /// </summary>
+        protected virtual void StartExpirationTask()
+        {
+            lock (_lock)
+            {
+                // we clear the seeded markets so they are reloaded
+                _cache = new Dictionary<AuxiliaryDataKey, MapFileResolver>();
+            }
+            _ = Task.Delay(CacheRefreshPeriod).ContinueWith(_ => StartExpirationTask());
+        }
+
+        private MapFileResolver GetMapFileResolver(AuxiliaryDataKey auxiliaryDataKey)
+        {
+            var market = auxiliaryDataKey.Market;
+            var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
+            var todayNewYork = timestamp.Date;
+            // If a lookup date was provided, use it, otherwise use yesterday as the starting point for our search
+            var yesterdayNewYork = _lookupDate ?? todayNewYork.AddDays(-1);
+            // To prevent infinite recursion if something is wrong with the zip files, we look back a maximum of 30 days
+            var endDate = yesterdayNewYork.AddDays(_lookupDate.HasValue ? 0 : -30);
+            
+            // start the search with yesterday, today's file will be available tomorrow
+            for (var date = yesterdayNewYork; date >= endDate; date = date.AddDays(-1))
+            {
+                var zipFileName = MapFileZipHelper.GetMapFileZipFileName(market, date, auxiliaryDataKey.SecurityType);
+
+                // Fetch a stream for our zip from our data provider
+                var stream = _dataProvider.Fetch(zipFileName);
+
+                // If we didn't find a file, continue to the next date
+                if (stream == null) continue;
+
+                Log.Trace($"LocalZipMapFileProvider.Get({market}): Fetched map files for: {date.ToShortDateString()} NY ({(date - todayNewYork).Days} days ago).");
+                var result = new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market, auxiliaryDataKey.SecurityType));
+                stream.DisposeSafely();
+                return result;
+            }
+
+            throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {endDate.ToShortDateString()} for {market}");
+        }
+    }
+}