chore: import upstream snapshot with attribution

Engine/DataFeeds/AggregationManager.cs

@@ -0,0 +1,184 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Data;
+using QuantConnect.Data.Consolidators;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using QuantConnect.Logging;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Aggregates ticks and bars based on given subscriptions.
+    /// Current implementation is based on <see cref="IDataConsolidator"/> that consolidates ticks and put them into enumerator.
+    /// </summary>
+    public class AggregationManager : IDataAggregator
+    {
+        private readonly ConcurrentDictionary<SecurityIdentifier, List<KeyValuePair<SubscriptionDataConfig, ScannableEnumerator<BaseData>>>> _enumerators
+            = new ConcurrentDictionary<SecurityIdentifier, List<KeyValuePair<SubscriptionDataConfig, ScannableEnumerator<BaseData>>>>();
+        private bool _dailyStrictEndTimeEnabled;
+
+        /// <summary>
+        /// Continuous UTC time provider
+        /// </summary>
+        protected ITimeProvider TimeProvider { get; set; } = RealTimeProvider.Instance;
+
+        /// <summary>
+        /// Initialize this instance
+        /// </summary>
+        /// <param name="parameters">The parameters dto instance</param>
+        public void Initialize(DataAggregatorInitializeParameters parameters)
+        {
+            _dailyStrictEndTimeEnabled = parameters.AlgorithmSettings.DailyPreciseEndTime;
+            Log.Trace($"AggregationManager.Initialize(): daily strict end times: {_dailyStrictEndTimeEnabled}");
+        }
+
+        /// <summary>
+        /// Add new subscription to current <see cref="IDataAggregator"/> instance
+        /// </summary>
+        /// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
+        /// <param name="newDataAvailableHandler">handler to be fired on new data available</param>
+        /// <returns>The new enumerator for this subscription request</returns>
+        public IEnumerator<BaseData> Add(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
+        {
+            var consolidator = GetConsolidator(dataConfig);
+            var isPeriodBased = (dataConfig.Type.Name == nameof(QuoteBar) ||
+                    dataConfig.Type.Name == nameof(TradeBar) ||
+                    dataConfig.Type.Name == nameof(OpenInterest)) &&
+                dataConfig.Resolution != Resolution.Tick;
+            var enumerator = new ScannableEnumerator<BaseData>(consolidator, dataConfig.ExchangeTimeZone, TimeProvider, newDataAvailableHandler, isPeriodBased);
+
+            _enumerators.AddOrUpdate(
+                dataConfig.Symbol.ID,
+                new List<KeyValuePair<SubscriptionDataConfig, ScannableEnumerator<BaseData>>> { new KeyValuePair<SubscriptionDataConfig, ScannableEnumerator<BaseData>>(dataConfig, enumerator) },
+                (k, v) => { return v.Concat(new[] { new KeyValuePair<SubscriptionDataConfig, ScannableEnumerator<BaseData>>(dataConfig, enumerator) }).ToList(); });
+
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Removes the handler with the specified identifier
+        /// </summary>
+        /// <param name="dataConfig">Subscription data configuration to be removed</param>
+        public bool Remove(SubscriptionDataConfig dataConfig)
+        {
+            List<KeyValuePair<SubscriptionDataConfig, ScannableEnumerator<BaseData>>> enumerators;
+            if (_enumerators.TryGetValue(dataConfig.Symbol.ID, out enumerators))
+            {
+                if (enumerators.Count == 1)
+                {
+                    List<KeyValuePair<SubscriptionDataConfig, ScannableEnumerator<BaseData>>> output;
+                    return _enumerators.TryRemove(dataConfig.Symbol.ID, out output);
+                }
+                else
+                {
+                    _enumerators[dataConfig.Symbol.ID] = enumerators.Where(pair => pair.Key != dataConfig).ToList();
+                    return true;
+                }
+            }
+            else
+            {
+                Log.Debug($"AggregationManager.Update(): IDataConsolidator for symbol ({dataConfig.Symbol.Value}) was not found.");
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Add new data to aggregator
+        /// </summary>
+        /// <param name="input">The new data</param>
+        public void Update(BaseData input)
+        {
+            try
+            {
+                List<KeyValuePair<SubscriptionDataConfig, ScannableEnumerator<BaseData>>> enumerators;
+                if (_enumerators.TryGetValue(input.Symbol.ID, out enumerators))
+                {
+                    for (var i = 0; i < enumerators.Count; i++)
+                    {
+                        var kvp = enumerators[i];
+
+                        // for non tick resolution subscriptions drop suspicious ticks
+                        if (kvp.Key.Resolution != Resolution.Tick)
+                        {
+                            var tick = input as Tick;
+                            if (tick != null && tick.Suspicious)
+                            {
+                                continue;
+                            }
+                        }
+
+                        kvp.Value.Update(input);
+                    }
+                }
+            }
+            catch (Exception exception)
+            {
+                Log.Error(exception);
+            }
+        }
+
+        /// <summary>
+        /// Dispose of the aggregation manager.
+        /// </summary>
+        public void Dispose() { }
+
+        /// <summary>
+        /// Gets the consolidator to aggregate data for the given config
+        /// </summary>
+        protected virtual IDataConsolidator GetConsolidator(SubscriptionDataConfig config)
+        {
+            var period = config.Resolution.ToTimeSpan();
+            if (config.Resolution == Resolution.Daily && (config.Type == typeof(QuoteBar) || config.Type == typeof(TradeBar)))
+            {
+                // in backtesting, daily resolution data does not have extended market hours even if requested, so let's respect the same behavior for live
+                // also this allows us to enable the daily strict end times if required. See 'SetStrictEndTimes'
+                return new MarketHourAwareConsolidator(_dailyStrictEndTimeEnabled, config.Resolution, typeof(Tick), config.TickType, extendedMarketHours: false);
+            }
+            if (config.Type == typeof(QuoteBar))
+            {
+                return new TickQuoteBarConsolidator(period);
+            }
+            if (config.Type == typeof(TradeBar))
+            {
+                return new TickConsolidator(period);
+            }
+            if (config.Type == typeof(OpenInterest))
+            {
+                return new OpenInterestConsolidator(period);
+            }
+            if (config.Type == typeof(Tick))
+            {
+                return FilteredIdentityDataConsolidator.ForTickType(config.TickType);
+            }
+            if (config.Type == typeof(Split))
+            {
+                return new IdentityDataConsolidator<Split>();
+            }
+            if (config.Type == typeof(Dividend))
+            {
+                return new IdentityDataConsolidator<Dividend>();
+            }
+
+            // streaming custom data subscriptions can pass right through
+            return new FilteredIdentityDataConsolidator<BaseData>(data => data.GetType() == config.Type);
+        }
+    }
+}

Engine/DataFeeds/ApiDataProvider.cs

@@ -0,0 +1,268 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using QuantConnect.Api;
+using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Configuration;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// An instance of the <see cref="IDataProvider"/> that will download and update data files as needed via QC's Api.
+    /// </summary>
+    public class ApiDataProvider : BaseDownloaderDataProvider
+    {
+        private decimal _purchaseLimit = Config.GetValue("data-purchase-limit", decimal.MaxValue); //QCC
+
+        private readonly HashSet<SecurityType> _unsupportedSecurityType;
+        private readonly DataPricesList _dataPrices;
+        private readonly IApi _api;
+        private readonly bool _subscribedToIndiaEquityMapAndFactorFiles;
+        private readonly bool _subscribedToUsaEquityMapAndFactorFiles;
+        private readonly bool _subscribedToFutureMapAndFactorFiles;
+        private volatile bool _invalidSecurityTypeLog;
+
+        /// <summary>
+        /// Initialize a new instance of the <see cref="ApiDataProvider"/>
+        /// </summary>
+        public ApiDataProvider()
+        {
+            _unsupportedSecurityType = new HashSet<SecurityType> { SecurityType.Future, SecurityType.FutureOption, SecurityType.Index, SecurityType.IndexOption };
+
+            _api = Composer.Instance.GetPart<IApi>();
+
+            // If we have no value for organization get account preferred
+            if (string.IsNullOrEmpty(Globals.OrganizationID))
+            {
+                var account = _api.ReadAccount();
+                Globals.OrganizationID = account?.OrganizationId;
+                Log.Trace($"ApiDataProvider(): Will use organization Id '{Globals.OrganizationID}'.");
+            }
+
+            // Read in data prices and organization details
+            _dataPrices = _api.ReadDataPrices(Globals.OrganizationID);
+            var organization = _api.ReadOrganization(Globals.OrganizationID);
+
+            foreach (var productItem in organization.Products.Where(x => x.Type == ProductType.Data).SelectMany(product => product.Items))
+            {
+                if (productItem.Id == 37)
+                {
+                    // Determine if the user is subscribed to Equity map and factor files (Data product Id 37)
+                    _subscribedToUsaEquityMapAndFactorFiles = true;
+                }
+                else if (productItem.Id == 137)
+                {
+                    // Determine if the user is subscribed to Future map and factor files (Data product Id 137)
+                    _subscribedToFutureMapAndFactorFiles = true;
+                }
+                else if (productItem.Id == 172)
+                {
+                    // Determine if the user is subscribed to India map and factor files (Data product Id 172)
+                    _subscribedToIndiaEquityMapAndFactorFiles = true;
+                }
+            }
+
+            // Verify user has agreed to data provider agreements
+            if (organization.DataAgreement.Signed)
+            {
+                //Log Agreement Highlights
+                Log.Trace("ApiDataProvider(): Data Terms of Use has been signed. \r\n" +
+                    $" Find full agreement at: {_dataPrices.AgreementUrl} \r\n" +
+                    "==========================================================================\r\n" +
+                    $"CLI API Access Agreement: On {organization.DataAgreement.SignedTime:d} You Agreed:\r\n" +
+                    " - Display or distribution of data obtained through CLI API Access is not permitted.  \r\n" +
+                    " - Data and Third Party Data obtained via CLI API Access can only be used for individual or internal employee's use.\r\n" +
+                    " - Data is provided in LEAN format can not be manipulated for transmission or use in other applications. \r\n" +
+                    " - QuantConnect is not liable for the quality of data received and is not responsible for trading losses. \r\n" +
+                    "==========================================================================");
+                Thread.Sleep(TimeSpan.FromSeconds(3));
+            }
+            else
+            {
+                // Log URL to go accept terms
+                throw new InvalidOperationException($"ApiDataProvider(): Must agree to terms at {_dataPrices.AgreementUrl}, before using the ApiDataProvider");
+            }
+
+            // Verify we have the balance to maintain our purchase limit, if not adjust it to meet our balance
+            var balance = organization.Credit.Balance;
+            if (balance < _purchaseLimit)
+            {
+                if (_purchaseLimit != decimal.MaxValue)
+                {
+                    Log.Error("ApiDataProvider(): Purchase limit is greater than balance." +
+                        $" Setting purchase limit to balance : {balance}");
+                }
+                _purchaseLimit = balance;
+            }
+        }
+
+        /// <summary>
+        /// Retrieves data to be used in an algorithm.
+        /// If file does not exist, an attempt is made to download them from the api
+        /// </summary>
+        /// <param name="key">File path representing where the data requested</param>
+        /// <returns>A <see cref="Stream"/> of the data requested</returns>
+        public override Stream Fetch(string key)
+        {
+            return DownloadOnce(key, s =>
+            {
+                // Verify we have enough credit to handle this
+                var pricePath = Api.Api.FormatPathForDataRequest(key);
+                var price = _dataPrices.GetPrice(pricePath);
+
+                // No price found
+                if (price == -1)
+                {
+                    throw new ArgumentException($"ApiDataProvider.Fetch(): No price found for {pricePath}");
+                }
+
+                if (_purchaseLimit < price)
+                {
+                    throw new ArgumentException($"ApiDataProvider.Fetch(): Cost {price} for {pricePath} data exceeds remaining purchase limit: {_purchaseLimit}");
+                }
+
+                if (DownloadData(key))
+                {
+                    // Update our purchase limit.
+                    _purchaseLimit -= price;
+                }
+            });
+        }
+
+        /// <summary>
+        /// Main filter to determine if this file needs to be downloaded
+        /// </summary>
+        /// <param name="filePath">File we are looking at</param>
+        /// <returns>True if should download</returns>
+        protected override bool NeedToDownload(string filePath)
+        {
+            // Ignore null
+            if (filePath == null)
+            {
+                return false;
+            }
+
+            // Some security types can't be downloaded, lets attempt to extract that information
+            if (LeanData.TryParseSecurityType(filePath, out SecurityType securityType, out var market) &&
+                _unsupportedSecurityType.Contains(securityType) &&
+                // We do support universe data for some security types (options and futures)
+                !IsUniverseData(securityType, filePath))
+            {
+                // we do support future auxiliary data (map and factor files)
+                if (securityType != SecurityType.Future || !IsAuxiliaryData(filePath))
+                {
+                    if (!_invalidSecurityTypeLog)
+                    {
+                        // let's log this once. Will still use any existing data on disk
+                        _invalidSecurityTypeLog = true;
+                        Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
+                    }
+                    return false;
+                }
+            }
+
+            if (securityType == SecurityType.Equity && filePath.Contains("fine", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains("fundamental", StringComparison.InvariantCultureIgnoreCase))
+            {
+                // Ignore fine fundamental data requests
+                return false;
+            }
+
+            // Only download if it doesn't exist or is out of date.
+            // Files are only "out of date" for non date based files (hour, daily, margins, etc.) because this data is stored all in one file
+            var shouldDownload = !File.Exists(filePath) || filePath.IsOutOfDate();
+
+            if (shouldDownload)
+            {
+                if (securityType == SecurityType.Future)
+                {
+                    if (!_subscribedToFutureMapAndFactorFiles)
+                    {
+                        throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
+                            "to download Future auxiliary data from QuantConnect. " +
+                            "Please visit https://www.quantconnect.com/datasets/quantconnect-us-futures-security-master for details.");
+                    }
+                }
+                // Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
+                else if (!_subscribedToUsaEquityMapAndFactorFiles && market.Equals(Market.USA, StringComparison.InvariantCultureIgnoreCase)
+                         && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
+                {
+                    throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
+                        "to download Equity data from QuantConnect. " +
+                        "Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
+                }
+                else if (!_subscribedToIndiaEquityMapAndFactorFiles && market.Equals(Market.India, StringComparison.InvariantCultureIgnoreCase)
+                         && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
+                {
+                    throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
+                        "to download India data from QuantConnect. " +
+                        "Please visit https://www.quantconnect.com/datasets/truedata-india-equity-security-master for details.");
+                }
+            }
+
+            return shouldDownload;
+        }
+
+        /// <summary>
+        /// Attempt to download data using the Api for and return a FileStream of that data.
+        /// </summary>
+        /// <param name="filePath">The path to store the file</param>
+        /// <returns>A FileStream of the data</returns>
+        protected virtual bool DownloadData(string filePath)
+        {
+            if (Log.DebuggingEnabled)
+            {
+                Log.Debug($"ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for {filePath}.");
+            }
+
+            if (_api.DownloadData(filePath, Globals.OrganizationID))
+            {
+                Log.Trace($"ApiDataProvider.Fetch(): Successfully retrieved data for {filePath}.");
+                return true;
+            }
+            // Failed to download; _api.DownloadData() will post error
+            return false;
+        }
+
+        /// <summary>
+        /// Helper method to determine if this filepath is auxiliary data
+        /// </summary>
+        /// <param name="filepath">The target file path</param>
+        /// <returns>True if this file is of auxiliary data</returns>
+        private static bool IsAuxiliaryData(string filepath)
+        {
+            return filepath.Contains("map_files", StringComparison.InvariantCulture)
+                || filepath.Contains("factor_files", StringComparison.InvariantCulture)
+                || filepath.Contains("fundamental", StringComparison.InvariantCulture)
+                || filepath.Contains("shortable", StringComparison.InvariantCulture);
+        }
+
+        /// <summary>
+        /// Helper method to determine if this file path if for a universe file
+        /// </summary>
+        private static bool IsUniverseData(SecurityType securityType, string filepath)
+        {
+            return (securityType.IsOption() || securityType == SecurityType.Future) &&
+                filepath.Contains("universes", StringComparison.InvariantCulture);
+        }
+    }
+}

Engine/DataFeeds/BacktestingChainProvider.cs

@@ -0,0 +1,119 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Base backtesting cache provider which will source symbols from local zip files
+    /// </summary>
+    public abstract class BacktestingChainProvider
+    {
+        /// <summary>
+        /// The map file provider instance to use
+        /// </summary>
+        protected IMapFileProvider MapFileProvider { get; private set; }
+
+        /// <summary>
+        /// The history provider instance to use
+        /// </summary>
+        protected IHistoryProvider HistoryProvider { get; private set; }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="BacktestingChainProvider"/> class
+        /// </summary>
+        protected BacktestingChainProvider()
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="BacktestingChainProvider"/> class
+        /// </summary>
+        /// <param name="parameters">The initialization parameters</param>
+        // TODO: This should be in the chain provider interfaces.
+        // They might be even be unified in a single interface (futures and options chains providers)
+        public void Initialize(ChainProviderInitializeParameters parameters)
+        {
+            HistoryProvider = parameters.HistoryProvider;
+            MapFileProvider = parameters.MapFileProvider;
+        }
+
+        /// <summary>
+        /// Get the contract symbols associated with the given canonical symbol and date
+        /// </summary>
+        /// <param name="canonicalSymbol">The canonical symbol</param>
+        /// <param name="date">The date to search for</param>
+        protected IEnumerable<Symbol> GetSymbols(Symbol canonicalSymbol, DateTime date)
+        {
+            var marketHoursDataBase = MarketHoursDatabase.FromDataFolder();
+            var universeType = canonicalSymbol.SecurityType.IsOption() ? typeof(OptionUniverse) : typeof(FutureUniverse);
+            // Use this GetEntry extension method since it's data type dependent, so we get the correct entry for the option universe
+            var marketHoursEntry = marketHoursDataBase.GetEntry(canonicalSymbol, new[] { universeType });
+
+            // We will add a safety measure in case the universe file for the current time is not available:
+            // we will use the latest available universe file within the last 3 trading dates.
+            // This is useful in cases like live trading when the algorithm is deployed at a time of day when
+            // the universe file is not available yet.
+            var history = (List<Slice>)null;
+            var periods = 1;
+            while ((history == null || history.Count == 0) && periods <= 3)
+            {
+                var startDate = Time.GetStartTimeForTradeBars(marketHoursEntry.ExchangeHours, date, Time.OneDay, periods++,
+                    extendedMarketHours: false, marketHoursEntry.DataTimeZone);
+                var request = new HistoryRequest(
+                    startDate.ConvertToUtc(marketHoursEntry.ExchangeHours.TimeZone),
+                    date.ConvertToUtc(marketHoursEntry.ExchangeHours.TimeZone),
+                    universeType,
+                    canonicalSymbol,
+                    Resolution.Daily,
+                    marketHoursEntry.ExchangeHours,
+                    marketHoursEntry.DataTimeZone,
+                    null,
+                    false,
+                    false,
+                    DataNormalizationMode.Raw,
+                    TickType.Quote);
+                history = HistoryProvider.GetHistory([request], marketHoursEntry.DataTimeZone)?.ToList();
+            }
+
+            var symbols = history == null || history.Count == 0
+                ? Enumerable.Empty<Symbol>()
+                : history.Take(1).GetUniverseData().SelectMany(x => x.Values.Single()).Select(x => x.Symbol);
+
+            if (canonicalSymbol.SecurityType.IsOption())
+            {
+                symbols = symbols.Where(symbol => symbol.SecurityType.IsOption());
+            }
+
+            return symbols.Where(symbol => symbol.ID.Date >= date.Date);
+        }
+
+        /// <summary>
+        /// Helper method to determine if a contract is expired for the requested date
+        /// </summary>
+        protected static bool IsContractExpired(Symbol symbol, DateTime date)
+        {
+            return symbol.ID.Date.Date < date.Date;
+        }
+    }
+}

Engine/DataFeeds/BacktestingFutureChainProvider.cs

@@ -0,0 +1,60 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// An implementation of <see cref="IFutureChainProvider"/> that reads the list of contracts from open interest zip data files
+    /// </summary>
+    public class BacktestingFutureChainProvider : BacktestingChainProvider, IFutureChainProvider
+    {
+        /// <summary>
+        /// Gets the list of future contracts for a given underlying symbol
+        /// </summary>
+        /// <param name="symbol">The underlying symbol</param>
+        /// <param name="date">The date for which to request the future chain (only used in backtesting)</param>
+        /// <returns>The list of future contracts</returns>
+        public virtual IEnumerable<Symbol> GetFutureContractList(Symbol symbol, DateTime date)
+        {
+            return GetSymbols(GetSymbol(symbol), date);
+        }
+
+        /// <summary>
+        /// Helper method to get the symbol to use
+        /// </summary>
+        protected static Symbol GetSymbol(Symbol symbol)
+        {
+            if (symbol.SecurityType != SecurityType.Future)
+            {
+                if (symbol.SecurityType == SecurityType.FutureOption && symbol.Underlying != null)
+                {
+                    // be user friendly and take the underlying
+                    symbol = symbol.Underlying;
+                }
+                else
+                {
+                    throw new NotSupportedException($"BacktestingFutureChainProvider.GetFutureContractList():" +
+                        $" {nameof(SecurityType.Future)} or {nameof(SecurityType.FutureOption)} is expected but was {symbol.SecurityType}");
+                }
+            }
+
+            return symbol.Canonical;
+        }
+    }
+}

Engine/DataFeeds/BacktestingOptionChainProvider.cs

@@ -0,0 +1,92 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// An implementation of <see cref="IOptionChainProvider"/> that reads the list of contracts from open interest zip data files
+    /// </summary>
+    public class BacktestingOptionChainProvider : BacktestingChainProvider, IOptionChainProvider
+    {
+        /// <summary>
+        /// Gets the list of option contracts for a given underlying symbol
+        /// </summary>
+        /// <param name="symbol">The option or the underlying symbol to get the option chain for.
+        /// Providing the option allows targeting an option ticker different than the default e.g. SPXW</param>
+        /// <param name="date">The date for which to request the option chain (only used in backtesting)</param>
+        /// <returns>The list of option contracts</returns>
+        public virtual IEnumerable<Symbol> GetOptionContractList(Symbol symbol, DateTime date)
+        {
+            Symbol canonicalSymbol;
+            if (!symbol.SecurityType.HasOptions())
+            {
+                // we got an option
+                if (symbol.SecurityType.IsOption() && symbol.Underlying != null)
+                {
+                    canonicalSymbol = GetCanonical(symbol, date);
+                }
+                else
+                {
+                    throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): " +
+                        $"{nameof(SecurityType.Equity)}, {nameof(SecurityType.Future)}, or {nameof(SecurityType.Index)} is expected but was {symbol.SecurityType}");
+                }
+            }
+            else
+            {
+                // we got the underlying
+                var mappedUnderlyingSymbol = MapUnderlyingSymbol(symbol, date);
+                canonicalSymbol = Symbol.CreateCanonicalOption(mappedUnderlyingSymbol);
+            }
+
+            return GetSymbols(canonicalSymbol, date);
+        }
+
+        private Symbol GetCanonical(Symbol optionSymbol, DateTime date)
+        {
+            // Resolve any mapping before requesting option contract list for equities
+            // Needs to be done in order for the data file key to be accurate
+            if (optionSymbol.Underlying.RequiresMapping())
+            {
+                var mappedUnderlyingSymbol = MapUnderlyingSymbol(optionSymbol.Underlying, date);
+
+                return Symbol.CreateCanonicalOption(mappedUnderlyingSymbol);
+            }
+            else
+            {
+                return optionSymbol.Canonical;
+            }
+        }
+
+        private Symbol MapUnderlyingSymbol(Symbol underlying, DateTime date)
+        {
+            if (underlying.RequiresMapping())
+            {
+                var mapFileResolver = MapFileProvider.Get(AuxiliaryDataKey.Create(underlying));
+                var mapFile = mapFileResolver.ResolveMapFile(underlying);
+                var ticker = mapFile.GetMappedSymbol(date, underlying.Value);
+                return underlying.UpdateMappedSymbol(ticker);
+            }
+            else
+            {
+                return underlying;
+            }
+        }
+    }
+}

Engine/DataFeeds/BaseDataCollectionAggregatorReader.cs

@@ -0,0 +1,95 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Data source reader that will aggregate data points into a base data collection
+    /// </summary>
+    public class BaseDataCollectionAggregatorReader : TextSubscriptionDataSourceReader
+    {
+        private readonly Type _collectionType;
+        private BaseDataCollection _collection;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="TextSubscriptionDataSourceReader"/> class
+        /// </summary>
+        /// <param name="dataCacheProvider">This provider caches files if needed</param>
+        /// <param name="config">The subscription's configuration</param>
+        /// <param name="date">The date this factory was produced to read data for</param>
+        /// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
+        /// <param name="objectStore">The object storage for data persistence</param>
+        public BaseDataCollectionAggregatorReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date,
+            bool isLiveMode, IObjectStore objectStore)
+            : base(dataCacheProvider, config, date, isLiveMode, objectStore)
+        {
+            // if the type is not a BaseDataCollection, we'll default to BaseDataCollection.
+            // e.g. custom Python dynamic folding collections need to be aggregated into a BaseDataCollection,
+            // but they implement PythonData, so casting an instance of PythonData to BaseDataCollection will fail.
+            _collectionType = config.Type.IsAssignableTo(typeof(BaseDataCollection)) ? config.Type : typeof(BaseDataCollection);
+        }
+
+        /// <summary>
+        /// Reads the specified <paramref name="source"/>
+        /// </summary>
+        /// <param name="source">The source to be read</param>
+        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
+        public override IEnumerable<BaseData> Read(SubscriptionDataSource source)
+        {
+            foreach (var point in base.Read(source))
+            {
+                if (point is BaseDataCollection collection && !collection.Data.IsNullOrEmpty())
+                {
+                    // if underlying already is returning an aggregated collection let it through as is
+                    yield return point;
+                }
+                else
+                {
+                    if (_collection != null && _collection.EndTime != point.EndTime)
+                    {
+                        // when we get a new time we flush current collection instance, if any
+                        yield return _collection;
+                        _collection = null;
+                    }
+
+                    if (_collection == null)
+                    {
+                        _collection = (BaseDataCollection)Activator.CreateInstance(_collectionType);
+                        _collection.Time = point.Time;
+                        _collection.Symbol = Config.Symbol;
+                        _collection.EndTime = point.EndTime;
+                    }
+                    // aggregate the data points
+                    _collection.Add(point);
+                }
+            }
+
+            // underlying reader ended, flush current collection instance if any
+            if (_collection != null)
+            {
+                yield return _collection;
+                _collection = null;
+            }
+        }
+    }
+}

Engine/DataFeeds/BaseDataExchange.cs

@@ -0,0 +1,296 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Threading;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Provides a means of distributing output from enumerators from a dedicated separate thread
+    /// </summary>
+    public class BaseDataExchange
+    {
+        private Thread _thread;
+        private uint _sleepInterval = 1;
+        private Func<Exception, bool> _isFatalError;
+        private readonly CancellationTokenSource _cancellationTokenSource;
+
+        private readonly string _name;
+        private ManualResetEventSlim _manualResetEventSlim;
+        private ConcurrentDictionary<Symbol, EnumeratorHandler> _enumerators;
+
+        /// <summary>
+        /// Gets or sets how long this thread will sleep when no data is available
+        /// </summary>
+        public uint SleepInterval
+        {
+            get => _sleepInterval;
+            set
+            {
+                if (value == 0)
+                {
+                    throw new ArgumentException("Sleep interval should be bigger than 0");
+                }
+                _sleepInterval = value;
+            }
+        }
+
+        /// <summary>
+        /// Gets a name for this exchange
+        /// </summary>
+        public string Name
+        {
+            get { return _name; }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="BaseDataExchange"/>
+        /// </summary>
+        /// <param name="name">A name for this exchange</param>
+        public BaseDataExchange(string name)
+        {
+            _name = name;
+            _isFatalError = x => false;
+            _cancellationTokenSource = new CancellationTokenSource();
+            _manualResetEventSlim = new ManualResetEventSlim(false);
+            _enumerators = new ConcurrentDictionary<Symbol, EnumeratorHandler>();
+        }
+
+        /// <summary>
+        /// Adds the enumerator to this exchange. If it has already been added
+        /// then it will remain registered in the exchange only once
+        /// </summary>
+        /// <param name="handler">The handler to use when this symbol's data is encountered</param>
+        public void AddEnumerator(EnumeratorHandler handler)
+        {
+            _enumerators[handler.Symbol] = handler;
+            _manualResetEventSlim.Set();
+        }
+
+        /// <summary>
+        /// Adds the enumerator to this exchange. If it has already been added
+        /// then it will remain registered in the exchange only once
+        /// </summary>
+        /// <param name="symbol">A unique symbol used to identify this enumerator</param>
+        /// <param name="enumerator">The enumerator to be added</param>
+        /// <param name="shouldMoveNext">Function used to determine if move next should be called on this
+        /// enumerator, defaults to always returning true</param>
+        /// <param name="enumeratorFinished">Delegate called when the enumerator move next returns false</param>
+        /// <param name="handleData">Handler for data if HandlesData=true</param>
+        public void AddEnumerator(Symbol symbol, IEnumerator<BaseData> enumerator, Func<bool> shouldMoveNext = null, Action<EnumeratorHandler> enumeratorFinished = null, Action<BaseData> handleData = null)
+        {
+            var enumeratorHandler = new EnumeratorHandler(symbol, enumerator, shouldMoveNext, handleData);
+            if (enumeratorFinished != null)
+            {
+                enumeratorHandler.EnumeratorFinished += (sender, args) => enumeratorFinished(args);
+            }
+            AddEnumerator(enumeratorHandler);
+        }
+
+        /// <summary>
+        /// Sets the specified function as the error handler. This function
+        /// returns true if it is a fatal error and queue consumption should
+        /// cease.
+        /// </summary>
+        /// <param name="isFatalError">The error handling function to use when an
+        /// error is encountered during queue consumption. Returns true if queue
+        /// consumption should be stopped, returns false if queue consumption should
+        /// continue</param>
+        public void SetErrorHandler(Func<Exception, bool> isFatalError)
+        {
+            // default to false;
+            _isFatalError = isFatalError ?? (x => false);
+        }
+
+        /// <summary>
+        /// Removes and returns enumerator handler with the specified symbol.
+        /// The removed handler is returned, null if not found
+        /// </summary>
+        public EnumeratorHandler RemoveEnumerator(Symbol symbol)
+        {
+            EnumeratorHandler handler;
+            if (_enumerators.TryRemove(symbol, out handler))
+            {
+                handler.OnEnumeratorFinished();
+                handler.Enumerator.Dispose();
+            }
+            return handler;
+        }
+
+        /// <summary>
+        /// Begins consumption of the wrapped <see cref="IDataQueueHandler"/> on
+        /// a separate thread
+        /// </summary>
+        public void Start()
+        {
+            var manualEvent = new ManualResetEventSlim(false);
+            _thread = new Thread(() =>
+            {
+                manualEvent.Set();
+                Log.Trace($"BaseDataExchange({Name}) Starting...");
+                ConsumeEnumerators();
+            }) {  IsBackground = true, Name = Name };
+            _thread.Start();
+
+            manualEvent.Wait();
+            manualEvent.DisposeSafely();
+        }
+
+        /// <summary>
+        /// Ends consumption of the wrapped <see cref="IDataQueueHandler"/>
+        /// </summary>
+        public void Stop()
+        {
+            _thread.StopSafely(TimeSpan.FromSeconds(5), _cancellationTokenSource);
+        }
+
+        /// <summary> Entry point for queue consumption </summary>
+        /// <remarks> This function only returns after <see cref="Stop"/> is called or the token is cancelled</remarks>
+        private void ConsumeEnumerators()
+        {
+            while (!_cancellationTokenSource.Token.IsCancellationRequested)
+            {
+                try
+                {
+                    // call move next each enumerator and invoke the appropriate handlers
+                    _manualResetEventSlim.Reset();
+                    var handled = false;
+                    foreach (var kvp in _enumerators)
+                    {
+                        if (_cancellationTokenSource.Token.IsCancellationRequested)
+                        {
+                            Log.Trace($"BaseDataExchange({Name}).ConsumeQueue(): Exiting...");
+                            return;
+                        }
+                        var enumeratorHandler = kvp.Value;
+                        var enumerator = enumeratorHandler.Enumerator;
+
+                        // check to see if we should advance this enumerator
+                        if (!enumeratorHandler.ShouldMoveNext()) continue;
+
+                        if (!enumerator.MoveNext())
+                        {
+                            enumeratorHandler.OnEnumeratorFinished();
+                            enumeratorHandler.Enumerator.Dispose();
+                            _enumerators.TryRemove(enumeratorHandler.Symbol, out enumeratorHandler);
+                            continue;
+                        }
+
+                        if (enumerator.Current == null) continue;
+
+                        handled = true;
+                        enumeratorHandler.HandleData(enumerator.Current);
+                    }
+
+                    if (!handled)
+                    {
+                        // if we didn't handle anything on this past iteration, take a nap
+                        // wait until we timeout, we are cancelled or there is a new enumerator added
+                        _manualResetEventSlim.Wait(Time.GetSecondUnevenWait((int)_sleepInterval), _cancellationTokenSource.Token);
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    // thrown by the event watcher
+                }
+                catch (Exception err)
+                {
+                    Log.Error(err);
+                    if (_isFatalError(err))
+                    {
+                        Log.Trace($"BaseDataExchange({Name}).ConsumeQueue(): Fatal error encountered. Exiting...");
+                        return;
+                    }
+                }
+            }
+
+            Log.Trace($"BaseDataExchange({Name}).ConsumeQueue(): Exiting...");
+        }
+
+        /// <summary>
+        /// Handler used to manage a single enumerator's move next/end of stream behavior
+        /// </summary>
+        public class EnumeratorHandler
+        {
+            private readonly Func<bool> _shouldMoveNext;
+            private readonly Action<BaseData> _handleData;
+
+            /// <summary>
+            /// Event fired when MoveNext returns false
+            /// </summary>
+            public event EventHandler<EnumeratorHandler> EnumeratorFinished;
+
+            /// <summary>
+            /// A unique symbol used to identify this enumerator
+            /// </summary>
+            public Symbol Symbol { get; init; }
+
+            /// <summary>
+            /// The enumerator this handler handles
+            /// </summary>
+            public IEnumerator<BaseData> Enumerator { get; init; }
+
+            /// <summary>
+            /// Initializes a new instance of the <see cref="EnumeratorHandler"/> class
+            /// </summary>
+            /// <param name="symbol">The symbol to identify this enumerator</param>
+            /// <param name="enumerator">The enumeator this handler handles</param>
+            /// <param name="shouldMoveNext">Predicate function used to determine if we should call move next
+            /// on the symbol's enumerator</param>
+            /// <param name="handleData">Handler for data if HandlesData=true</param>
+            public EnumeratorHandler(Symbol symbol, IEnumerator<BaseData> enumerator, Func<bool> shouldMoveNext = null, Action<BaseData> handleData = null)
+            {
+                Symbol = symbol;
+                Enumerator = enumerator;
+
+                _handleData = handleData;
+                _shouldMoveNext = shouldMoveNext ?? (() => true);
+            }
+
+            /// <summary>
+            /// Event invocator for the <see cref="EnumeratorFinished"/> event
+            /// </summary>
+            public void OnEnumeratorFinished()
+            {
+                EnumeratorFinished?.Invoke(this, this);
+            }
+
+            /// <summary>
+            /// Returns true if this enumerator should move next
+            /// </summary>
+            public bool ShouldMoveNext()
+            {
+                return _shouldMoveNext();
+            }
+
+            /// <summary>
+            /// Handles the specified data.
+            /// </summary>
+            /// <param name="data">The data to be handled</param>
+            public void HandleData(BaseData data)
+            {
+                _handleData?.Invoke(data);
+            }
+        }
+    }
+}

Engine/DataFeeds/BaseDownloaderDataProvider.cs

@@ -0,0 +1,69 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.IO;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Base downloader implementation with some helper methods
+    /// </summary>
+    public abstract class BaseDownloaderDataProvider : DefaultDataProvider
+    {
+        /// <summary>
+        /// Synchronizer in charge of guaranteeing a single download per path request
+        /// </summary>
+        private readonly KeyStringSynchronizer _singleDownloadSynchronizer = new();
+
+        /// <summary>
+        /// Helper method which guarantees each requested key is downloaded only once concurrently if required based on <see cref="NeedToDownload"/>
+        /// </summary>
+        /// <param name="key">A string representing where the data is stored</param>
+        /// <param name="download">The download operation we want to perform once concurrently per key</param>
+        /// <returns>A <see cref="Stream"/> of the data requested</returns>
+        protected Stream DownloadOnce(string key, Action<string> download)
+        {
+            // If we don't already have this file or its out of date, download it
+            if (NeedToDownload(key))
+            {
+                // only the first thread will download the rest will wait for him to finish
+                _singleDownloadSynchronizer.Execute(key, singleExecution: true, () => download(key));
+                // single download finished, let's get the stream!
+                return GetStream(key);
+            }
+
+            // even if we are not downloading the file because it exists we need to synchronize because the download might still be updating the file on disk
+            return _singleDownloadSynchronizer.Execute(key, () => GetStream(key));
+        }
+
+        /// <summary>
+        /// Get's the stream for a given file path
+        /// </summary>
+        protected virtual Stream GetStream(string key)
+        {
+            return base.Fetch(key);
+        }
+
+        /// <summary>
+        /// Main filter to determine if this file needs to be downloaded
+        /// </summary>
+        /// <param name="filePath">File we are looking at</param>
+        /// <returns>True if should download</returns>
+        protected abstract bool NeedToDownload(string filePath);
+    }
+}

Engine/DataFeeds/BaseSubscriptionDataSourceReader.cs

@@ -0,0 +1,143 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using System.ComponentModel;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Lean.Engine.DataFeeds.Transport;
+using QuantConnect.Algorithm;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// A base class for implementations of the <see cref="ISubscriptionDataSourceReader"/>
+    /// </summary>
+    public abstract class BaseSubscriptionDataSourceReader : ISubscriptionDataSourceReader
+    {
+        /// <summary>
+        /// True if we're in live mode, false for backtesting
+        /// </summary>
+        protected bool IsLiveMode { get; }
+
+        /// <summary>
+        /// The data cache provider to use
+        /// </summary>
+        protected IDataCacheProvider DataCacheProvider { get; }
+
+        /// <summary>
+        /// The object store to use
+        /// </summary>
+        protected IObjectStore ObjectStore { get; }
+
+        /// <summary>
+        /// Event fired when the specified source is considered invalid, this may
+        /// be from a missing file or failure to download a remote source
+        /// </summary>
+        public event EventHandler<InvalidSourceEventArgs> InvalidSource;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        protected BaseSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, bool isLiveMode, IObjectStore objectStore)
+        {
+            DataCacheProvider = dataCacheProvider;
+            IsLiveMode = isLiveMode;
+            ObjectStore = objectStore;
+        }
+
+        /// <summary>
+        /// Reads the specified <paramref name="source"/>
+        /// </summary>
+        /// <param name="source">The source to be read</param>
+        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
+        public abstract IEnumerable<BaseData> Read(SubscriptionDataSource source);
+
+        /// <summary>
+        /// Creates a new <see cref="IStreamReader"/> for the specified <paramref name="subscriptionDataSource"/>
+        /// </summary>
+        /// <param name="subscriptionDataSource">The source to produce an <see cref="IStreamReader"/> for</param>
+        /// <returns>A new instance of <see cref="IStreamReader"/> to read the source, or null if there was an error</returns>
+        protected IStreamReader CreateStreamReader(SubscriptionDataSource subscriptionDataSource)
+        {
+            IStreamReader reader = null;
+            try
+            {
+                switch (subscriptionDataSource.TransportMedium)
+                {
+                    case SubscriptionTransportMedium.LocalFile:
+                        reader = new LocalFileSubscriptionStreamReader(DataCacheProvider, subscriptionDataSource.Source);
+                        break;
+
+                    case SubscriptionTransportMedium.RemoteFile:
+                        reader = HandleRemoteSourceFile(subscriptionDataSource);
+                        break;
+
+                    case SubscriptionTransportMedium.Rest:
+                        reader = new RestSubscriptionStreamReader(subscriptionDataSource.Source, subscriptionDataSource.Headers, IsLiveMode);
+                        break;
+
+                    case SubscriptionTransportMedium.ObjectStore:
+                        reader = new ObjectStoreSubscriptionStreamReader(ObjectStore, subscriptionDataSource.Source);
+                        break;
+
+                    default:
+                        throw new InvalidEnumArgumentException("Unexpected SubscriptionTransportMedium specified: " + subscriptionDataSource.TransportMedium);
+                }
+            }
+            catch (Exception e)
+            {
+                OnInvalidSource(subscriptionDataSource, e);
+                return reader;
+            }
+
+            if (reader == null || reader.EndOfStream)
+            {
+                OnInvalidSource(subscriptionDataSource, new Exception($"The reader was empty for source: ${subscriptionDataSource.Source}"));
+                return null;
+            }
+            return reader;
+        }
+
+        /// <summary>
+        /// Event invocator for the <see cref="InvalidSource"/> event
+        /// </summary>
+        /// <param name="source">The <see cref="SubscriptionDataSource"/> that was invalid</param>
+        /// <param name="exception">The exception if one was raised, otherwise null</param>
+        protected void OnInvalidSource(SubscriptionDataSource source, Exception exception)
+        {
+            InvalidSource?.Invoke(this, new InvalidSourceEventArgs(source, exception));
+        }
+
+        /// <summary>
+        /// Opens up an IStreamReader for a remote file source
+        /// </summary>
+        private IStreamReader HandleRemoteSourceFile(SubscriptionDataSource source)
+        {
+            SubscriptionDataSourceReader.CheckRemoteFileCache();
+
+            try
+            {
+                // this will fire up a web client in order to download the 'source' file to the cache
+                return new RemoteFileSubscriptionStreamReader(DataCacheProvider, source.Source, Globals.Cache, source.Headers);
+            }
+            catch (Exception)
+            {
+                return null;
+            }
+        }
+    }
+}

Engine/DataFeeds/CachingFutureChainProvider.cs

@@ -0,0 +1,77 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// An implementation of <see cref="IFutureChainProvider"/> that will cache by date future contracts returned by another future chain provider.
+    /// </summary>
+    public class CachingFutureChainProvider : IFutureChainProvider
+    {
+        private readonly ConcurrentDictionary<Symbol, FutureChainCacheEntry> _cache = new ConcurrentDictionary<Symbol, FutureChainCacheEntry>();
+        private readonly IFutureChainProvider _futureChainProvider;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CachingFutureChainProvider"/> class
+        /// </summary>
+        /// <param name="futureChainProvider"></param>
+        public CachingFutureChainProvider(IFutureChainProvider futureChainProvider)
+        {
+            _futureChainProvider = futureChainProvider;
+        }
+
+        /// <summary>
+        /// Gets the list of future contracts for a given underlying symbol
+        /// </summary>
+        /// <param name="symbol">The underlying symbol</param>
+        /// <param name="date">The date for which to request the future chain (only used in backtesting)</param>
+        /// <returns>The list of future contracts</returns>
+        public IEnumerable<Symbol> GetFutureContractList(Symbol symbol, DateTime date)
+        {
+            List<Symbol> symbols;
+
+            FutureChainCacheEntry entry;
+            if (!_cache.TryGetValue(symbol, out entry) || date.Date != entry.Date)
+            {
+                symbols = _futureChainProvider.GetFutureContractList(symbol, date.Date).ToList();
+                _cache[symbol] = new FutureChainCacheEntry(date.Date, symbols);
+            }
+            else
+            {
+                symbols = entry.Symbols;
+            }
+
+            return symbols;
+        }
+
+        private class FutureChainCacheEntry
+        {
+            public DateTime Date { get; }
+            public List<Symbol> Symbols { get; }
+
+            public FutureChainCacheEntry(DateTime date, List<Symbol> symbols)
+            {
+                Date = date;
+                Symbols = symbols;
+            }
+        }
+    }
+}

Engine/DataFeeds/CachingOptionChainProvider.cs

@@ -0,0 +1,78 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// An implementation of <see cref="IOptionChainProvider"/> that will cache by date option contracts returned by another option chain provider.
+    /// </summary>
+    public class CachingOptionChainProvider : IOptionChainProvider
+    {
+        private readonly ConcurrentDictionary<Symbol, OptionChainCacheEntry> _cache = new ConcurrentDictionary<Symbol, OptionChainCacheEntry>();
+        private readonly IOptionChainProvider _optionChainProvider;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CachingOptionChainProvider"/> class
+        /// </summary>
+        /// <param name="optionChainProvider"></param>
+        public CachingOptionChainProvider(IOptionChainProvider optionChainProvider)
+        {
+            _optionChainProvider = optionChainProvider;
+        }
+
+        /// <summary>
+        /// Gets the list of option contracts for a given underlying symbol
+        /// </summary>
+        /// <param name="symbol">The option or the underlying symbol to get the option chain for.
+        /// Providing the option allows targetting an option ticker different than the default e.g. SPXW</param>
+        /// <param name="date">The date for which to request the option chain (only used in backtesting)</param>
+        /// <returns>The list of option contracts</returns>
+        public IEnumerable<Symbol> GetOptionContractList(Symbol symbol, DateTime date)
+        {
+            List<Symbol> symbols;
+
+            OptionChainCacheEntry entry;
+            if (!_cache.TryGetValue(symbol, out entry) || date.Date != entry.Date)
+            {
+                symbols = _optionChainProvider.GetOptionContractList(symbol, date.Date).ToList();
+                _cache[symbol] = new OptionChainCacheEntry(date.Date, symbols);
+            }
+            else
+            {
+                symbols = entry.Symbols;
+            }
+
+            return symbols;
+        }
+
+        private class OptionChainCacheEntry
+        {
+            public DateTime Date { get; }
+            public List<Symbol> Symbols { get; }
+
+            public OptionChainCacheEntry(DateTime date, List<Symbol> symbols)
+            {
+                Date = date;
+                Symbols = symbols;
+            }
+        }
+    }
+}

Engine/DataFeeds/ChainProviderInitializeParameters.cs

@@ -0,0 +1,46 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// DTO for initializing the <see cref="BacktestingOptionChainProvider"/>
+    /// </summary>
+    public class ChainProviderInitializeParameters
+    {
+        /// <summary>
+        /// The map file provider instance to us
+        /// </summary>
+        public IMapFileProvider MapFileProvider { get; set; }
+
+        /// <summary>
+        /// The history provider to use
+        /// </summary>
+        public IHistoryProvider HistoryProvider { get; set; }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ChainProviderInitializeParameters"/> class
+        /// </summary>
+        /// <param name="mapFileProvider">The map file provider instance to use</param>
+        /// <param name="historyProvider">The history provider to use</param>
+        public ChainProviderInitializeParameters(IMapFileProvider mapFileProvider, IHistoryProvider historyProvider)
+        {
+            MapFileProvider = mapFileProvider;
+            HistoryProvider = historyProvider;
+        }
+    }
+}

Engine/DataFeeds/CollectionSubscriptionDataSourceReader.cs

@@ -0,0 +1,135 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Collection Subscription Factory takes a BaseDataCollection from BaseData factories
+    /// and yields it one point at a time to the algorithm
+    /// </summary>
+    public class CollectionSubscriptionDataSourceReader : BaseSubscriptionDataSourceReader
+    {
+        private readonly DateTime _date;
+        private readonly BaseData _factory;
+        private readonly SubscriptionDataConfig _config;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CollectionSubscriptionDataSourceReader"/> class
+        /// </summary>
+        /// <param name="dataCacheProvider">Used to cache data for requested from the IDataProvider</param>
+        /// <param name="config">The subscription's configuration</param>
+        /// <param name="date">The date this factory was produced to read data for</param>
+        /// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
+        public CollectionSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode, IObjectStore objectStore)
+            :base(dataCacheProvider, isLiveMode, objectStore)
+        {
+            _date = date;
+            _config = config;
+            _factory = _config.GetBaseDataInstance();
+        }
+
+        /// <summary>
+        /// Event fired when an exception is thrown during a call to
+        /// <see cref="BaseData.Reader(SubscriptionDataConfig, string, DateTime, bool)"/>
+        /// </summary>
+        public event EventHandler<ReaderErrorEventArgs> ReaderError;
+
+        /// <summary>
+        /// Reads the specified <paramref name="source"/>
+        /// </summary>
+        /// <param name="source">The source to be read</param>
+        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
+        public override IEnumerable<BaseData> Read(SubscriptionDataSource source)
+        {
+            SubscriptionDataSourceReader.CheckRemoteFileCache();
+
+            IStreamReader reader = null;
+            try
+            {
+                reader = CreateStreamReader(source);
+                if (reader == null)
+                {
+                    yield break;
+                }
+
+                var raw = "";
+                while (!reader.EndOfStream)
+                {
+                    BaseDataCollection instances = null;
+                    try
+                    {
+                        raw = reader.ReadLine();
+                        var result = _factory.Reader(_config, raw, _date, IsLiveMode);
+                        instances = result as BaseDataCollection;
+                        if (instances == null && !reader.ShouldBeRateLimited)
+                        {
+                            OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
+                            continue;
+                        }
+                    }
+                    catch (Exception err)
+                    {
+                        OnReaderError(raw, err);
+                        if (!reader.ShouldBeRateLimited)
+                        {
+                            continue;
+                        }
+                    }
+
+                    if (IsLiveMode
+                        // this shouldn't happen, rest reader is the only one to be rate limited
+                        // and in live mode, but just in case...
+                        || instances == null && reader.ShouldBeRateLimited)
+                    {
+                        // in live trading these data points will be unrolled at the
+                        // 'LiveCustomDataSubscriptionEnumeratorFactory' level
+                        yield return instances;
+                    }
+                    else
+                    {
+                        foreach (var instance in instances.Data)
+                        {
+                            if (instance != null && instance.EndTime != default(DateTime))
+                            {
+                                yield return instance;
+                            }
+                        }
+                    }
+                }
+            }
+            finally
+            {
+                reader.DisposeSafely();
+            }
+        }
+
+        /// <summary>
+        /// Event invocator for the <see cref="ReaderError"/> event
+        /// </summary>
+        /// <param name="line">The line that caused the exception</param>
+        /// <param name="exception">The exception that was caught</param>
+        private void OnReaderError(string line, Exception exception)
+        {
+            var handler = ReaderError;
+            if (handler != null) handler(this, new ReaderErrorEventArgs(line, exception));
+        }
+    }
+}

Engine/DataFeeds/CompositeDataProvider.cs

@@ -0,0 +1,95 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using Newtonsoft.Json;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using QuantConnect.Configuration;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// This data provider will wrap and use multiple data providers internally in the provided order
+    /// </summary>
+    public class CompositeDataProvider : IDataProvider
+    {
+        /// <summary>
+        /// Event raised each time data fetch is finished (successfully or not)
+        /// </summary>
+        public event EventHandler<DataProviderNewDataRequestEventArgs> NewDataRequest;
+
+        private readonly List<IDataProvider> _dataProviders;
+
+        /// <summary>
+        /// Creates a new instance and initialize data providers used
+        /// </summary>
+        public CompositeDataProvider()
+        {
+            _dataProviders = new List<IDataProvider>();
+
+            var dataProvidersConfig = Config.Get("composite-data-providers");
+            if (!string.IsNullOrEmpty(dataProvidersConfig))
+            {
+                var dataProviders = JsonConvert.DeserializeObject<List<string>>(dataProvidersConfig);
+                foreach (var dataProvider in dataProviders)
+                {
+                    _dataProviders.Add(Composer.Instance.GetExportedValueByTypeName<IDataProvider>(dataProvider));
+                }
+
+                if (_dataProviders.Count == 0)
+                {
+                    throw new ArgumentException("CompositeDataProvider(): requires at least 1 valid data provider in 'composite-data-providers'");
+                }
+            }
+            else
+            {
+                throw new ArgumentException("CompositeDataProvider(): requires 'composite-data-providers' to be set with a valid type name");
+            }
+
+            _dataProviders.ForEach(x => x.NewDataRequest += OnNewDataRequest);
+        }
+
+        /// <summary>
+        /// Retrieves data to be used in an algorithm
+        /// </summary>
+        /// <param name="key">A string representing where the data is stored</param>
+        /// <returns>A <see cref="Stream"/> of the data requested</returns>
+        public Stream Fetch(string key)
+        {
+            for (var i = 0; i < _dataProviders.Count; i++)
+            {
+                var result = _dataProviders[i].Fetch(key);
+
+                if (result != null)
+                {
+                    return result;
+                }
+            }
+
+            return null;
+        }
+
+        /// <summary>
+        /// Event invocator for the <see cref="NewDataRequest"/> event
+        /// </summary>
+        private void OnNewDataRequest(object sender, DataProviderNewDataRequestEventArgs e)
+        {
+            NewDataRequest?.Invoke(this, e);
+        }
+    }
+}

Engine/DataFeeds/CompositeTimeProvider.cs

@@ -0,0 +1,58 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// The composite time provider will source it's current time using the smallest time from the given providers
+    /// </summary>
+    public class CompositeTimeProvider : ITimeProvider
+    {
+        private readonly ITimeProvider[] _timeProviders;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="timeProviders">The time providers to use. Will default to the real time provider if empty</param>
+        public CompositeTimeProvider(IEnumerable<ITimeProvider> timeProviders)
+        {
+            _timeProviders = timeProviders.DefaultIfEmpty(RealTimeProvider.Instance).ToArray();
+        }
+
+        /// <summary>
+        /// Gets the current time in UTC
+        /// </summary>
+        /// <returns>The current time in UTC</returns>
+        public DateTime GetUtcNow()
+        {
+            var result = DateTime.MaxValue;
+            for (var i = 0; i < _timeProviders.Length; i++)
+            {
+                var utcNow = _timeProviders[i].GetUtcNow();
+
+                if (utcNow < result)
+                {
+                    // we return the smallest
+                    result = utcNow;
+                }
+            }
+            return result;
+        }
+    }
+}

Engine/DataFeeds/CreateStreamReaderErrorEventArgs.cs

@@ -0,0 +1,53 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Event arguments for the <see cref="TextSubscriptionDataSourceReader"/>'s CreateStreamReader event
+    /// </summary>
+    public sealed class CreateStreamReaderErrorEventArgs : EventArgs
+    {
+        /// <summary>
+        /// Gets the date of the source
+        /// </summary>
+        public DateTime Date
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the source that caused the error
+        /// </summary>
+        public SubscriptionDataSource Source
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CreateStreamReaderErrorEventArgs"/> class
+        /// </summary>
+        /// <param name="date">The date of the source</param>
+        /// <param name="source">The source that cause the error</param>
+        public CreateStreamReaderErrorEventArgs(DateTime date, SubscriptionDataSource source)
+        {
+            Date = date;
+            Source = source;
+        }
+    }
+}

Engine/DataFeeds/CurrencySubscriptionDataConfigManager.cs

@@ -0,0 +1,163 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Brokerages;
+using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Helper class to keep track of required internal currency <see cref="SubscriptionDataConfig"/>.
+    /// This class is used by the <see cref="UniverseSelection"/>
+    /// </summary>
+    public class CurrencySubscriptionDataConfigManager
+    {
+        private readonly HashSet<SubscriptionDataConfig> _toBeAddedCurrencySubscriptionDataConfigs;
+        private readonly HashSet<SubscriptionDataConfig> _addedCurrencySubscriptionDataConfigs;
+        private bool _ensureCurrencyDataFeeds;
+        private bool _pendingSubscriptionDataConfigs;
+        private readonly CashBook _cashBook;
+        private readonly Resolution _defaultResolution;
+        private readonly SecurityManager _securityManager;
+        private readonly SubscriptionManager _subscriptionManager;
+        private readonly ISecurityService _securityService;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="cashBook">The cash book instance</param>
+        /// <param name="securityManager">The SecurityManager, required by the cash book for creating new securities</param>
+        /// <param name="subscriptionManager">The SubscriptionManager, required by the cash book for creating new subscription data configs</param>
+        /// <param name="securityService">The SecurityService, required by the cash book for creating new securities</param>
+        /// <param name="defaultResolution">The default resolution to use for the internal subscriptions</param>
+        public CurrencySubscriptionDataConfigManager(CashBook cashBook,
+            SecurityManager securityManager,
+            SubscriptionManager subscriptionManager,
+            ISecurityService securityService,
+            Resolution defaultResolution)
+        {
+            cashBook.Updated += (sender, args) =>
+            {
+                if (args.UpdateType == CashBookUpdateType.Added)
+                {
+                    _ensureCurrencyDataFeeds = true;
+                }
+            };
+
+            _defaultResolution = defaultResolution;
+            _pendingSubscriptionDataConfigs = false;
+            _securityManager = securityManager;
+            _subscriptionManager = subscriptionManager;
+            _securityService = securityService;
+            _cashBook = cashBook;
+            _addedCurrencySubscriptionDataConfigs = new HashSet<SubscriptionDataConfig>();
+            _toBeAddedCurrencySubscriptionDataConfigs = new HashSet<SubscriptionDataConfig>();
+        }
+
+        /// <summary>
+        /// Will verify if there are any <see cref="SubscriptionDataConfig"/> to be removed
+        /// for a given added <see cref="Symbol"/>.
+        /// </summary>
+        /// <param name="addedSymbol">The symbol that was added to the data feed system</param>
+        /// <returns>The SubscriptionDataConfig to be removed, null if none</returns>
+        public SubscriptionDataConfig GetSubscriptionDataConfigToRemove(Symbol addedSymbol)
+        {
+            if (addedSymbol.SecurityType == SecurityType.Crypto
+                || addedSymbol.SecurityType == SecurityType.CryptoFuture
+                || addedSymbol.SecurityType == SecurityType.Forex
+                || addedSymbol.SecurityType == SecurityType.Cfd)
+            {
+                var currencyDataFeed = _addedCurrencySubscriptionDataConfigs
+                    .FirstOrDefault(x => x.Symbol == addedSymbol);
+                if (currencyDataFeed != null)
+                {
+                    return currencyDataFeed;
+                }
+            }
+            return null;
+        }
+
+        /// <summary>
+        /// Will update pending currency <see cref="SubscriptionDataConfig"/>
+        /// </summary>
+        /// <returns>True when there are pending currency subscriptions <see cref="GetPendingSubscriptionDataConfigs"/></returns>
+        public bool UpdatePendingSubscriptionDataConfigs(IBrokerageModel brokerageModel)
+        {
+            if (_ensureCurrencyDataFeeds)
+            {
+                // this allows us to handle the case where SetCash is called when no security has been really added
+                EnsureCurrencySubscriptionDataConfigs(SecurityChanges.None, brokerageModel);
+            }
+            return _pendingSubscriptionDataConfigs;
+        }
+
+        /// <summary>
+        /// Will return any pending internal currency <see cref="SubscriptionDataConfig"/> and remove them as pending.
+        /// </summary>
+        /// <returns>Will return the <see cref="SubscriptionDataConfig"/> to be added</returns>
+        public IEnumerable<SubscriptionDataConfig> GetPendingSubscriptionDataConfigs()
+        {
+            var result = new List<SubscriptionDataConfig>();
+            if (_pendingSubscriptionDataConfigs)
+            {
+                foreach (var subscriptionDataConfig in _toBeAddedCurrencySubscriptionDataConfigs)
+                {
+                    _addedCurrencySubscriptionDataConfigs.Add(subscriptionDataConfig);
+                    result.Add(subscriptionDataConfig);
+                }
+                _toBeAddedCurrencySubscriptionDataConfigs.Clear();
+                _pendingSubscriptionDataConfigs = false;
+            }
+            return result;
+        }
+
+        /// <summary>
+        /// Checks the current <see cref="SubscriptionDataConfig"/> and adds new necessary currency pair feeds to provide real time conversion data
+        /// </summary>
+        /// <returns>True if a new currency was introduced, either as a new internal conversion feed or as a new cash
+        /// entry added to the cashbook. Lets callers skip follow up work like seeding the new conversion rates when
+        /// nothing was added</returns>
+        public bool EnsureCurrencySubscriptionDataConfigs(SecurityChanges securityChanges, IBrokerageModel brokerageModel)
+        {
+            // a new cash added to the cashbook also needs its conversion rate seeded, even when its conversion
+            // security is an already subscribed one and no new internal feed is introduced below
+            var newCashAdded = _ensureCurrencyDataFeeds;
+            _ensureCurrencyDataFeeds = false;
+            // remove any 'to be added' if the security has already been added
+            _toBeAddedCurrencySubscriptionDataConfigs.RemoveWhere(
+                config => securityChanges.AddedSecurities.Any(x => x.Symbol == config.Symbol));
+
+            var newConfigs = _cashBook.EnsureCurrencyDataFeeds(
+                _securityManager,
+                _subscriptionManager,
+                brokerageModel.DefaultMarkets,
+                securityChanges,
+                _securityService,
+                _defaultResolution);
+            foreach (var config in newConfigs)
+            {
+                _toBeAddedCurrencySubscriptionDataConfigs.Add(config);
+            }
+            _pendingSubscriptionDataConfigs = _toBeAddedCurrencySubscriptionDataConfigs.Any();
+
+            return newConfigs.Count > 0 || newCashAdded;
+        }
+    }
+}

Engine/DataFeeds/DataChannelProvider.cs

@@ -0,0 +1,57 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Packets;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.Market;
+using QuantConnect.Data.Fundamental;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Specifies data channel settings
+    /// </summary>
+    public class DataChannelProvider : IDataChannelProvider
+    {
+        /// <summary>
+        /// Initializes the instance with an algorithm node packet
+        /// </summary>
+        /// <param name="packet">Algorithm node packet</param>
+        public virtual void Initialize(AlgorithmNodePacket packet)
+        {
+        }
+
+        /// <summary>
+        /// True if this subscription request should be streamed
+        /// </summary>
+        public virtual bool ShouldStreamSubscription(SubscriptionDataConfig config)
+        {
+            return IsStreamingType(config) || !config.IsCustomData && config.Type != typeof(CoarseFundamental) && config.Type != typeof(Fundamental) && config.Type != typeof(MarginInterestRate);
+        }
+
+        /// <summary>
+        /// Returns true if the data type for the given subscription configuration supports streaming
+        /// </summary>
+        protected static bool IsStreamingType(SubscriptionDataConfig configuration)
+        {
+            var dataTypeInstance = configuration.Type.GetBaseDataInstance();
+            var source = dataTypeInstance.GetSource(configuration, DateTime.UtcNow, true);
+            return source != null && source.TransportMedium == SubscriptionTransportMedium.Streaming;
+        }
+    }
+}

Engine/DataFeeds/DataDownloader/CanonicalDataDownloaderDecorator.cs

@@ -0,0 +1,292 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using System.Threading;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using System.Threading.Tasks;
+using QuantConnect.Interfaces;
+using QuantConnect.Configuration;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using QuantConnect.Lean.Engine.HistoricalData;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.DataDownloader
+{
+    /// <summary>
+    /// Decorates an <see cref="IDataDownloader"/> to support canonical symbols by automatically
+    /// resolving their option or future contract chains and downloading data for each constituent contract.
+    /// </summary>
+    public class CanonicalDataDownloaderDecorator : IDataDownloader
+    {
+        /// <summary>
+        /// Prevents multiple warnings being fired when the underlying data downloader doesn't support canonical symbols.
+        /// </summary>
+        private bool _firedCanonicalNotSupportedWarning;
+
+        /// <summary>
+        /// Lazily initialized option chain provider for resolving option contract lists.
+        /// </summary>
+        private readonly IOptionChainProvider _optionChainProvider;
+
+        /// <summary>
+        /// Lazily initialized future chain provider for resolving future contract lists.
+        /// </summary>
+        private readonly IFutureChainProvider _futureChainProvider;
+
+        /// <summary>
+        /// The underlying data downloader that performs the actual data retrieval.
+        /// </summary>
+        private readonly IDataDownloader _dataDownloader;
+
+        /// <summary>
+        /// Controls parallelism for concurrent operations, 
+        /// limiting execution to a configurable number of threads (default: 4) on the default task scheduler.
+        /// </summary>
+        private readonly ParallelOptions _parallelOptions = new()
+        {
+            MaxDegreeOfParallelism = Config.GetInt("downloader-thread-count", 4),
+            TaskScheduler = TaskScheduler.Default
+        };
+
+        /// <summary>
+        /// Configurable look-back period for canonical option symbols, used to limit the date range of underlying contract data downloads.
+        /// </summary>
+        private static readonly int _optionLookbackYeard = Config.GetInt("options-lookback-years", 1);
+
+        /// <summary>
+        /// Configurable look-back period for canonical future symbols, used to limit the date range of underlying contract data downloads.
+        /// </summary>
+        private static readonly int _futureLookbackYeard = Config.GetInt("futures-lookback-years", 2);
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CanonicalDataDownloaderDecorator"/> class.
+        /// </summary>
+        /// <param name="dataDownloader">The underlying data downloader to decorate with canonical symbol support.</param>
+        /// <param name="dataProvider">The data provider used for initializing chain providers.</param>
+        /// <param name="mapFileProvider">The map file provider used for initializing chain providers.</param>
+        /// <param name="factorFileProvider">The factor file provider used for initializing chain providers.</param>
+        public CanonicalDataDownloaderDecorator(IDataDownloader dataDownloader, IDataProvider dataProvider, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
+        {
+            _dataDownloader = dataDownloader;
+
+            var historyManager = new HistoryProviderManager();
+            historyManager.Initialize(
+                new HistoryProviderInitializeParameters(
+                    job: null,
+                    api: null,
+                    dataProvider,
+                    new SingleEntryDataCacheProvider(dataProvider, isDataEphemeral: true),
+                    mapFileProvider,
+                    factorFileProvider: factorFileProvider, // Probably not needed since canonical data doesn't require factor files
+                    statusUpdateAction: null,
+                    parallelHistoryRequestsEnabled: false,
+                    new DataPermissionManager(),
+                    objectStore: null,
+                    new AlgorithmSettings()));
+
+            _optionChainProvider = Composer.Instance.GetPart<IOptionChainProvider>();
+            if (_optionChainProvider == null)
+            {
+                var baseOptionChainProvider = new LiveOptionChainProvider();
+                baseOptionChainProvider.Initialize(new(mapFileProvider, historyManager));
+                _optionChainProvider = new CachingOptionChainProvider(baseOptionChainProvider);
+                Composer.Instance.AddPart(_optionChainProvider);
+            }
+
+            _futureChainProvider = Composer.Instance.GetPart<IFutureChainProvider>();
+            if (_futureChainProvider == null)
+            {
+                var baseFutureChainProvider = new BacktestingFutureChainProvider();
+                baseFutureChainProvider.Initialize(new(mapFileProvider, historyManager));
+                _futureChainProvider = new CachingFutureChainProvider(baseFutureChainProvider);
+                Composer.Instance.AddPart(_futureChainProvider);
+            }
+        }
+
+        /// <summary>
+        /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
+        /// For canonical symbols, automatically resolves and downloads data for all underlying contracts.
+        /// </summary>
+        /// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
+        /// <returns>Enumerable of base data for this symbol</returns>
+        public IEnumerable<BaseData>? Get(DataDownloaderGetParameters dataDownloaderGetParameters)
+        {
+            var downloadedData = default(IEnumerable<BaseData>?);
+            try
+            {
+                downloadedData = _dataDownloader.Get(dataDownloaderGetParameters);
+            }
+            catch (Exception ex)
+            {
+                Log.Error($"{nameof(CanonicalDataDownloaderDecorator)}.{nameof(Get)}.Exceptoin: {ex.Message}");
+            }
+
+            if (downloadedData == null && dataDownloaderGetParameters.Symbol.IsCanonical())
+            {
+                if (!_firedCanonicalNotSupportedWarning)
+                {
+                    _firedCanonicalNotSupportedWarning = true;
+                    Log.Trace($"{nameof(CanonicalDataDownloaderDecorator)}.{nameof(Get)}: {_dataDownloader.GetType().Name} does not support canonical symbols. Falling back to chain provider.");
+                }
+                downloadedData = GetContractsData(dataDownloaderGetParameters);
+            }
+
+            return downloadedData;
+        }
+
+        /// <summary>
+        /// Tries to adjust the date range for a given contract based on its security type and expiry date.
+        /// The start date is clamped to a minimum look-back period and the end date is clamped to the contract expiry date.
+        /// Returns false if the minimum look-back period exceeds the requested end date, meaning no valid range exists.
+        /// </summary>
+        /// <param name="contract">The contract symbol containing the security type and expiry date.</param>
+        /// <param name="originalStartDateUtc">The requested start date in UTC.</param>
+        /// <param name="originalEndDateUtc">The requested end date in UTC.</param>
+        /// <param name="adjustedStartDateUtc">The adjusted start date in UTC, or default if no valid range exists.</param>
+        /// <param name="adjustedEndDateUtc">The adjusted end date in UTC, or default if no valid range exists.</param>
+        /// <returns>True if a valid adjusted date range was found, false otherwise.</returns>
+        public static bool TryAdjustDateRangeForContract(Symbol contract, DateTime originalStartDateUtc, DateTime originalEndDateUtc,
+            out DateTime adjustedStartDateUtc, out DateTime adjustedEndDateUtc)
+        {
+            var expiryDate = contract.ID.Date;
+            var minLookBack = expiryDate;
+            if (contract.ID.SecurityType.IsOption())
+            {
+                minLookBack = expiryDate.AddYears(-_optionLookbackYeard);
+            }
+            else if (contract.ID.SecurityType == SecurityType.Future)
+            {
+                minLookBack = expiryDate.AddYears(-_futureLookbackYeard);
+            }
+
+            if (minLookBack > originalEndDateUtc || expiryDate < originalStartDateUtc)
+            {
+                adjustedStartDateUtc = default;
+                adjustedEndDateUtc = default;
+                return false;
+            }
+
+            adjustedStartDateUtc = minLookBack >= originalStartDateUtc ? minLookBack : originalStartDateUtc;
+            adjustedEndDateUtc = expiryDate <= originalEndDateUtc ? expiryDate.AddDays(1) : originalEndDateUtc;
+            return true;
+        }
+
+        /// <summary>
+        /// Downloads data for all contracts of a canonical symbol in parallel, streaming results as they arrive.
+        /// </summary>
+        private IEnumerable<BaseData>? GetContractsData(DataDownloaderGetParameters parameters)
+        {
+            var contracts = GetContracts(parameters.Symbol, parameters.StartUtc, parameters.EndUtc);
+
+            var blockingCollection = new BlockingCollection<BaseData>();
+
+            var processedContracts = 0L;
+            var producerTask = Task.Run(() =>
+            {
+                try
+                {
+                    Parallel.ForEach(
+                        contracts,
+                        _parallelOptions,
+                        contract =>
+                        {
+                            Interlocked.Increment(ref processedContracts);
+                            if (!TryAdjustDateRangeForContract(contract, parameters.StartUtc, parameters.EndUtc, out var adjustedStartDateUtc, out var adjustedEndDateUtc))
+                            {
+                                return;
+                            }
+
+                            var contractParameters = new DataDownloaderGetParameters(
+                                contract,
+                                parameters.Resolution,
+                                adjustedStartDateUtc,
+                                adjustedEndDateUtc,
+                                parameters.TickType);
+
+                            try
+                            {
+                                var contractData = _dataDownloader.Get(contractParameters);
+
+                                foreach (var data in contractData)
+                                {
+                                    blockingCollection.Add(data);
+                                }
+                            }
+                            catch (Exception ex)
+                            {
+                                Log.Debug($"{nameof(CanonicalDataDownloaderDecorator)}.{nameof(GetContractsData)}: " +
+                                    $"Error downloading data for {contractParameters}. Exception: {ex.Message}. Continuing...");
+                                return;
+                            }
+                        });
+                }
+                finally
+                {
+                    Log.Debug($"{nameof(CanonicalDataDownloaderDecorator)}.{nameof(GetContractsData)}: Finished downloading {processedContracts} for canonical symbol.");
+                    blockingCollection.CompleteAdding();
+                }
+            });
+
+            var consumingEnumerable = blockingCollection.GetConsumingEnumerable();
+
+            if (!consumingEnumerable.Any())
+            {
+                if (Interlocked.Read(ref processedContracts) == 0)
+                {
+                    Log.Error($"{nameof(CanonicalDataDownloaderDecorator)}.{nameof(GetContractsData)}: No contracts were found. Do you have universe data?");
+                }
+                return null;
+            }
+
+            return consumingEnumerable;
+        }
+
+        /// <summary>
+        /// Retrieves unique contracts for the given canonical symbol across the specified date range.
+        /// </summary>
+        private IEnumerable<Symbol> GetContracts(Symbol symbol, DateTime startUtc, DateTime endUtc)
+        {
+            var chainProvider = default(Func<Symbol, DateTime, IEnumerable<Symbol>>);
+            if (symbol.SecurityType == SecurityType.Future)
+            {
+                chainProvider = _futureChainProvider.GetFutureContractList;
+            }
+            else if (symbol.SecurityType.IsOption())
+            {
+                chainProvider = _optionChainProvider.GetOptionContractList;
+            }
+            else
+            {
+                throw new ArgumentException($"Unsupported security type {symbol.SecurityType} for canonical data downloader", nameof(symbol));
+            }
+
+            var contractsCache = new HashSet<Symbol>();
+            foreach (var date in Time.EachDay(startUtc.Date, endUtc.Date))
+            {
+                foreach (var contract in chainProvider(symbol, date))
+                {
+                    if (contractsCache.Add(contract))
+                    {
+                        yield return contract;
+                    }
+                }
+            }
+        }
+    }
+}

Engine/DataFeeds/DataDownloader/DataDownloaderSelector.cs

@@ -0,0 +1,66 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using QuantConnect.Configuration;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.DataDownloader
+{
+    /// <summary>Selects the appropriate data downloader based on the data type.</summary>
+    public class DataDownloaderSelector : IDisposable
+    {
+        private readonly IDataDownloader _baseDataDownloader;
+        private readonly CanonicalDataDownloaderDecorator _canonicalDataDownloaderDecorator;
+
+        /// <summary>Initializes a new instance of the <see cref="DataDownloaderSelector"/> class.</summary>
+        /// <param name="baseDataDownloader">The base data downloader instance.</param>
+        /// <param name="mapFileProvider">The map file provider used for initializing chain providers.</param>
+        /// <param name="dataProvider">The data provider used for initializing chain providers.</param>
+        /// <param name="factorFileProvider">The factor file provider used for initializing chain providers.</param>
+        public DataDownloaderSelector(
+            IDataDownloader baseDataDownloader,
+            IMapFileProvider mapFileProvider,
+            IDataProvider dataProvider,
+            IFactorFileProvider factorFileProvider = null)
+        {
+            factorFileProvider ??= Composer.Instance.GetPart<IFactorFileProvider>();
+            if (factorFileProvider == null)
+            {
+                factorFileProvider = Composer.Instance.GetExportedValueByTypeName<IFactorFileProvider>(Config.Get("factor-file-provider", "LocalDiskFactorFileProvider"));
+                factorFileProvider.Initialize(mapFileProvider, dataProvider);
+            }
+
+            _baseDataDownloader = baseDataDownloader;
+            _canonicalDataDownloaderDecorator = new CanonicalDataDownloaderDecorator(_baseDataDownloader, dataProvider, mapFileProvider, factorFileProvider);
+        }
+
+        /// <summary>Disposes the base downloader and the decorator if it was initialized.</summary>
+        public void Dispose()
+        {
+            (_baseDataDownloader as IDisposable)?.DisposeSafely();
+        }
+
+        /// <summary>Returns the appropriate downloader for the given data type.</summary>
+        /// <param name="dataType">The type of data to download.</param>
+        /// <returns>The base downloader for common lean data types, otherwise the canonical decorator.</returns>
+        public IDataDownloader GetDataDownloader(Type dataType)
+        {
+            return LeanData.IsCommonLeanDataType(dataType) ? _canonicalDataDownloaderDecorator : _baseDataDownloader;
+        }
+    }
+}

Engine/DataFeeds/DataFeedPacket.cs

@@ -0,0 +1,102 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Defines a container type to hold data produced by a data feed subscription
+    /// </summary>
+    public class DataFeedPacket
+    {
+        private static readonly IReadOnlyRef<bool> _false = Ref.CreateReadOnly(() => false);
+        private readonly IReadOnlyRef<bool> _isRemoved;
+
+        /// <summary>
+        /// The security
+        /// </summary>
+        public ISecurityPrice Security
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// The subscription configuration that produced this data
+        /// </summary>
+        public SubscriptionDataConfig Configuration
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the number of data points held within this packet
+        /// </summary>
+        public int Count => Data.Count;
+
+        /// <summary>
+        /// The data for the security
+        /// </summary>
+        public List<BaseData> Data { get; }
+
+        /// <summary>
+        /// Gets whether or not this packet should be filtered out due to the subscription being removed
+        /// </summary>
+        public bool IsSubscriptionRemoved => _isRemoved.Value;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="DataFeedPacket"/> class
+        /// </summary>
+        /// <param name="security">The security whose data is held in this packet</param>
+        /// <param name="configuration">The subscription configuration that produced this data</param>
+        /// <param name="isSubscriptionRemoved">Reference to whether or not the subscription has since been removed, defaults to false</param>
+        public DataFeedPacket(ISecurityPrice security, SubscriptionDataConfig configuration, IReadOnlyRef<bool> isSubscriptionRemoved = null)
+            : this(security,
+                configuration,
+                new List<BaseData>(4), // performance: by default the list has 0 capacity, so lets initialize it with at least 4 (which is the default)
+                isSubscriptionRemoved)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="DataFeedPacket"/> class
+        /// </summary>
+        /// <param name="security">The security whose data is held in this packet</param>
+        /// <param name="configuration">The subscription configuration that produced this data</param>
+        /// <param name="data">The data to add to this packet. The list reference is reused
+        /// internally and NOT copied.</param>
+        /// <param name="isSubscriptionRemoved">Reference to whether or not the subscription has since been removed, defaults to false</param>
+        public DataFeedPacket(ISecurityPrice security, SubscriptionDataConfig configuration, List<BaseData> data, IReadOnlyRef<bool> isSubscriptionRemoved = null)
+        {
+            Security = security;
+            Configuration = configuration;
+            Data = data;
+            _isRemoved = isSubscriptionRemoved ?? _false;
+        }
+
+        /// <summary>
+        /// Adds the specified data to this packet
+        /// </summary>
+        /// <param name="data">The data to be added to this packet</param>
+        public void Add(BaseData data)
+        {
+            Data.Add(data);
+        }
+    }
+}

Engine/DataFeeds/DataManager.cs

@@ -0,0 +1,766 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// DataManager will manage the subscriptions for both the DataFeeds and the SubscriptionManager
+    /// </summary>
+    public class DataManager : IAlgorithmSubscriptionManager, IDataFeedSubscriptionManager, IDataManager
+    {
+        private readonly IDataFeed _dataFeed;
+        private readonly MarketHoursDatabase _marketHoursDatabase;
+        private readonly ITimeKeeper _timeKeeper;
+        private readonly bool _liveMode;
+        private bool _sentUniverseScheduleWarning;
+        private readonly IRegisteredSecurityDataTypesProvider _registeredTypesProvider;
+        private readonly IDataPermissionManager _dataPermissionManager;
+        private List<SubscriptionDataConfig> _subscriptionDataConfigsEnumerator;
+        private readonly IAlgorithm _algorithm;
+
+        private bool _unsupportedUniverseSettingsResolutionWarningSent;
+
+        /// There is no ConcurrentHashSet collection in .NET,
+        /// so we use ConcurrentDictionary with byte value to minimize memory usage
+        private readonly Dictionary<SubscriptionDataConfig, SubscriptionDataConfig> _subscriptionManagerSubscriptions = new();
+
+        /// <summary>
+        /// Event fired when a new subscription is added
+        /// </summary>
+        public event EventHandler<Subscription> SubscriptionAdded;
+
+        /// <summary>
+        /// Event fired when an existing subscription is removed
+        /// </summary>
+        public event EventHandler<Subscription> SubscriptionRemoved;
+
+        /// <summary>
+        /// Creates a new instance of the DataManager
+        /// </summary>
+        public DataManager(
+            IDataFeed dataFeed,
+            UniverseSelection universeSelection,
+            IAlgorithm algorithm,
+            ITimeKeeper timeKeeper,
+            MarketHoursDatabase marketHoursDatabase,
+            bool liveMode,
+            IRegisteredSecurityDataTypesProvider registeredTypesProvider,
+            IDataPermissionManager dataPermissionManager)
+        {
+            _dataFeed = dataFeed;
+            UniverseSelection = universeSelection;
+            UniverseSelection.SetDataManager(this);
+            AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
+            _timeKeeper = timeKeeper;
+            _marketHoursDatabase = marketHoursDatabase;
+            _liveMode = liveMode;
+            _registeredTypesProvider = registeredTypesProvider;
+            _dataPermissionManager = dataPermissionManager;
+            _algorithm = algorithm;
+
+            // wire ourselves up to receive notifications when universes are added/removed
+            algorithm.UniverseManager.CollectionChanged += (sender, args) =>
+            {
+                var universe = args.Value;
+                switch (args.Action)
+                {
+                    case NotifyCollectionChangedAction.Replace:
+                    case NotifyCollectionChangedAction.Add:
+                        var config = universe.Configuration;
+                        var start = algorithm.UtcTime;
+                        if (algorithm.GetLocked() && args.Action == NotifyCollectionChangedAction.Add && universe is UserDefinedUniverse)
+                        {
+                            // If it is an add, after initialize, we will set time 1 tick ahead to properly sync data
+                            // with next timeslice, avoid emitting now twice, if it is a remove then we will set time to now
+                            // we do the same in the 'DataManager' when handling FF resolution changes
+                            start = start.AddTicks(1);
+                        }
+
+                        var end = algorithm.LiveMode ? Time.EndOfTime
+                            : algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
+
+                        Security security;
+                        if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
+                        {
+                            // create a canonical security object if it doesn't exist
+                            security = new Security(
+                                _marketHoursDatabase.GetExchangeHours(config),
+                                config,
+                                algorithm.Portfolio.CashBook[algorithm.AccountCurrency],
+                                SymbolProperties.GetDefault(algorithm.AccountCurrency),
+                                algorithm.Portfolio.CashBook,
+                                RegisteredSecurityDataTypesProvider.Null,
+                                new SecurityCache()
+                             );
+                        }
+
+                        // Let's adjust the start time to the previous tradable date
+                        // so universe selection always happens right away at the start of the algorithm.
+                        var universeType = universe.GetType();
+                        if (
+                            // We exclude the UserDefinedUniverse because their selection already happens at the algorithm start time.
+                            // For instance, ETFs universe selection depends its first trigger time to be before the equity universe
+                            // (the UserDefinedUniverse), because the ETFs are EndTime-indexed and that would make their first selection
+                            // time to be before the algorithm start time, with the EndTime being the algorithms's start date,
+                            // and both the Equity and the ETFs constituents first selection to happen together.
+                            !universeType.IsAssignableTo(typeof(UserDefinedUniverse)) &&
+                            // We exclude the ScheduledUniverse because it's already scheduled to run at a specific time.
+                            // Adjusting the start time would cause the first selection trigger time to be before the algorithm start time,
+                            // making the selection to be triggered at the first algorithm time, which would be the exact StartDate.
+                            universeType != typeof(ScheduledUniverse))
+                        {
+                            const int maximumLookback = 365;
+                            var loopCount = 0;
+                            var startLocalTime = start.ConvertFromUtc(security.Exchange.TimeZone);
+                            if (universe.UniverseSettings.Schedule.Initialized)
+                            {
+                                do
+                                {
+                                    // determine if there's a scheduled selection time at the current start local time date, note that next
+                                    // we get the previous day of the first scheduled date we find, so we are sure the data is available to trigger selection
+                                    if (universe.UniverseSettings.Schedule.Get(startLocalTime.Date, startLocalTime.Date).Any())
+                                    {
+                                        break;
+                                    }
+                                    startLocalTime = startLocalTime.AddDays(-1);
+                                    if (++loopCount >= maximumLookback)
+                                    {
+                                        // fallback to the original, we found none
+                                        startLocalTime = algorithm.UtcTime.ConvertFromUtc(security.Exchange.TimeZone);
+                                        if (!_sentUniverseScheduleWarning)
+                                        {
+                                            // just in case
+                                            _sentUniverseScheduleWarning = true;
+                                            algorithm.Debug($"Warning: Found no valid start time for scheduled universe, will use default");
+                                        }
+                                    }
+                                } while (loopCount < maximumLookback);
+                            }
+
+                            startLocalTime = Time.GetStartTimeForTradeBars(security.Exchange.Hours, startLocalTime,
+                                // disable universe selection on extended market hours, for example futures/index options have a sunday pre market we are not interested on
+                                Time.OneDay, 1, extendedMarketHours: false, config.DataTimeZone,
+                                LeanData.UseDailyStrictEndTimes(algorithm.Settings, config.Type, security.Symbol, Time.OneDay, security.Exchange.Hours));
+                            start = startLocalTime.ConvertToUtc(security.Exchange.TimeZone);
+                        }
+
+                        AddSubscription(
+                            new SubscriptionRequest(true,
+                                universe,
+                                security,
+                                config,
+                                start,
+                                end));
+                        break;
+
+                    case NotifyCollectionChangedAction.Remove:
+                        // removing the subscription will be handled by the SubscriptionSynchronizer
+                        // in the next loop as well as executing a UniverseSelection one last time.
+                        if (!universe.DisposeRequested)
+                        {
+                            universe.Dispose();
+                        }
+                        break;
+
+                    default:
+                        throw new NotImplementedException("The specified action is not implemented: " + args.Action);
+                }
+            };
+
+            DataFeedSubscriptions = new SubscriptionCollection();
+            if (!_liveMode)
+            {
+                DataFeedSubscriptions.FillForwardResolutionChanged += (object sender, FillForwardResolutionChangedEvent changedEvent) =>
+                {
+                    var requests = DataFeedSubscriptions
+                        // we don't fill forward tick resolution so we don't need to touch their subscriptions
+                        .Where(subscription => subscription.Configuration.FillDataForward && subscription.Configuration.Resolution != Resolution.Tick)
+                        .SelectMany(subscription => subscription.SubscriptionRequests)
+                        .ToList();
+
+                    if (requests.Count > 0)
+                    {
+                        Log.Trace($"DataManager(): Fill forward resolution has changed from {changedEvent.Old} to {changedEvent.New} at utc: {algorithm.UtcTime}. " +
+                            $"Restarting {requests.Count} subscriptions...");
+
+                        // disable reentry while we remove and re add
+                        DataFeedSubscriptions.FreezeFillForwardResolution(true);
+
+                        // remove
+                        foreach (var request in requests)
+                        {
+                            // force because we want them actually removed even if still a member of the universe, because the FF res changed
+                            // which means we will drop any data points that could be in the next potential slice being created
+                            RemoveSubscriptionInternal(request.Configuration, universe: request.Universe, forceSubscriptionRemoval: true);
+                        }
+
+                        // re add
+                        foreach (var request in requests)
+                        {
+                            // If it is an add we will set time 1 tick ahead to properly sync data
+                            // with next timeslice, avoid emitting now twice.
+                            // We do the same in the 'TimeTriggeredUniverseSubscriptionEnumeratorFactory' when handling changes
+                            var startUtc = algorithm.UtcTime;
+                            // If the algorithm is not initialized (locked) the request start time can be even before the algorithm start time,
+                            // like in the case of universe requests that are scheduled to run at a specific time in the past for immediate selection.
+                            if (!algorithm.GetLocked() && request.StartTimeUtc < startUtc)
+                            {
+                                startUtc = request.StartTimeUtc;
+                            }
+                            AddSubscription(new SubscriptionRequest(request,
+                                startTimeUtc: startUtc.AddTicks(1),
+                                configuration: new SubscriptionDataConfig(request.Configuration)));
+                        }
+
+                        DataFeedSubscriptions.FreezeFillForwardResolution(false);
+                    }
+                };
+            }
+        }
+
+        #region IDataFeedSubscriptionManager
+
+        /// <summary>
+        /// Gets the data feed subscription collection
+        /// </summary>
+        public SubscriptionCollection DataFeedSubscriptions { get; }
+
+        /// <summary>
+        /// Will remove all current <see cref="Subscription"/>
+        /// </summary>
+        public void RemoveAllSubscriptions()
+        {
+            // remove each subscription from our collection
+            foreach (var subscription in DataFeedSubscriptions)
+            {
+                try
+                {
+                    RemoveSubscription(subscription.Configuration);
+                }
+                catch (Exception err)
+                {
+                    Log.Error(err, "DataManager.RemoveAllSubscriptions():" +
+                        $"Error removing: {subscription.Configuration}");
+                }
+            }
+        }
+
+        /// <summary>
+        /// Adds a new <see cref="Subscription"/> to provide data for the specified security.
+        /// </summary>
+        /// <param name="request">Defines the <see cref="SubscriptionRequest"/> to be added</param>
+        /// <returns>True if the subscription was created and added successfully, false otherwise</returns>
+        public bool AddSubscription(SubscriptionRequest request)
+        {
+            lock (_subscriptionManagerSubscriptions)
+            {
+                // guarantee the configuration is present in our config collection
+                // this is related to GH issue 3877: where we added a configuration which we also removed
+                if (_subscriptionManagerSubscriptions.TryAdd(request.Configuration, request.Configuration))
+                {
+                    _subscriptionDataConfigsEnumerator = null;
+                }
+            }
+
+            Subscription subscription;
+            if (DataFeedSubscriptions.TryGetValue(request.Configuration, out subscription))
+            {
+                if (!subscription.EndOfStream)
+                {
+                    // duplicate subscription request
+                    subscription.AddSubscriptionRequest(request);
+                    // only result true if the existing subscription is internal, we actually added something from the users perspective
+                    return subscription.Configuration.IsInternalFeed;
+                }
+                DataFeedSubscriptions.TryRemove(request.Configuration, out _);
+            }
+
+            if (request.Configuration.DataNormalizationMode == DataNormalizationMode.ScaledRaw)
+            {
+                throw new InvalidOperationException($"{DataNormalizationMode.ScaledRaw} normalization mode only intended for history requests.");
+            }
+
+            // before adding the configuration to the data feed let's assert it's valid
+            _dataPermissionManager.AssertConfiguration(request.Configuration, request.StartTimeLocal, request.EndTimeLocal);
+
+            subscription = _dataFeed.CreateSubscription(request);
+
+            if (subscription == null)
+            {
+                Log.Trace($"DataManager.AddSubscription(): Unable to add subscription for: {request.Configuration}");
+                // subscription will be null when there's no tradeable dates for the security between the requested times, so
+                // don't even try to load the data
+                return false;
+            }
+
+            if (_liveMode)
+            {
+                OnSubscriptionAdded(subscription);
+                Log.Trace($"DataManager.AddSubscription(): Added {request.Configuration}." +
+                    $" Start: {request.StartTimeUtc}. End: {request.EndTimeUtc}");
+            }
+            else if (Log.DebuggingEnabled)
+            {
+                // for performance lets not create the message string if debugging is not enabled
+                // this can be executed many times and its in the algorithm thread
+                Log.Debug($"DataManager.AddSubscription(): Added {request.Configuration}." +
+                    $" Start: {request.StartTimeUtc}. End: {request.EndTimeUtc}");
+            }
+
+            return DataFeedSubscriptions.TryAdd(subscription);
+        }
+
+        /// <summary>
+        /// Removes the <see cref="Subscription"/>, if it exists
+        /// </summary>
+        /// <param name="configuration">The <see cref="SubscriptionDataConfig"/> of the subscription to remove</param>
+        /// <param name="universe">Universe requesting to remove <see cref="Subscription"/>.
+        /// Default value, null, will remove all universes</param>
+        /// <returns>True if the subscription was successfully removed, false otherwise</returns>
+        public bool RemoveSubscription(SubscriptionDataConfig configuration, Universe universe = null)
+        {
+            return RemoveSubscriptionInternal(configuration, universe, forceSubscriptionRemoval: false);
+        }
+
+        /// <summary>
+        /// Removes the <see cref="Subscription"/>, if it exists
+        /// </summary>
+        /// <param name="configuration">The <see cref="SubscriptionDataConfig"/> of the subscription to remove</param>
+        /// <param name="universe">Universe requesting to remove <see cref="Subscription"/>.
+        /// Default value, null, will remove all universes</param>
+        /// <param name="forceSubscriptionRemoval">We force the subscription removal by marking it as removed from universe, so that all it's data is dropped</param>
+        /// <returns>True if the subscription was successfully removed, false otherwise</returns>
+        private bool RemoveSubscriptionInternal(SubscriptionDataConfig configuration, Universe universe, bool forceSubscriptionRemoval)
+        {
+            // remove the subscription from our collection, if it exists
+            Subscription subscription;
+
+            if (DataFeedSubscriptions.TryGetValue(configuration, out subscription))
+            {
+                // we remove the subscription when there are no other requests left
+                if (subscription.RemoveSubscriptionRequest(universe))
+                {
+                    if (!DataFeedSubscriptions.TryRemove(configuration, out subscription))
+                    {
+                        Log.Error($"DataManager.RemoveSubscription(): Unable to remove {configuration}");
+                        return false;
+                    }
+
+                    _dataFeed.RemoveSubscription(subscription);
+
+                    if (_liveMode)
+                    {
+                        OnSubscriptionRemoved(subscription);
+                    }
+
+                    subscription.Dispose();
+
+                    RemoveSubscriptionDataConfig(subscription);
+
+                    if (forceSubscriptionRemoval)
+                    {
+                        subscription.MarkAsRemovedFromUniverse();
+                    }
+
+                    if (_liveMode)
+                    {
+                        Log.Trace($"DataManager.RemoveSubscription(): Removed {configuration}");
+                    }
+                    else if (Log.DebuggingEnabled)
+                    {
+                        // for performance lets not create the message string if debugging is not enabled
+                        // this can be executed many times and its in the algorithm thread
+                        Log.Debug($"DataManager.RemoveSubscription(): Removed {configuration}");
+                    }
+                    return true;
+                }
+            }
+            else if (universe != null)
+            {
+                // a universe requested removal of a subscription which wasn't present anymore, this can happen when a subscription ends
+                // it will get removed from the data feed subscription list, but the configuration will remain until the universe removes it
+                // why? the effect I found is that the fill models are using these subscriptions to determine which data they could use
+                lock (_subscriptionManagerSubscriptions)
+                {
+                    if (_subscriptionManagerSubscriptions.Remove(configuration))
+                    {
+                        _subscriptionDataConfigsEnumerator = null;
+                    }
+                }
+                return true;
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Event invocator for the <see cref="SubscriptionAdded"/> event
+        /// </summary>
+        /// <param name="subscription">The added subscription</param>
+        private void OnSubscriptionAdded(Subscription subscription)
+        {
+            SubscriptionAdded?.Invoke(this, subscription);
+        }
+
+        /// <summary>
+        /// Event invocator for the <see cref="SubscriptionRemoved"/> event
+        /// </summary>
+        /// <param name="subscription">The removed subscription</param>
+        private void OnSubscriptionRemoved(Subscription subscription)
+        {
+            SubscriptionRemoved?.Invoke(this, subscription);
+        }
+
+        #endregion
+
+        #region IAlgorithmSubscriptionManager
+
+        /// <summary>
+        /// Gets all the current data config subscriptions that are being processed for the SubscriptionManager
+        /// </summary>
+        public IEnumerable<SubscriptionDataConfig> SubscriptionManagerSubscriptions
+        {
+            get
+            {
+                lock (_subscriptionManagerSubscriptions)
+                {
+                    if (_subscriptionDataConfigsEnumerator == null)
+                    {
+                        _subscriptionDataConfigsEnumerator = _subscriptionManagerSubscriptions.Values.ToList();
+                    }
+                    return _subscriptionDataConfigsEnumerator;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets existing or adds new <see cref="SubscriptionDataConfig" />
+        /// </summary>
+        /// <returns>Returns the SubscriptionDataConfig instance used</returns>
+        public SubscriptionDataConfig SubscriptionManagerGetOrAdd(SubscriptionDataConfig newConfig)
+        {
+            SubscriptionDataConfig config;
+            lock (_subscriptionManagerSubscriptions)
+            {
+                if (!_subscriptionManagerSubscriptions.TryGetValue(newConfig, out config))
+                {
+                    _subscriptionManagerSubscriptions[newConfig] = config = newConfig;
+                    _subscriptionDataConfigsEnumerator = null;
+                }
+            }
+
+            // if the reference is not the same, means it was already there and we did not add anything new
+            if (!ReferenceEquals(config, newConfig))
+            {
+                // for performance lets not create the message string if debugging is not enabled
+                // this can be executed many times and its in the algorithm thread
+                if (Log.DebuggingEnabled)
+                {
+                    Log.Debug("DataManager.SubscriptionManagerGetOrAdd(): subscription already added: " + config);
+                }
+            }
+            else
+            {
+                // add the time zone to our time keeper
+                _timeKeeper.AddTimeZone(newConfig.ExchangeTimeZone);
+            }
+
+            return config;
+        }
+
+        /// <summary>
+        /// Will try to remove a <see cref="SubscriptionDataConfig"/> and update the corresponding
+        /// consumers accordingly
+        /// </summary>
+        /// <param name="subscription">The <see cref="Subscription"/> owning the configuration to remove</param>
+        private void RemoveSubscriptionDataConfig(Subscription subscription)
+        {
+            // the subscription could of ended but might still be part of the universe
+            if (subscription.RemovedFromUniverse.Value)
+            {
+                lock (_subscriptionManagerSubscriptions)
+                {
+                    if (_subscriptionManagerSubscriptions.Remove(subscription.Configuration))
+                    {
+                        _subscriptionDataConfigsEnumerator = null;
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Returns the amount of data config subscriptions processed for the SubscriptionManager
+        /// </summary>
+        public int SubscriptionManagerCount()
+        {
+            lock (_subscriptionManagerSubscriptions)
+            {
+                return _subscriptionManagerSubscriptions.Count;
+            }
+        }
+
+        #region ISubscriptionDataConfigService
+
+        /// <summary>
+        /// The different <see cref="TickType" /> each <see cref="SecurityType" /> supports
+        /// </summary>
+        public Dictionary<SecurityType, List<TickType>> AvailableDataTypes { get; }
+
+        /// <summary>
+        /// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
+        /// Can optionally pass in desired subscription data type to use.
+        /// If the config already existed will return existing instance instead
+        /// </summary>
+        public SubscriptionDataConfig Add(
+            Type dataType,
+            Symbol symbol,
+            Resolution? resolution = null,
+            bool fillForward = true,
+            bool extendedMarketHours = false,
+            bool isFilteredSubscription = true,
+            bool isInternalFeed = false,
+            bool isCustomData = false,
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
+            )
+        {
+            return Add(symbol, resolution, fillForward, extendedMarketHours, isFilteredSubscription, isInternalFeed, isCustomData,
+                new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType)) },
+                dataNormalizationMode, dataMappingMode, contractDepthOffset)
+                .First();
+        }
+
+        /// <summary>
+        /// Creates and adds a list of <see cref="SubscriptionDataConfig" /> for a given symbol and configuration.
+        /// Can optionally pass in desired subscription data types to use.
+        ///  If the config already existed will return existing instance instead
+        /// </summary>
+        public List<SubscriptionDataConfig> Add(
+            Symbol symbol,
+            Resolution? resolution = null,
+            bool fillForward = true,
+            bool extendedMarketHours = false,
+            bool isFilteredSubscription = true,
+            bool isInternalFeed = false,
+            bool isCustomData = false,
+            List<Tuple<Type, TickType>> subscriptionDataTypes = null,
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
+            )
+        {
+            var dataTypes = subscriptionDataTypes;
+            if (dataTypes == null)
+            {
+                if (symbol.SecurityType == SecurityType.Base && SecurityIdentifier.TryGetCustomDataTypeInstance(symbol.ID.Symbol, out var type))
+                {
+                    // we've detected custom data request if we find a type let's use it
+                    dataTypes = new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(type, TickType.Trade) };
+                }
+                else
+                {
+                    dataTypes = LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution ?? _algorithm.UniverseSettings.Resolution, symbol.IsCanonical());
+                }
+            }
+
+            if (!dataTypes.Any())
+            {
+                throw new ArgumentNullException(nameof(dataTypes), "At least one type needed to create new subscriptions");
+            }
+
+            var resolutionWasProvided = resolution.HasValue;
+            foreach (var typeTuple in dataTypes)
+            {
+                var baseInstance = typeTuple.Item1.GetBaseDataInstance();
+                baseInstance.Symbol = symbol;
+                if (!resolutionWasProvided)
+                {
+                    var defaultResolution = baseInstance.DefaultResolution();
+                    if (LeanData.IsCommonLeanDataType(typeTuple.Item1))
+                    {
+                        var res = _algorithm.UniverseSettings.Resolution;
+
+                        if (!_liveMode && !baseInstance.SupportedResolutions().Contains(res))
+                        {
+                            if (!_unsupportedUniverseSettingsResolutionWarningSent)
+                            {
+                                _algorithm.Log($"Warning: Resolution {_algorithm.UniverseSettings.Resolution} for {symbol} and type {typeTuple.Item1} is not supported. " +
+                                    $"The data type default resolution '{defaultResolution}' will be used instead");
+                                _unsupportedUniverseSettingsResolutionWarningSent = true;
+                            }
+                        }
+                        else
+                        {
+                            defaultResolution = res;
+                        }
+                    }
+
+                    if (resolution.HasValue && resolution != defaultResolution)
+                    {
+                        // we are here because there are multiple 'dataTypes'.
+                        // if we get different default resolutions lets throw, this shouldn't happen
+                        throw new InvalidOperationException(
+                            $"Different data types ({string.Join(",", dataTypes.Select(tuple => tuple.Item1))})" +
+                            $" provided different default resolutions {defaultResolution} and {resolution}, this is an unexpected invalid operation.");
+                    }
+                    resolution = defaultResolution;
+                }
+                else
+                {
+                    // only assert resolution in backtesting, live can use other data source
+                    // for example daily data for options
+                    if (!_liveMode)
+                    {
+                        var supportedResolutions = baseInstance.SupportedResolutions();
+                        if (!supportedResolutions.Contains(resolution.Value))
+                        {
+                            throw new ArgumentException($"Sorry {resolution.ToStringInvariant()} is not a supported resolution for {typeTuple.Item1.Name}" +
+                                                        $" and SecurityType.{symbol.SecurityType.ToStringInvariant()}." +
+                                                        $" Please change your AddData to use one of the supported resolutions ({string.Join(",", supportedResolutions)}).");
+                        }
+                    }
+                }
+            }
+            var marketHoursDbEntry = _marketHoursDatabase.GetEntry(symbol, dataTypes.Select(tuple => tuple.Item1));
+
+            var exchangeHours = marketHoursDbEntry.ExchangeHours;
+            if (symbol.ID.SecurityType.IsOption() ||
+                symbol.ID.SecurityType == SecurityType.Index)
+            {
+                dataNormalizationMode = DataNormalizationMode.Raw;
+            }
+
+            if (marketHoursDbEntry.DataTimeZone == null)
+            {
+                throw new ArgumentNullException(nameof(marketHoursDbEntry.DataTimeZone),
+                    "DataTimeZone is a required parameter for new subscriptions. Set to the time zone the raw data is time stamped in.");
+            }
+
+            if (exchangeHours.TimeZone == null)
+            {
+                throw new ArgumentNullException(nameof(exchangeHours.TimeZone),
+                    "ExchangeTimeZone is a required parameter for new subscriptions. Set to the time zone the security exchange resides in.");
+            }
+
+            var result = (from subscriptionDataType in dataTypes
+                          let dataType = subscriptionDataType.Item1
+                          let tickType = subscriptionDataType.Item2
+                          select new SubscriptionDataConfig(
+                              dataType,
+                              symbol,
+                              resolution.Value,
+                              marketHoursDbEntry.DataTimeZone,
+                              exchangeHours.TimeZone,
+                              fillForward,
+                              extendedMarketHours,
+                              // if the subscription data types were not provided and the tick type is OpenInterest we make it internal
+                              subscriptionDataTypes == null && tickType == TickType.OpenInterest || isInternalFeed,
+                              isCustomData,
+                              isFilteredSubscription: isFilteredSubscription,
+                              tickType: tickType,
+                              dataNormalizationMode: dataNormalizationMode,
+                              dataMappingMode: dataMappingMode,
+                              contractDepthOffset: contractDepthOffset)).ToList();
+
+            for (int i = 0; i < result.Count; i++)
+            {
+                result[i] = SubscriptionManagerGetOrAdd(result[i]);
+
+                // track all registered data types
+                _registeredTypesProvider.RegisterType(result[i].Type);
+            }
+            return result;
+        }
+
+        /// <summary>
+        /// Get the data feed types for a given <see cref="SecurityType" /> <see cref="Resolution" />
+        /// </summary>
+        /// <param name="symbolSecurityType">The <see cref="SecurityType" /> used to determine the types</param>
+        /// <param name="resolution">The resolution of the data requested</param>
+        /// <param name="isCanonical">Indicates whether the security is Canonical (future and options)</param>
+        /// <returns>Types that should be added to the <see cref="SubscriptionDataConfig" /></returns>
+        /// <remarks>TODO: data type additions are very related to ticktype and should be more generic/independent of each other</remarks>
+        public List<Tuple<Type, TickType>> LookupSubscriptionConfigDataTypes(
+            SecurityType symbolSecurityType,
+            Resolution resolution,
+            bool isCanonical
+            )
+        {
+            if (isCanonical)
+            {
+                if (symbolSecurityType.IsOption())
+                {
+                    return new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(typeof(OptionUniverse), TickType.Quote) };
+                }
+
+                return new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(typeof(FutureUniverse), TickType.Quote) };
+            }
+
+            IEnumerable<TickType> availableDataType = AvailableDataTypes[symbolSecurityType]
+                // Equities will only look for trades in case of low resolutions.
+                .Where(tickType => LeanData.IsValidConfiguration(symbolSecurityType, resolution, tickType));
+
+            var result = availableDataType
+                .Select(tickType => new Tuple<Type, TickType>(LeanData.GetDataType(resolution, tickType), tickType)).ToList();
+
+            if (symbolSecurityType == SecurityType.CryptoFuture)
+            {
+                result.Add(new Tuple<Type, TickType>(typeof(MarginInterestRate), TickType.Quote));
+            }
+            return result;
+        }
+
+        /// <summary>
+        /// Gets a list of all registered <see cref="SubscriptionDataConfig"/> for a given <see cref="Symbol"/>
+        /// </summary>
+        /// <remarks>Will not return internal subscriptions by default</remarks>
+        public List<SubscriptionDataConfig> GetSubscriptionDataConfigs(Symbol symbol = null, bool includeInternalConfigs = false)
+        {
+            lock (_subscriptionManagerSubscriptions)
+            {
+                return _subscriptionManagerSubscriptions.Keys
+                    .Where(config => (includeInternalConfigs || !config.IsInternalFeed) && (symbol == null || config.Symbol.ID == symbol.ID))
+                    .OrderBy(config => config.IsInternalFeed)
+                    .ToList();
+            }
+        }
+
+        #endregion
+
+        #endregion
+
+        #region IDataManager
+
+        /// <summary>
+        /// Get the universe selection instance
+        /// </summary>
+        public UniverseSelection UniverseSelection { get; }
+
+        #endregion
+    }
+}

Engine/DataFeeds/DataPermissionManager.cs

@@ -0,0 +1,67 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Packets;
+using QuantConnect.Interfaces;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Entity in charge of handling data permissions
+    /// </summary>
+    public class DataPermissionManager : IDataPermissionManager
+    {
+        /// <summary>
+        /// The data channel provider instance
+        /// </summary>
+        public IDataChannelProvider DataChannelProvider { get; private set; }
+
+        [DynamicDependency(DynamicallyAccessedMemberTypes.PublicConstructors, typeof(DataPermissionManager))]
+        public DataPermissionManager()
+        {
+        }
+
+        /// <summary>
+        /// Initialize the data permission manager
+        /// </summary>
+        /// <param name="job">The job packet</param>
+        public virtual void Initialize(AlgorithmNodePacket job)
+        {
+            var liveJob = job as LiveNodePacket;
+            if (liveJob != null)
+            {
+                Log.Trace($"LiveTradingDataFeed.GetDataChannelProvider(): will use {liveJob.DataChannelProvider}");
+                DataChannelProvider = Composer.Instance.GetExportedValueByTypeName<IDataChannelProvider>(liveJob.DataChannelProvider);
+                DataChannelProvider.Initialize(liveJob);
+            }
+        }
+
+        /// <summary>
+        /// Will assert the requested configuration is valid for the current job
+        /// </summary>
+        /// <param name="subscriptionRequest">The data subscription configuration to assert</param>
+        /// <param name="startTimeLocal">The start time of this request</param>
+        /// <param name="endTimeLocal">The end time of this request</param>
+        public virtual void AssertConfiguration(SubscriptionDataConfig subscriptionRequest, DateTime startTimeLocal, DateTime endTimeLocal)
+        {
+        }
+    }
+}

Engine/DataFeeds/DataQueueHandlerManager.cs

@@ -0,0 +1,274 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using QuantConnect.Packets;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// This is an implementation of <see cref="IDataQueueHandler"/> used to handle multiple live datafeeds
+    /// </summary>
+    public class DataQueueHandlerManager : IDataQueueHandler, IDataQueueUniverseProvider
+    {
+        private readonly IAlgorithmSettings _algorithmSettings;
+        private readonly Dictionary<SubscriptionDataConfig, Queue<IDataQueueHandler>> _dataConfigAndDataHandler = new();
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public DataQueueHandlerManager(IAlgorithmSettings settings)
+        {
+            _algorithmSettings = settings;
+        }
+
+        /// <summary>
+        /// Frontier time provider to use
+        /// </summary>
+        /// <remarks>Protected for testing purposes</remarks>
+        protected ITimeProvider FrontierTimeProvider { get; set; }
+
+        /// <summary>
+        /// Collection of data queue handles being used
+        /// </summary>
+        /// <remarks>Protected for testing purposes</remarks>
+        protected List<IDataQueueHandler> DataHandlers { get; set; } = new();
+
+        /// <summary>
+        /// True if the composite queue handler has any <see cref="IDataQueueUniverseProvider"/> instance
+        /// </summary>
+        public bool HasUniverseProvider => DataHandlers.OfType<IDataQueueUniverseProvider>().Any();
+
+        /// <summary>
+        /// Event triggered when an unsupported configuration is detected
+        /// </summary>
+        public event EventHandler<SubscriptionDataConfig> UnsupportedConfiguration;
+
+        /// <summary>
+        /// Subscribe to the specified configuration
+        /// </summary>
+        /// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
+        /// <param name="newDataAvailableHandler">handler to be fired on new data available</param>
+        /// <returns>The new enumerator for this subscription request</returns>
+        public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
+        {
+            Exception failureException = null;
+            foreach (var dataHandler in DataHandlers)
+            {
+                // Emit ticks & custom data as soon as we get them, they don't need any kind of batching behavior applied to them
+                // only use the frontier time provider if we need to
+                var immediateEmission = dataConfig.Resolution == Resolution.Tick || dataConfig.IsCustomData || FrontierTimeProvider == null;
+                var exchangeTimeZone = dataConfig.ExchangeTimeZone;
+
+                IEnumerator<BaseData> enumerator;
+                try
+                {
+                    enumerator = dataHandler.Subscribe(dataConfig, immediateEmission ? newDataAvailableHandler
+                        : (sender, eventArgs) => {
+                            // let's only wake up the main thread if the data point is allowed to be emitted, else we could fill forward previous bar and not let this one through
+                            var dataAvailable = eventArgs as NewDataAvailableEventArgs;
+                            if (dataAvailable == null || dataAvailable.DataPoint == null
+                                || dataAvailable.DataPoint.EndTime.ConvertToUtc(exchangeTimeZone) <= FrontierTimeProvider.GetUtcNow())
+                            {
+                                newDataAvailableHandler?.Invoke(sender, eventArgs);
+                            }
+                        });
+                }
+                catch (Exception exception)
+                {
+                    // we will try the next DQH if any, if it handles the request correctly we ignore the error
+                    failureException = exception;
+                    continue;
+                }
+
+                // Check if the enumerator is not empty
+                if (enumerator != null)
+                {
+                    if (!_dataConfigAndDataHandler.TryGetValue(dataConfig, out var dataQueueHandlers))
+                    {
+                        // we can get the same subscription request multiple times, the aggregator manager handles updating each enumerator
+                        // but we need to keep track so we can call unsubscribe later to the target data queue handler
+                        _dataConfigAndDataHandler[dataConfig] = dataQueueHandlers = new Queue<IDataQueueHandler>();
+                    }
+                    dataQueueHandlers.Enqueue(dataHandler);
+
+                    if (immediateEmission)
+                    {
+                        return enumerator;
+                    }
+
+                    var utcStartTime = FrontierTimeProvider.GetUtcNow();
+
+                    var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(dataConfig.Symbol.ID.Market, dataConfig.Symbol, dataConfig.Symbol.SecurityType);
+                    if (LeanData.UseStrictEndTime(_algorithmSettings.DailyPreciseEndTime, dataConfig.Symbol, dataConfig.Increment, exchangeHours))
+                    {
+                        // before the first frontier enumerator we adjust the endtimes if required
+                        enumerator = new StrictDailyEndTimesEnumerator(enumerator, exchangeHours, utcStartTime.ConvertFromUtc(exchangeTimeZone));
+                    }
+
+                    return new FrontierAwareEnumerator(enumerator, FrontierTimeProvider,
+                        new TimeZoneOffsetProvider(exchangeTimeZone, utcStartTime, Time.EndOfTime)
+                    );
+                }
+            }
+
+            if (failureException != null)
+            {
+                // we were not able to serve the request with any DQH and we got an exception, let's bubble it up
+                throw failureException;
+            }
+
+            // filter out warning for expected cases to reduce noise
+            if (!dataConfig.Symbol.Value.Contains("-UNIVERSE-", StringComparison.InvariantCultureIgnoreCase)
+                && dataConfig.Type != typeof(Delisting)
+                && !dataConfig.Symbol.IsCanonical())
+            {
+                UnsupportedConfiguration?.Invoke(this, dataConfig);
+            }
+            return null;
+        }
+
+        /// <summary>
+        /// Removes the specified configuration
+        /// </summary>
+        /// <param name="dataConfig">Subscription config to be removed</param>
+        public virtual void Unsubscribe(SubscriptionDataConfig dataConfig)
+        {
+            if (_dataConfigAndDataHandler.TryGetValue(dataConfig, out var dataHandlers))
+            {
+                var dataHandler = dataHandlers.Dequeue();
+                dataHandler.Unsubscribe(dataConfig);
+
+                if (dataHandlers.Count == 0)
+                {
+                    // nothing left
+                    _dataConfigAndDataHandler.Remove(dataConfig);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Sets the job we're subscribing for
+        /// </summary>
+        /// <param name="job">Job we're subscribing for</param>
+        public void SetJob(LiveNodePacket job)
+        {
+            var dataHandlersConfig = job.DataQueueHandler;
+            Log.Trace($"CompositeDataQueueHandler.SetJob(): will use {dataHandlersConfig}");
+            foreach (var dataHandlerName in dataHandlersConfig.DeserializeList())
+            {
+                var dataHandler = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(dataHandlerName);
+                dataHandler.SetJob(job);
+                DataHandlers.Add(dataHandler);
+            }
+
+            FrontierTimeProvider = InitializeFrontierTimeProvider();
+        }
+
+        /// <summary>
+        /// Returns whether the data provider is connected
+        /// </summary>
+        /// <returns>true if the data provider is connected</returns>
+        public bool IsConnected => true;
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            foreach (var dataHandler in DataHandlers)
+            {
+                dataHandler.Dispose();
+            }
+        }
+
+        /// <summary>
+        /// Method returns a collection of Symbols that are available at the data source.
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup</param>
+        /// <param name="includeExpired">Include expired contracts</param>
+        /// <param name="securityCurrency">Expected security currency(if any)</param>
+        /// <returns>Enumerable of Symbols, that are associated with the provided Symbol</returns>
+        public IEnumerable<Symbol> LookupSymbols(Symbol symbol, bool includeExpired, string securityCurrency = null)
+        {
+            foreach (var dataHandler in GetUniverseProviders())
+            {
+                var symbols = dataHandler.LookupSymbols(symbol, includeExpired, securityCurrency);
+                if (symbols == null)
+                {
+                    // the universe provider does not support it
+                    continue;
+                }
+
+                var result = symbols.ToList();
+                if (result.Any())
+                {
+                    return result;
+                }
+            }
+            return Enumerable.Empty<Symbol>();
+        }
+
+        /// <summary>
+        /// Returns whether selection can take place or not.
+        /// </summary>
+        /// <remarks>This is useful to avoid a selection taking place during invalid times, for example IB reset times or when not connected,
+        /// because if allowed selection would fail since IB isn't running and would kill the algorithm</remarks>
+        /// <returns>True if selection can take place</returns>
+        public bool CanPerformSelection()
+        {
+            return GetUniverseProviders().Any(provider => provider.CanPerformSelection());
+        }
+
+        /// <summary>
+        /// Creates the frontier time provider instance
+        /// </summary>
+        /// <remarks>Protected for testing purposes</remarks>
+        protected virtual ITimeProvider InitializeFrontierTimeProvider()
+        {
+            var timeProviders = DataHandlers.OfType<ITimeProvider>().ToList();
+            if (timeProviders.Any())
+            {
+                Log.Trace($"DataQueueHandlerManager.InitializeFrontierTimeProvider(): will use the following IDQH frontier time providers: [{string.Join(",", timeProviders.Select(x => x.GetType()))}]");
+                return new CompositeTimeProvider(timeProviders);
+            }
+            return null;
+        }
+
+        private IEnumerable<IDataQueueUniverseProvider> GetUniverseProviders()
+        {
+            var yielded = false;
+            foreach (var universeProvider in DataHandlers.OfType<IDataQueueUniverseProvider>())
+            {
+                yielded = true;
+                yield return universeProvider;
+            }
+
+            if (!yielded)
+            {
+                throw new NotSupportedException("The DataQueueHandler does not support Options and Futures.");
+            }
+        }
+    }
+}

Engine/DataFeeds/DateChangeTimeKeeper.cs

@@ -0,0 +1,332 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using System.Collections.Generic;
+using QuantConnect.Securities;
+using System.Runtime.CompilerServices;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Time keeper specialization to keep time for a subscription both in data and exchange time zones.
+    /// It also emits events when the exchange date changes, which is useful to emit date change events
+    /// required for some daily actions like mapping symbols, delistings, splits, etc.
+    /// </summary>
+    internal class DateChangeTimeKeeper : TimeKeeper, IDisposable
+    {
+        private IEnumerator<DateTime> _tradableDatesInDataTimeZone;
+        private SubscriptionDataConfig _config;
+        private SecurityExchangeHours _exchangeHours;
+        private DateTime _delistingDate;
+
+        private DateTime _previousNewExchangeDate;
+
+        private bool _needsMoveNext;
+        private bool _initialized;
+
+        private DateTime _exchangeTime;
+        private DateTime _dataTime;
+        private bool _exchangeTimeNeedsUpdate;
+        private bool _dataTimeNeedsUpdate;
+
+        /// <summary>
+        /// The current time in the data time zone
+        /// </summary>
+        public DateTime DataTime
+        {
+            get
+            {
+                if (_dataTimeNeedsUpdate)
+                {
+                    _dataTime = GetTimeIn(_config.DataTimeZone);
+                    _dataTimeNeedsUpdate = false;
+                }
+                return _dataTime;
+            }
+        }
+
+        /// <summary>
+        /// The current time in the exchange time zone
+        /// </summary>
+        public DateTime ExchangeTime
+        {
+            get
+            {
+                if (_exchangeTimeNeedsUpdate)
+                {
+                    _exchangeTime = GetTimeIn(_exchangeHours.TimeZone);
+                    _exchangeTimeNeedsUpdate = false;
+                }
+                return _exchangeTime;
+            }
+        }
+
+        /// <summary>
+        /// Event that fires every time the exchange date changes
+        /// </summary>
+        public event EventHandler<DateTime> NewExchangeDate;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="DateChangeTimeKeeper"/> class
+        /// </summary>
+        /// <param name="tradableDatesInDataTimeZone">The tradable dates in data time zone</param>
+        /// <param name="config">The subscription data configuration this instance will keep track of time for</param>
+        /// <param name="exchangeHours">The exchange hours</param>
+        /// <param name="delistingDate">The symbol's delisting date</param>
+        public DateChangeTimeKeeper(IEnumerable<DateTime> tradableDatesInDataTimeZone, SubscriptionDataConfig config,
+            SecurityExchangeHours exchangeHours, DateTime delistingDate)
+            : base(Time.BeginningOfTime, new[] { config.DataTimeZone, config.ExchangeTimeZone })
+        {
+            _tradableDatesInDataTimeZone = tradableDatesInDataTimeZone.GetEnumerator();
+            _config = config;
+            _exchangeHours = exchangeHours;
+            _delistingDate = delistingDate;
+            _exchangeTimeNeedsUpdate = true;
+            _dataTimeNeedsUpdate = true;
+            _needsMoveNext = true;
+        }
+
+        /// <summary>
+        /// Disposes the resources
+        /// </summary>
+        public void Dispose()
+        {
+            _tradableDatesInDataTimeZone.DisposeSafely();
+        }
+
+        /// <summary>
+        /// Sets the current UTC time for this time keeper
+        /// </summary>
+        /// <param name="utcDateTime">The current time in UTC</param>
+        public override void SetUtcDateTime(DateTime utcDateTime)
+        {
+            base.SetUtcDateTime(utcDateTime);
+            _exchangeTimeNeedsUpdate = true;
+            _dataTimeNeedsUpdate = true;
+        }
+
+        /// <summary>
+        /// Advances the time keeper towards the target exchange time.
+        /// If an exchange date is found before the target time, it is emitted and the time keeper is set to that date.
+        /// The caller must check whether the target time was reached or if the time keeper was set to a new exchange date before the target time.
+        /// </summary>
+        public void AdvanceTowardsExchangeTime(DateTime targetExchangeTime)
+        {
+            if (!_initialized)
+            {
+                throw new InvalidOperationException($"The time keeper has not been initialized. " +
+                    $"{nameof(TryAdvanceUntilNextDataDate)} needs to be called at least once to flush the first date before advancing.");
+            }
+
+            var currentExchangeTime = ExchangeTime;
+            // Advancing within the same exchange date, just update the time, no new exchange date will be emitted
+            if (targetExchangeTime.Date == currentExchangeTime.Date)
+            {
+                SetExchangeTime(targetExchangeTime);
+                return;
+            }
+
+            while (currentExchangeTime < targetExchangeTime)
+            {
+                var newExchangeTime = currentExchangeTime + Time.OneDay;
+                if (newExchangeTime > targetExchangeTime)
+                {
+                    newExchangeTime = targetExchangeTime;
+                }
+
+                var newExchangeDate = newExchangeTime.Date;
+
+                // We found a new exchange date before the target time, emit it first
+                if (newExchangeDate != currentExchangeTime.Date &&
+                    _exchangeHours.IsDateOpen(newExchangeDate, _config.ExtendedMarketHours))
+                {
+                    // Stop here, set the new exchange date
+                    SetExchangeTime(newExchangeDate);
+                    EmitNewExchangeDate(newExchangeDate);
+                    return;
+                }
+
+                currentExchangeTime = newExchangeTime;
+            }
+
+            // We reached the target time, set it
+            SetExchangeTime(targetExchangeTime);
+        }
+
+        /// <summary>
+        /// Advances the time keeper until the next data date, emitting the new exchange date if this happens before the new data date
+        /// </summary>
+        public bool TryAdvanceUntilNextDataDate()
+        {
+            if (!_initialized)
+            {
+                return EmitFirstExchangeDate();
+            }
+
+            // Before moving forward, check whether we need to emit a new exchange date
+            if (TryEmitPassedExchangeDate())
+            {
+                return true;
+            }
+
+            if (!_needsMoveNext || _tradableDatesInDataTimeZone.MoveNext())
+            {
+                var nextDataDate = _tradableDatesInDataTimeZone.Current;
+                var nextExchangeTime = nextDataDate.ConvertTo(_config.DataTimeZone, _exchangeHours.TimeZone);
+                var nextExchangeDate = nextExchangeTime.Date;
+
+                if (nextExchangeDate > _delistingDate)
+                {
+                    // We are done, but an exchange date might still need to be emitted
+                    TryEmitPassedExchangeDate();
+                    _needsMoveNext = false;
+                    return false;
+                }
+
+                // If the exchange is not behind the data, the data might have not been enough to emit the exchange date,
+                // which already passed if we are moving on to the next data date. So we need to check if we need to emit it here.
+                // e.g. moving data date from tuesday to wednesday, but the exchange date is already past the end of tuesday
+                // (by N hours, depending on the time zones offset). If data didn't trigger the exchange date change, we need to do it here.
+                if (!IsExchangeBehindData(nextExchangeTime, nextDataDate) && nextExchangeDate > _previousNewExchangeDate)
+                {
+                    EmitNewExchangeDate(nextExchangeDate);
+                    SetExchangeTime(nextExchangeDate);
+                    // nextExchangeDate == DataTime means time zones are synchronized, need to move next only when exchange is actually ahead
+                    _needsMoveNext = nextExchangeDate == DataTime;
+                    return true;
+                }
+
+                _needsMoveNext = true;
+                SetDataTime(nextDataDate);
+                return true;
+            }
+
+            _needsMoveNext = false;
+            return false;
+        }
+
+        /// <summary>
+        /// Emits the first exchange date for the algorithm so that the first daily events are triggered (mappings, delistings, etc.)
+        /// </summary>
+        /// <returns>True if the new exchange date is emitted. False if already done or the tradable dates enumerable is empty</returns>
+        private bool EmitFirstExchangeDate()
+        {
+            if (_initialized)
+            {
+                return false;
+            }
+
+            if (!_tradableDatesInDataTimeZone.MoveNext())
+            {
+                _initialized = true;
+                return false;
+            }
+
+            var firstDataDate = _tradableDatesInDataTimeZone.Current;
+            var firstExchangeTime = firstDataDate.ConvertTo(_config.DataTimeZone, _exchangeHours.TimeZone);
+            var firstExchangeDate = firstExchangeTime.Date;
+
+            DateTime exchangeDateToEmit;
+            // The exchange is ahead of the data, so we need to emit the current exchange date, which already passed
+            if (firstExchangeTime < firstDataDate && _exchangeHours.IsDateOpen(firstExchangeDate, _config.ExtendedMarketHours))
+            {
+                exchangeDateToEmit = firstExchangeDate;
+                SetExchangeTime(exchangeDateToEmit);
+                // Don't move, the current data date still needs to be consumed
+                _needsMoveNext = false;
+            }
+            // The exchange is behind of (or in sync with) data: exchange has not passed to this new date, but with emit it here
+            // so that first daily things are done (mappings, delistings, etc.)
+            else
+            {
+                exchangeDateToEmit = firstDataDate;
+                SetDataTime(firstDataDate);
+                _needsMoveNext = true;
+            }
+
+            EmitNewExchangeDate(exchangeDateToEmit);
+            _initialized = true;
+            return true;
+        }
+
+        /// <summary>
+        /// Determines whether the exchange time zone is behind the data time zone
+        /// </summary>
+        /// <returns></returns>
+        public bool IsExchangeBehindData()
+        {
+            return IsExchangeBehindData(ExchangeTime, DataTime);
+        }
+
+        /// <summary>
+        /// Determines whether the exchange time zone is behind the data time zone
+        /// </summary>
+        private static bool IsExchangeBehindData(DateTime exchangeTime, DateTime dataTime)
+        {
+            return dataTime > exchangeTime;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void SetExchangeTime(DateTime exchangeTime)
+        {
+            SetUtcDateTime(exchangeTime.ConvertToUtc(_exchangeHours.TimeZone));
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void SetDataTime(DateTime dataTime)
+        {
+            SetUtcDateTime(dataTime.ConvertToUtc(_config.DataTimeZone));
+        }
+
+        /// <summary>
+        /// Checks that before moving to the next data date, if the exchange date has already passed and has been emitted, else it emits it.
+        /// This can happen when the exchange is behind of the data. e.g We advance data date from Monday to Tuesday, then the data itself
+        /// will drive the exchange data change (N hours later, depending on the time zones offset).
+        /// But if there is no enough data or the file is not found, the new exchange date will not be emitted, so we need to do it here.
+        /// </summary>
+        private bool TryEmitPassedExchangeDate()
+        {
+            if (_needsMoveNext && _tradableDatesInDataTimeZone.Current != default)
+            {
+                // This data date passed, and it should have emitted as an exchange tradable date when detected
+                // as a date change in the data itself, if not, emit it now before moving to the next data date
+                var currentDataDate = _tradableDatesInDataTimeZone.Current;
+                if (_previousNewExchangeDate < currentDataDate &&
+                    _exchangeHours.IsDateOpen(currentDataDate, _config.ExtendedMarketHours))
+                {
+                    var nextExchangeDate = currentDataDate;
+                    SetExchangeTime(nextExchangeDate);
+                    EmitNewExchangeDate(nextExchangeDate);
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Emits a new exchange date event
+        /// </summary>
+        private void EmitNewExchangeDate(DateTime newExchangeDate)
+        {
+            NewExchangeDate?.Invoke(this, newExchangeDate);
+            _previousNewExchangeDate = newExchangeDate;
+        }
+    }
+}

Engine/DataFeeds/DefaultDataProvider.cs

@@ -0,0 +1,89 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Default file provider functionality that retrieves data from disc to be used in an algorithm
+    /// </summary>
+    public class DefaultDataProvider : IDataProvider, IDisposable
+    {
+        private bool _oneTimeWarningLog;
+
+        /// <summary>
+        /// Event raised each time data fetch is finished (successfully or not)
+        /// </summary>
+        public event EventHandler<DataProviderNewDataRequestEventArgs> NewDataRequest;
+
+        /// <summary>
+        /// Retrieves data from disc to be used in an algorithm
+        /// </summary>
+        /// <param name="key">A string representing where the data is stored</param>
+        /// <returns>A <see cref="Stream"/> of the data requested</returns>
+        public virtual Stream Fetch(string key)
+        {
+            var success = true;
+            var errorMessage = string.Empty;
+            try
+            {
+                return new FileStream(FileExtension.ToNormalizedPath(key), FileMode.Open, FileAccess.Read, FileShare.Read);
+            }
+            catch (Exception exception)
+            {
+                success = false;
+                errorMessage = exception.Message;
+                if (exception is DirectoryNotFoundException)
+                {
+                    if (!_oneTimeWarningLog)
+                    {
+                        _oneTimeWarningLog = true;
+                        Logging.Log.Debug($"DefaultDataProvider.Fetch(): DirectoryNotFoundException: please review data paths, current 'Globals.DataFolder': {Globals.DataFolder}");
+                    }
+                    return null;
+                }
+                else if (exception is FileNotFoundException)
+                {
+                    return null;
+                }
+
+                throw;
+            }
+            finally
+            {
+                OnNewDataRequest(new DataProviderNewDataRequestEventArgs(key, success, errorMessage));
+            }
+        }
+
+        /// <summary>
+        /// The stream created by this type is passed up the stack to the IStreamReader
+        /// The stream is closed when the StreamReader that wraps this stream is disposed</summary>
+        public void Dispose()
+        {
+            //
+        }
+
+        /// <summary>
+        /// Event invocator for the <see cref="NewDataRequest"/> event
+        /// </summary>
+        protected virtual void OnNewDataRequest(DataProviderNewDataRequestEventArgs e)
+        {
+            NewDataRequest?.Invoke(this, e);
+        }
+    }
+}

Engine/DataFeeds/DownloaderDataProvider.cs

@@ -0,0 +1,350 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NodaTime;
+using System.IO;
+using System.Linq;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Configuration;
+using System.Collections.Concurrent;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Lean.Engine.DataFeeds.DataDownloader;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Data provider which downloads data using an <see cref="IDataDownloader"/> or <see cref="IBrokerage"/> implementation
+    /// </summary>
+    public class DownloaderDataProvider : BaseDownloaderDataProvider
+    {
+        /// <summary>
+        /// Synchronizer in charge of guaranteeing a single operation per file path
+        /// </summary>
+        private readonly static KeyStringSynchronizer DiskSynchronizer = new();
+
+        private bool _customDataDownloadError;
+        private readonly ConcurrentDictionary<Symbol, Symbol> _marketHoursWarning = new();
+        private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
+        private readonly DataDownloaderSelector _dataDownloader;
+        private readonly IDataCacheProvider _dataCacheProvider = new DiskDataCacheProvider(DiskSynchronizer);
+        private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart<IMapFileProvider>();
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public DownloaderDataProvider()
+        {
+            var dataDownloaderConfig = Config.Get("data-downloader");
+            if (!string.IsNullOrEmpty(dataDownloaderConfig))
+            {
+                _dataDownloader = new DataDownloaderSelector(Composer.Instance.GetExportedValueByTypeName<IDataDownloader>(dataDownloaderConfig), _mapFileProvider, this);
+            }
+            else
+            {
+                throw new ArgumentException("DownloaderDataProvider(): requires 'data-downloader' to be set with a valid type name");
+            }
+        }
+
+        /// <summary>
+        /// Creates a new instance using a target data downloader used for testing
+        /// </summary>
+        public DownloaderDataProvider(IDataDownloader dataDownloader)
+        {
+            _dataDownloader = new DataDownloaderSelector(dataDownloader, _mapFileProvider, this);
+        }
+
+        /// <summary>
+        /// Determines if it should downloads new data and retrieves data from disc
+        /// </summary>
+        /// <param name="key">A string representing where the data is stored</param>
+        /// <returns>A <see cref="Stream"/> of the data requested</returns>
+        public override Stream Fetch(string key)
+        {
+            return DownloadOnce(key, s =>
+            {
+                if (LeanData.TryParsePath(key, out var symbol, out var date, out var resolution, out var tickType, out var dataType))
+                {
+                    if (symbol.SecurityType == SecurityType.Base)
+                    {
+                        if (!_customDataDownloadError)
+                        {
+                            _customDataDownloadError = true;
+                            // lean data writter doesn't support it
+                            Log.Trace($"DownloaderDataProvider.Get(): custom data is not supported, requested: {symbol}");
+                        }
+                        return;
+                    }
+
+                    MarketHoursDatabase.Entry entry;
+                    try
+                    {
+                        entry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.SecurityType);
+                    }
+                    catch
+                    {
+                        // this could happen for some sources using the data provider but with not market hours data base entry, like interest rates
+                        if (_marketHoursWarning.TryAdd(symbol, symbol))
+                        {
+                            // log once
+                            Log.Trace($"DownloaderDataProvider.Get(): failed to find market hours for {symbol}, skipping");
+                        }
+                        // this shouldn't happen for data we want can download
+                        return;
+                    }
+
+                    var dataTimeZone = entry.DataTimeZone;
+                    var exchangeTimeZone = entry.ExchangeHours.TimeZone;
+                    DateTime startTimeUtc;
+                    DateTime endTimeUtc;
+                    // we will download until yesterday so we are sure we don't get partial data
+                    var endTimeUtcLimit = DateTime.UtcNow.Date.AddDays(-1);
+                    if (resolution < Resolution.Hour)
+                    {
+                        // we can get the date from the path
+                        startTimeUtc = date.ConvertToUtc(dataTimeZone);
+                        // let's get the whole day
+                        endTimeUtc = date.AddDays(1).ConvertToUtc(dataTimeZone);
+                        if (endTimeUtc > endTimeUtcLimit)
+                        {
+                            // we are at the limit, avoid getting partial data
+                            return;
+                        }
+                    }
+                    else
+                    {
+                        // since hourly & daily are a single file we fetch the whole file
+                        endTimeUtc = endTimeUtcLimit;
+                        try
+                        {
+                            // we don't really know when Futures, FutureOptions, Cryptos, etc, start date so let's give it a good guess
+                            if (symbol.SecurityType == SecurityType.Crypto)
+                            {
+                                // bitcoin start
+                                startTimeUtc = new DateTime(2009, 1, 1);
+                            }
+                            else if (symbol.SecurityType.IsOption() && symbol.SecurityType != SecurityType.FutureOption)
+                            {
+                                // For options, an hourly or daily file contains a year of data, so we need to get the year of the date
+                                startTimeUtc = new DateTime(date.Year, 1, 1);
+                                endTimeUtc = startTimeUtc.AddYears(1);
+                            }
+                            else
+                            {
+                                startTimeUtc = symbol.ID.Date;
+                            }
+                        }
+                        catch (InvalidOperationException)
+                        {
+                            startTimeUtc = Time.Start;
+                        }
+
+                        if (startTimeUtc < Time.Start)
+                        {
+                            startTimeUtc = Time.Start;
+                        }
+
+                        if (endTimeUtc > endTimeUtcLimit)
+                        {
+                            endTimeUtc = endTimeUtcLimit;
+                        }
+                    }
+
+                    try
+                    {
+                        if (dataType == typeof(OptionUniverse))
+                        {
+                            var processingDate = date.ConvertToUtc(dataTimeZone);
+                            UniverseExtensions.RunUniverseDownloader(_dataDownloader.GetDataDownloader(dataType), new DataUniverseDownloaderGetParameters(symbol, processingDate, processingDate.AddDays(1), entry.ExchangeHours));
+                            return;
+                        }
+
+                        LeanDataWriter writer = null;
+                        var getParams = new DataDownloaderGetParameters(symbol, resolution, startTimeUtc, endTimeUtc, tickType);
+
+                        var downloaderDataParameters = getParams.GetDataDownloaderParameterForAllMappedSymbols(_mapFileProvider, exchangeTimeZone);
+
+                        var downloadedData = GetDownloadedData(downloaderDataParameters, symbol, exchangeTimeZone, dataTimeZone, dataType);
+
+                        foreach (var dataPerSymbol in downloadedData)
+                        {
+                            if (writer == null)
+                            {
+                                writer = new LeanDataWriter(resolution, symbol, Globals.DataFolder, tickType, mapSymbol: true, dataCacheProvider: _dataCacheProvider);
+                            }
+                            // Save the data
+                            writer.Write(dataPerSymbol);
+                        }
+                    }
+                    catch (Exception e)
+                    {
+                        Log.Error(e);
+                    }
+                }
+            });
+        }
+
+        /// <summary>
+        /// Retrieves downloaded data grouped by symbol based on <see cref="IDownloadProvider"/>.
+        /// </summary>
+        /// <param name="downloaderDataParameters">Parameters specifying the data to be retrieved.</param>
+        /// <param name="symbol">Represents a unique security identifier, generate by ticker name.</param>
+        /// <param name="exchangeTimeZone">The time zone of the exchange where the symbol is traded.</param>
+        /// <param name="dataTimeZone">The time zone in which the data is represented.</param>
+        /// <param name="dataType">The type of data to be retrieved. (e.g. <see cref="Data.Market.TradeBar"/>)</param>
+        /// <returns>An IEnumerable containing groups of data grouped by symbol. Each group contains data related to a specific symbol.</returns>
+        /// <exception cref="ArgumentException"> Thrown when the downloaderDataParameters collection is null or empty.</exception>
+        public IEnumerable<IGrouping<Symbol, BaseData>> GetDownloadedData(
+            IEnumerable<DataDownloaderGetParameters> downloaderDataParameters,
+            Symbol symbol,
+            DateTimeZone exchangeTimeZone,
+            DateTimeZone dataTimeZone,
+            Type dataType)
+        {
+            if (downloaderDataParameters.IsNullOrEmpty())
+            {
+                throw new ArgumentException($"{nameof(DownloaderDataProvider)}.{nameof(GetDownloadedData)}: DataDownloaderGetParameters are empty or equal to null.");
+            }
+
+            foreach (var downloaderDataParameter in downloaderDataParameters)
+            {
+                var downloadedData = _dataDownloader.GetDataDownloader(dataType).Get(downloaderDataParameter);
+
+                if (downloadedData == null)
+                {
+                    // doesn't support this download request, that's okay
+                    continue;
+                }
+
+                var groupedData = FilterAndGroupDownloadDataBySymbol(
+                    downloadedData,
+                    symbol,
+                    dataType,
+                    exchangeTimeZone,
+                    dataTimeZone,
+                    downloaderDataParameter.StartUtc,
+                    downloaderDataParameter.EndUtc);
+
+                foreach (var data in groupedData)
+                {
+                    yield return data;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Get's the stream for a given file path
+        /// </summary>
+        protected override Stream GetStream(string key)
+        {
+            if (LeanData.TryParsePath(key, out var symbol, out var date, out var resolution, out var _) && resolution > Resolution.Minute && symbol.RequiresMapping())
+            {
+                // because the file could be updated even after it's created because of symbol mapping we can't stream from disk
+                return DiskSynchronizer.Execute(key, () =>
+                {
+                    var baseStream = base.Fetch(key);
+                    if (baseStream != null)
+                    {
+                        var result = new MemoryStream();
+                        baseStream.CopyTo(result);
+                        baseStream.Dispose();
+                        // move position back to the start
+                        result.Position = 0;
+
+                        return result;
+                    }
+                    return null;
+                });
+            }
+
+            return base.Fetch(key);
+        }
+
+        /// <summary>
+        /// Main filter to determine if this file needs to be downloaded
+        /// </summary>
+        /// <param name="filePath">File we are looking at</param>
+        /// <returns>True if should download</returns>
+        protected override bool NeedToDownload(string filePath)
+        {
+            // Ignore null and invalid data requests
+            if (filePath == null
+                || filePath.Contains("fine", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains("fundamental", StringComparison.InvariantCultureIgnoreCase)
+                || filePath.Contains("map_files", StringComparison.InvariantCultureIgnoreCase)
+                || filePath.Contains("factor_files", StringComparison.InvariantCultureIgnoreCase)
+                || filePath.Contains("margins", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains("future", StringComparison.InvariantCultureIgnoreCase))
+            {
+                return false;
+            }
+
+            // Only download if it doesn't exist or is out of date.
+            // Files are only "out of date" for non date based files (hour, daily, margins, etc.) because this data is stored all in one file
+            return !File.Exists(filePath) || filePath.IsOutOfDate();
+        }
+
+        /// <summary>
+        /// Filters and groups the provided download data by symbol, based on specified criteria.
+        /// </summary>
+        /// <param name="downloadData">The collection of download data to process.</param>
+        /// <param name="symbol">The symbol to filter the data for.</param>
+        /// <param name="dataType">The type of data to filter for.</param>
+        /// <param name="exchangeTimeZone">The time zone of the exchange.</param>
+        /// <param name="dataTimeZone">The desired time zone for the data.</param>
+        /// <param name="downloaderStartTimeUtc">The start time of data downloading in UTC.</param>
+        /// <param name="downloaderEndTimeUtc">The end time of data downloading in UTC.</param>
+        /// <returns>
+        /// An enumerable collection of groupings of download data, grouped by symbol.
+        /// </returns>
+        public static IEnumerable<IGrouping<Symbol, BaseData>> FilterAndGroupDownloadDataBySymbol(
+            IEnumerable<BaseData> downloadData,
+            Symbol symbol,
+            Type dataType,
+            DateTimeZone exchangeTimeZone,
+            DateTimeZone dataTimeZone,
+            DateTime downloaderStartTimeUtc,
+            DateTime downloaderEndTimeUtc)
+        {
+            var startDateTimeInExchangeTimeZone = downloaderStartTimeUtc.ConvertFromUtc(exchangeTimeZone);
+            var endDateTimeInExchangeTimeZone = downloaderEndTimeUtc.ConvertFromUtc(exchangeTimeZone);
+
+            return downloadData
+                .Where(baseData =>
+                {
+                    // Sometimes, external Downloader provider returns excess data
+                    if (baseData.Time < startDateTimeInExchangeTimeZone || baseData.Time > endDateTimeInExchangeTimeZone)
+                    {
+                        return false;
+                    }
+
+                    if (symbol.SecurityType == SecurityType.Base || baseData.GetType() == dataType)
+                    {
+                        // we need to store the data in data time zone
+                        baseData.Time = baseData.Time.ConvertTo(exchangeTimeZone, dataTimeZone);
+                        baseData.EndTime = baseData.EndTime.ConvertTo(exchangeTimeZone, dataTimeZone);
+                        return true;
+                    }
+                    return false;
+                })
+                // for canonical symbols, downloader will return data for all of the chain
+                .GroupBy(baseData => baseData.Symbol);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/AuxiliaryDataEnumerator.cs

@@ -0,0 +1,149 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using System.Collections;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Auxiliary data enumerator that will, initialize and call the <see cref="ITradableDateEventProvider.GetEvents"/>
+    /// implementation each time there is a new tradable day for every <see cref="ITradableDateEventProvider"/>
+    /// provided.
+    /// </summary>
+    public class AuxiliaryDataEnumerator : IEnumerator<BaseData>
+    {
+        private readonly Queue<BaseData> _auxiliaryData;
+        private bool _initialized;
+        private DateTime _startTime;
+        private IMapFileProvider _mapFileProvider;
+        private IFactorFileProvider _factorFileProvider;
+        private ITradableDateEventProvider[] _tradableDateEventProviders;
+
+        /// <summary>
+        /// The associated data configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; }
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="tradableDateEventProviders">The tradable dates event providers</param>
+        /// <param name="tradableDayNotifier">Tradable dates provider</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public AuxiliaryDataEnumerator(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            ITradableDateEventProvider []tradableDateEventProviders,
+            ITradableDatesNotifier tradableDayNotifier,
+            DateTime startTime)
+        {
+            Config = config;
+            _startTime = startTime;
+            _mapFileProvider = mapFileProvider;
+            _auxiliaryData = new Queue<BaseData>();
+            _factorFileProvider = factorFileProvider;
+            _tradableDateEventProviders = tradableDateEventProviders;
+
+            if (tradableDayNotifier != null)
+            {
+                tradableDayNotifier.NewTradableDate += NewTradableDate;
+            }
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element.
+        /// </summary>
+        /// <returns>Always true</returns>
+        public virtual bool MoveNext()
+        {
+            Current = _auxiliaryData.Count != 0 ? _auxiliaryData.Dequeue() : null;
+            return true;
+        }
+
+        /// <summary>
+        /// Handle a new tradable date, drives the <see cref="ITradableDateEventProvider"/> instances
+        /// </summary>
+        protected void NewTradableDate(object sender, NewTradableDateEventArgs eventArgs)
+        {
+            Initialize();
+            for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+            {
+                foreach (var newEvent in _tradableDateEventProviders[i].GetEvents(eventArgs))
+                {
+                    _auxiliaryData.Enqueue(newEvent);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the underlying tradable data event providers
+        /// </summary>
+        protected void Initialize()
+        {
+            if (!_initialized)
+            {
+                _initialized = true;
+                // Late initialization so it is performed in the data feed stack
+                for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+                {
+                    _tradableDateEventProviders[i].Initialize(Config, _factorFileProvider, _mapFileProvider, _startTime);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Dispose of the Stream Reader and close out the source stream and file connections.
+        /// </summary>
+        public void Dispose()
+        {
+            for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+            {
+                var disposable =_tradableDateEventProviders[i] as IDisposable;
+                disposable?.DisposeSafely();
+            }
+        }
+
+        /// <summary>
+        /// Reset the IEnumeration
+        /// </summary>
+        /// <remarks>Not used</remarks>
+        public void Reset()
+        {
+            throw new NotImplementedException("Reset method not implemented. Assumes loop will only be used once.");
+        }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Last read BaseData object from this type and source
+        /// </summary>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/BaseDataCollectionAggregatorEnumerator.cs

@@ -0,0 +1,234 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="IEnumerator{BaseDataCollection}"/>
+    /// that aggregates an underlying <see cref="IEnumerator{BaseData}"/> into a single
+    /// data packet
+    /// </summary>
+    public class BaseDataCollectionAggregatorEnumerator : IEnumerator<BaseDataCollection>
+    {
+        private bool _endOfStream;
+        private bool _needsMoveNext;
+        private bool _liveMode;
+        private readonly Symbol _symbol;
+        private readonly IEnumerator<BaseData> _enumerator;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="BaseDataCollectionAggregatorEnumerator"/> class
+        /// This will aggregate instances emitted from the underlying enumerator and tag them with the
+        /// specified symbol
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to aggregate</param>
+        /// <param name="symbol">The symbol to place on the aggregated collection</param>
+        /// <param name="liveMode">True if running in live mode</param>
+        public BaseDataCollectionAggregatorEnumerator(IEnumerator<BaseData> enumerator, Symbol symbol, bool liveMode = false)
+        {
+            _symbol = symbol;
+            _enumerator = enumerator;
+            _liveMode = liveMode;
+            _needsMoveNext = true;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            if (_endOfStream)
+            {
+                return false;
+            }
+
+            BaseDataCollection collection = null;
+            while (true)
+            {
+                if (_needsMoveNext)
+                {
+                    // move next if we dequeued the last item last time we were invoked
+                    if (!_enumerator.MoveNext())
+                    {
+                        _endOfStream = true;
+                        if (!IsValid(collection))
+                        {
+                            // we don't emit
+                            collection = null;
+                        }
+                        break;
+                    }
+                }
+
+                if (_enumerator.Current == null)
+                {
+                    // the underlying returned null, stop here and start again on the next call
+                    _needsMoveNext = true;
+                    break;
+                }
+
+                if (collection == null)
+                {
+                    // we have new data, set the collection's symbol/times
+                    var current = _enumerator.Current;
+                    collection = CreateCollection(_symbol, current.Time, current.EndTime);
+                }
+
+                if (collection.EndTime != _enumerator.Current.EndTime)
+                {
+                    // the data from the underlying is at a different time, stop here
+                    _needsMoveNext = false;
+                    if (IsValid(collection))
+                    {
+                        // we emit
+                        break;
+                    }
+                    // we try again
+                    collection = null;
+                    continue;
+                }
+
+                // this data belongs in this collection, keep going until null or bad time
+                Add(collection, _enumerator.Current);
+                _needsMoveNext = true;
+            }
+
+            Current = collection;
+            return _liveMode || collection != null;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseDataCollection Current
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Creates a new, empty <see cref="BaseDataCollection"/>.
+        /// </summary>
+        /// <param name="symbol">The base data collection symbol</param>
+        /// <param name="time">The start time of the collection</param>
+        /// <param name="endTime">The end time of the collection</param>
+        /// <returns>A new, empty <see cref="BaseDataCollection"/></returns>
+        private BaseDataCollection CreateCollection(Symbol symbol, DateTime time, DateTime endTime)
+        {
+            return new BaseDataCollection
+            {
+                Symbol = symbol,
+                Time = time,
+                EndTime = endTime
+            };
+        }
+
+        /// <summary>
+        /// Adds the specified instance of <see cref="BaseData"/> to the current collection
+        /// </summary>
+        /// <param name="collection">The collection to be added to</param>
+        /// <param name="current">The data to be added</param>
+        private void Add(BaseDataCollection collection, BaseData current)
+        {
+            var baseDataCollection = current as BaseDataCollection;
+            if (_symbol.HasUnderlying && _symbol.Underlying == current.Symbol)
+            {
+                // if the underlying has been aggregated, even if it shouldn't need to be, let's handle it nicely
+                if (baseDataCollection != null)
+                {
+                    collection.Underlying = baseDataCollection.Data[0];
+                }
+                else
+                {
+                    collection.Underlying = current;
+                }
+            }
+            else
+            {
+                if (baseDataCollection != null)
+                {
+                    // datapoint is already aggregated, let's see if it's a single point or a collection we can use already
+                    if(baseDataCollection.Data.Count > 1)
+                    {
+                        collection.Data = baseDataCollection.Data;
+                    }
+                    else
+                    {
+                        collection.Data.Add(baseDataCollection.Data[0]);
+                    }
+
+                    // Let's keep the underlying in case it's already there
+                    collection.Underlying ??= baseDataCollection.Underlying;
+                }
+                else
+                {
+                    collection.Data.Add(current);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Determines if a given data point is valid and can be emitted
+        /// </summary>
+        /// <param name="collection">The collection to be emitted</param>
+        /// <returns>True if its a valid data point</returns>
+        private static bool IsValid(BaseDataCollection collection)
+        {
+            return collection != null && collection.Data?.Count > 0;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/ConcatEnumerator.cs

@@ -0,0 +1,133 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using System.Collections;
+using QuantConnect.Logging;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that will concatenate enumerators together sequentially enumerating them in the provided order
+    /// </summary>
+    public class ConcatEnumerator : IEnumerator<BaseData>
+    {
+        private readonly List<IEnumerator<BaseData>> _enumerators;
+        private readonly bool _skipDuplicateEndTimes;
+        private DateTime? _lastEnumeratorEndTime;
+        private int _currentIndex;
+
+        /// <summary>
+        /// The current BaseData object
+        /// </summary>
+        public BaseData Current { get; set; }
+
+        /// <summary>
+        /// True if emitting a null data point is expected
+        /// </summary>
+        /// <remarks>Warmup enumerators are not allowed to return true and setting current to Null, this is because it's not a valid behavior for backtesting enumerators,
+        /// for example <see cref="FillForwardEnumerator"/></remarks>
+        public bool CanEmitNull { get; set; }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="skipDuplicateEndTimes">True will skip data points from enumerators if before or at the last end time</param>
+        /// <param name="enumerators">The sequence of enumerators to concatenate. Note that the order here matters, it will consume enumerators
+        /// and dispose of them, even if they return true and their current is null, except for the last which will be kept!</param>
+        public ConcatEnumerator(bool skipDuplicateEndTimes,
+            params IEnumerator<BaseData>[] enumerators
+            )
+        {
+            CanEmitNull = true;
+            _skipDuplicateEndTimes = skipDuplicateEndTimes;
+            _enumerators = enumerators.Where(enumerator => enumerator != null).ToList();
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>True if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.</returns>
+        public bool MoveNext()
+        {
+            for (; _currentIndex < _enumerators.Count; _currentIndex++)
+            {
+                var enumerator = _enumerators[_currentIndex];
+                while (enumerator.MoveNext())
+                {
+                    if (enumerator.Current == null && (_currentIndex < _enumerators.Count - 1 || !CanEmitNull))
+                    {
+                        // if there are more enumerators and the current stopped providing data drop it
+                        // in live trading, some enumerators will always return true (see TimeTriggeredUniverseSubscriptionEnumeratorFactory & InjectionEnumerator)
+                        // but unless it's the last enumerator we drop it, because these first are the warmup enumerators
+                        // or we are not allowed to return null
+                        break;
+                    }
+
+                    if (_skipDuplicateEndTimes
+                        && _lastEnumeratorEndTime.HasValue
+                        && enumerator.Current != null
+                        && enumerator.Current.EndTime <= _lastEnumeratorEndTime)
+                    {
+                        continue;
+                    }
+
+                    Current = enumerator.Current;
+                    return true;
+                }
+
+                _lastEnumeratorEndTime = Current?.EndTime;
+
+                if (Log.DebuggingEnabled)
+                {
+                    Log.Debug($"ConcatEnumerator.MoveNext(): disposing enumerator at position: {_currentIndex} Name: {enumerator.GetType().Name}");
+                }
+
+                // we wont be using this enumerator again, dispose of it and clear reference
+                enumerator.DisposeSafely();
+                _enumerators[_currentIndex] = null;
+            }
+
+            Current = null;
+            return false;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        public void Reset()
+        {
+            throw new InvalidOperationException($"Can not reset {nameof(ConcatEnumerator)}");
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            foreach (var enumerator in _enumerators)
+            {
+                enumerator.DisposeSafely();
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/DelistingEventProvider.cs

@@ -0,0 +1,118 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="Delisting"/> events
+    /// </summary>
+    public class DelistingEventProvider : ITradableDateEventProvider
+    {
+        // we'll use these flags to denote we've already fired off the DelistingType.Warning
+        // and a DelistedType.Delisted Delisting object, the _delistingType object is save here
+        // since we need to wait for the next trading day before emitting
+        private bool _delisted;
+        private bool _delistedWarning;
+        private IMapFileProvider _mapFileProvider;
+
+        /// <summary>
+        /// The delisting date
+        /// </summary>
+        protected ReferenceWrapper<DateTime> DelistingDate { get; set; }
+
+        /// <summary>
+        /// The current instance being used
+        /// </summary>
+        protected MapFile MapFile { get; private set; }
+
+        /// <summary>
+        /// The associated configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; private set; }
+
+        /// <summary>
+        /// Initializes this instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public virtual void Initialize(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime)
+        {
+            Config = config;
+            _mapFileProvider = mapFileProvider;
+
+            InitializeMapFile();
+        }
+
+        /// <summary>
+        /// Check for delistings
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New delisting event if any</returns>
+        public virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            if (Config.Symbol == eventArgs.Symbol)
+            {
+                // we send the delisting warning when we reach the delisting date, here we make sure we compare using the date component
+                // of the delisting date since for example some futures can trade a few hours in their delisting date, else we would skip on
+                // emitting the delisting warning, which triggers us to handle liquidation once delisted
+                if (!_delistedWarning && eventArgs.Date >= DelistingDate.Value.Date)
+                {
+                    _delistedWarning = true;
+                    var price = eventArgs.LastBaseData?.Price ?? 0;
+                    yield return new Delisting(
+                        eventArgs.Symbol,
+                        DelistingDate.Value.Date,
+                        price,
+                        DelistingType.Warning);
+                }
+                if (!_delisted && eventArgs.Date > DelistingDate.Value)
+                {
+                    _delisted = true;
+                    var price = eventArgs.LastBaseData?.Price ?? 0;
+                    // delisted at EOD
+                    yield return new Delisting(
+                        eventArgs.Symbol,
+                        DelistingDate.Value.AddDays(1),
+                        price,
+                        DelistingType.Delisted);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the factor file to use
+        /// </summary>
+        protected void InitializeMapFile()
+        {
+            MapFile = _mapFileProvider.ResolveMapFile(Config);
+            DelistingDate = new ReferenceWrapper<DateTime>(Config.Symbol.GetDelistingDate(MapFile));
+        }
+    }
+}

Engine/DataFeeds/Enumerators/DividendEventProvider.cs

@@ -0,0 +1,118 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="Dividend"/> events
+    /// </summary>
+    public class DividendEventProvider : ITradableDateEventProvider
+    {
+        // we set the price factor ratio when we encounter a dividend in the factor file
+        // and on the next trading day we use this data to produce the dividend instance
+        private decimal? _priceFactorRatio;
+        private decimal _referencePrice;
+        private IFactorFileProvider _factorFileProvider;
+        private MapFile _mapFile;
+
+        /// <summary>
+        /// The current instance being used
+        /// </summary>
+        protected CorporateFactorProvider FactorFile { get; private set; }
+
+        /// <summary>
+        /// The associated configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; private set; }
+
+        /// <summary>
+        /// Initializes this instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public void Initialize(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime)
+        {
+            Config = config;
+            _factorFileProvider = factorFileProvider;
+            _mapFile = mapFileProvider.ResolveMapFile(Config);
+            InitializeFactorFile();
+        }
+
+        /// <summary>
+        /// Check for dividends and returns them
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New Dividend event if any</returns>
+        public  virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            if (Config.Symbol == eventArgs.Symbol
+                && FactorFile != null
+                && _mapFile.HasData(eventArgs.Date))
+            {
+                if (_priceFactorRatio != null)
+                {
+                    if (_referencePrice == 0)
+                    {
+                        throw new InvalidOperationException($"Zero reference price for {Config.Symbol} dividend at {eventArgs.Date}");
+                    }
+
+                    var baseData = Dividend.Create(
+                        Config.Symbol,
+                        eventArgs.Date,
+                        _referencePrice,
+                        _priceFactorRatio.Value
+                    );
+                    // let the config know about it for normalization
+                    Config.SumOfDividends += baseData.Distribution;
+                    _priceFactorRatio = null;
+                    _referencePrice = 0;
+
+                    yield return baseData;
+                }
+
+                // check the factor file to see if we have a dividend event tomorrow
+                decimal priceFactorRatio;
+                decimal referencePrice;
+                if (FactorFile.HasDividendEventOnNextTradingDay(eventArgs.Date, out priceFactorRatio, out referencePrice))
+                {
+                    _priceFactorRatio = priceFactorRatio;
+                    _referencePrice = referencePrice;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the factor file to use
+        /// </summary>
+        protected void InitializeFactorFile()
+        {
+            FactorFile = _factorFileProvider.Get(Config.Symbol) as CorporateFactorProvider;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/EnqueueableEnumerator.cs

@@ -0,0 +1,214 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Threading;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// An implementation of <see cref="IEnumerator{T}"/> that relies on the
+    /// <see cref="Enqueue"/> method being called and only ends when <see cref="Stop"/>
+    /// is called
+    /// </summary>
+    /// <typeparam name="T">The item type yielded by the enumerator</typeparam>
+    public class EnqueueableEnumerator<T> : IEnumerator<T>
+    {
+        private T _current;
+        private bool _end;
+
+        private readonly bool _isBlocking;
+        private long _consumerCount;
+        private Queue<T> _consumer = new();
+        private Queue<T> _producer = new();
+        private readonly object _lock = new object();
+        private readonly ManualResetEventSlim _resetEvent = new(false);
+
+        /// <summary>
+        /// Gets the current number of items held in the internal queue
+        /// </summary>
+        public int Count
+        {
+            get
+            {
+                lock (_lock)
+                {
+                    if (_end) return 0;
+                    return _producer.Count + (int)Interlocked.Read(ref _consumerCount);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Returns true if the enumerator has finished and will not accept any more data
+        /// </summary>
+        public bool HasFinished
+        {
+            get { return _end; }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="EnqueueableEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="blocking">Specifies whether or not to use the blocking behavior</param>
+        public EnqueueableEnumerator(bool blocking = false)
+        {
+            _isBlocking = blocking;
+        }
+
+        /// <summary>
+        /// Enqueues the new data into this enumerator
+        /// </summary>
+        /// <param name="data">The data to be enqueued</param>
+        public void Enqueue(T data)
+        {
+            lock (_lock)
+            {
+                _producer.Enqueue(data);
+                // most of the time this will be set
+                if(!_resetEvent.IsSet)
+                {
+                    _resetEvent.Set();
+                }
+            }
+        }
+
+        /// <summary>
+        /// Signals the enumerator to stop enumerating when the items currently
+        /// held inside are gone. No more items will be added to this enumerator.
+        /// </summary>
+        public void Stop()
+        {
+            lock (_lock)
+            {
+                if (_end) return;
+                _end = true;
+
+                // no more items can be added, so no need to wait anymore
+                _resetEvent.Set();
+                _resetEvent.Dispose();
+            }
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            // we read with no lock most of the time
+            if (_consumer.TryDequeue(out _current))
+            {
+                Interlocked.Decrement(ref _consumerCount);
+                return true;
+            }
+
+            bool ended;
+            do
+            {
+                var producer = _producer;
+                lock (_lock)
+                {
+                    // swap queues
+                    ended = _end;
+                    _producer = _consumer;
+                }
+                _consumer = producer;
+                if(_consumer.Count > 0)
+                {
+                    _current = _consumer.Dequeue();
+                    Interlocked.Exchange(ref _consumerCount, _consumer.Count);
+                    break;
+                }
+
+                // if we are here no queue has data
+                if (ended)
+                {
+                    return false;
+                }
+
+                if (_isBlocking)
+                {
+                    try
+                    {
+                        _resetEvent.Wait(Timeout.Infinite);
+                        _resetEvent.Reset();
+                    }
+                    catch (ObjectDisposedException)
+                    {
+                        // can happen if disposed
+                    }
+                }
+                else
+                {
+                    break;
+                }
+            }
+            while (!ended);
+
+            // even if we don't have data to return, we haven't technically
+            // passed the end of the collection, so always return true until
+            // the enumerator is explicitly disposed or ended
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            throw new NotImplementedException("EnqueableEnumerator.Reset() has not been implemented yet.");
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            Stop();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/BaseDataCollectionSubscriptionEnumeratorFactory.cs

@@ -0,0 +1,74 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ISubscriptionEnumeratorFactory"/> that reads
+    /// an entire <see cref="SubscriptionDataSource"/> into a single <see cref="BaseDataCollection"/>
+    /// to be emitted on the tradable date at midnight
+    /// </summary>
+    /// <remarks>This enumerator factory is currently only used in backtesting with coarse data</remarks>
+    public class BaseDataCollectionSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
+    {
+        private IObjectStore _objectStore;
+
+        /// <summary>
+        /// Instanciates a new <see cref="BaseDataCollectionSubscriptionEnumeratorFactory"/>
+        /// </summary>
+        /// <param name="objectStore">The object store to use</param>
+        public BaseDataCollectionSubscriptionEnumeratorFactory(IObjectStore objectStore)
+        {
+            _objectStore = objectStore;
+        }
+
+        /// <summary>
+        /// Creates an enumerator to read the specified request
+        /// </summary>
+        /// <param name="request">The subscription request to be read</param>
+        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <returns>An enumerator reading the subscription request</returns>
+        public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
+        {
+            using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
+            {
+                var configuration = request.Configuration;
+                var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
+
+                // Behaves in the same way as in live trading
+                // (i.e. only emit coarse data on dates following a trading day)
+                // The shifting of dates is needed to ensure we never emit coarse data on the same date,
+                // because it would enable look-ahead bias.
+
+                foreach (var date in request.TradableDaysInDataTimeZone)
+                {
+                    var source = sourceFactory.GetSource(configuration, date, false);
+                    var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, configuration, date, false, sourceFactory,
+                        dataProvider, _objectStore);
+                    var coarseFundamentalForDate = factory.Read(source);
+                    //  shift all date of emitting the file forward one day to model emitting coarse midnight the next day.
+                    yield return new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate);
+                }
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/CorporateEventEnumeratorFactory.cs

@@ -0,0 +1,101 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Helper class used to create the corporate event providers
+    /// <see cref="MappingEventProvider"/>, <see cref="SplitEventProvider"/>,
+    /// <see cref="DividendEventProvider"/>, <see cref="DelistingEventProvider"/>
+    /// </summary>
+    public static class CorporateEventEnumeratorFactory
+    {
+        /// <summary>
+        /// Creates a new <see cref="AuxiliaryDataEnumerator"/> that will hold the
+        /// corporate event providers
+        /// </summary>
+        /// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">Used for getting factor files</param>
+        /// <param name="tradableDayNotifier">Tradable dates provider</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        /// <param name="endTime">
+        /// End date for the data request.
+        /// This will be used for <see cref="DataNormalizationMode.ScaledRaw"/> data normalization mode to adjust prices to the given end date
+        /// </param>
+        /// <param name="enablePriceScaling">Applies price factor</param>
+        /// <returns>The new auxiliary data enumerator</returns>
+        public static IEnumerator<BaseData> CreateEnumerators(
+            IEnumerator<BaseData> rawDataEnumerator,
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            ITradableDatesNotifier tradableDayNotifier,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime,
+            DateTime endTime,
+            bool enablePriceScaling = true)
+        {
+
+            var tradableEventProviders = new List<ITradableDateEventProvider>();
+
+            if (config.EmitSplitsAndDividends())
+            {
+                tradableEventProviders.Add(new SplitEventProvider());
+                tradableEventProviders.Add(new DividendEventProvider());
+            }
+
+            if (config.TickerShouldBeMapped())
+            {
+                tradableEventProviders.Add(new MappingEventProvider());
+            }
+
+            if (config.CanBeDelisted())
+            {
+                tradableEventProviders.Add(new DelistingEventProvider());
+            }
+
+            var enumerator = new AuxiliaryDataEnumerator(
+                config,
+                factorFileProvider,
+                mapFileProvider,
+                tradableEventProviders.ToArray(),
+                tradableDayNotifier,
+                startTime);
+
+            // avoid price scaling for backtesting; calculate it directly in worker
+            // and allow subscription to extract the the data depending on config data mode
+            var dataEnumerator = rawDataEnumerator;
+            if (enablePriceScaling && config.PricesShouldBeScaled())
+            {
+                dataEnumerator = new PriceScaleFactorEnumerator(
+                    rawDataEnumerator,
+                    config,
+                    factorFileProvider,
+                    endDate: endTime);
+            }
+
+            return new SynchronizingBaseDataEnumerator(dataEnumerator, enumerator);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs

@@ -0,0 +1,216 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Python.Runtime;
+using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ISubscriptionEnumeratorFactory"/> to handle live custom data.
+    /// </summary>
+    public class LiveCustomDataSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
+    {
+        private readonly TimeSpan _minimumIntervalCheck;
+        private readonly ITimeProvider _timeProvider;
+        private readonly Func<DateTime, DateTime> _dateAdjustment;
+        private readonly IObjectStore _objectStore;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LiveCustomDataSubscriptionEnumeratorFactory"/> class
+        /// </summary>
+        /// <param name="timeProvider">Time provider from data feed</param>
+        /// <param name="objectStore">The object store to use</param>
+        /// <param name="dateAdjustment">Func that allows adjusting the datetime to use</param>
+        /// <param name="minimumIntervalCheck">Allows specifying the minimum interval between each enumerator refresh and data check, default is 30 minutes</param>
+        public LiveCustomDataSubscriptionEnumeratorFactory(ITimeProvider timeProvider, IObjectStore objectStore,
+            Func<DateTime, DateTime> dateAdjustment = null, TimeSpan? minimumIntervalCheck = null)
+        {
+            _timeProvider = timeProvider;
+            _dateAdjustment = dateAdjustment;
+            _minimumIntervalCheck = minimumIntervalCheck ?? TimeSpan.FromMinutes(30);
+            _objectStore = objectStore;
+        }
+
+        /// <summary>
+        /// Creates an enumerator to read the specified request.
+        /// </summary>
+        /// <param name="request">The subscription request to be read</param>
+        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <returns>An enumerator reading the subscription request</returns>
+        public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
+        {
+            var config = request.Configuration;
+
+            // frontier value used to prevent emitting duplicate time stamps between refreshed enumerators
+            // also provides some immediate fast-forward to handle spooling through remote files quickly
+            var frontier = Ref.Create(_dateAdjustment?.Invoke(request.StartTimeLocal) ?? request.StartTimeLocal);
+            var lastSourceRefreshTime = DateTime.MinValue;
+            var sourceFactory = config.GetBaseDataInstance();
+
+            // this is refreshing the enumerator stack for each new source
+            var refresher = new RefreshEnumerator<BaseData>(() =>
+            {
+                // rate limit the refresh of this enumerator stack
+                var utcNow = _timeProvider.GetUtcNow();
+                var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment, _minimumIntervalCheck);
+                if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls)
+                {
+                    return Enumerable.Empty<BaseData>().GetEnumerator();
+                }
+
+                lastSourceRefreshTime = utcNow;
+                var localDate = _dateAdjustment?.Invoke(utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date) ?? utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date;
+                var source = sourceFactory.GetSource(config, localDate, true);
+
+                // fetch the new source and enumerate the data source reader
+                var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate, sourceFactory);
+
+                if (SourceRequiresFastForward(source))
+                {
+                    // The FastForwardEnumerator implements these two features:
+                    // (1) make sure we never emit past data
+                    // (2) data filtering based on a maximum data age
+                    // For custom data we don't want feature (2) because we would reject data points emitted later
+                    // (e.g. Quandl daily data after a weekend), so we disable it using a huge maximum data age.
+
+                    // apply fast forward logic for file transport mediums
+                    var maximumDataAge = GetMaximumDataAge(Time.MaxTimeSpan);
+                    enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge);
+                }
+                else
+                {
+                    // rate limit calls to this enumerator stack
+                    enumerator = new RateLimitEnumerator<BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls);
+                }
+
+                if (source.Format == FileFormat.UnfoldingCollection)
+                {
+                    // unroll collections into individual data points after fast forward/rate limiting applied
+                    enumerator = enumerator.SelectMany(data =>
+                    {
+                        var collection = data as BaseDataCollection;
+                        IEnumerator<BaseData> collectionEnumerator;
+                        if (collection != null)
+                        {
+                            if (source.TransportMedium == SubscriptionTransportMedium.Rest || source.TransportMedium == SubscriptionTransportMedium.RemoteFile)
+                            {
+                                // we want to make sure the data points we *unroll* are not past
+                                collectionEnumerator = collection.Data
+                                    .Where(baseData => baseData.EndTime > frontier.Value)
+                                    .GetEnumerator();
+                            }
+                            else
+                            {
+                                collectionEnumerator = collection.Data.GetEnumerator();
+                            }
+                        }
+                        else
+                        {
+                            collectionEnumerator = new List<BaseData> { data }.GetEnumerator();
+                        }
+                        return collectionEnumerator;
+                    });
+                }
+
+                return enumerator;
+            });
+
+            return refresher;
+        }
+
+        private IEnumerator<BaseData> EnumerateDataSourceReader(SubscriptionDataConfig config, IDataProvider dataProvider, Ref<DateTime> localFrontier, SubscriptionDataSource source, DateTime localDate, BaseData baseDataInstance)
+        {
+            using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
+            {
+                var newLocalFrontier = localFrontier.Value;
+                var dataSourceReader = GetSubscriptionDataSourceReader(source, dataCacheProvider, config, localDate, baseDataInstance, dataProvider);
+                using var subscriptionEnumerator = SortEnumerator<DateTime>.TryWrapSortEnumerator(source.Sort, dataSourceReader.Read(source));
+                foreach (var datum in subscriptionEnumerator)
+                {
+                    // always skip past all times emitted on the previous invocation of this enumerator
+                    // this allows data at the same time from the same refresh of the source while excluding
+                    // data from different refreshes of the source
+                    if (datum != null && datum.EndTime > localFrontier.Value)
+                    {
+                        yield return datum;
+                    }
+                    else if (!SourceRequiresFastForward(source))
+                    {
+                        // if the 'source' is Rest and there is no new value,
+                        // we *break*, else we will be caught in a tight loop
+                        // because Rest source never ends!
+                        // edit: we 'break' vs 'return null' so that the source is refreshed
+                        // allowing date changes to impact the source value
+                        // note it will respect 'minimumTimeBetweenCalls'
+                        break;
+                    }
+
+                    if (datum != null)
+                    {
+                        newLocalFrontier = Time.Max(datum.EndTime, newLocalFrontier);
+
+                        if (!SourceRequiresFastForward(source))
+                        {
+                            // if the 'source' is Rest we need to update the localFrontier here
+                            // because Rest source never ends!
+                            // Should be advance frontier for all source types here?
+                            localFrontier.Value = newLocalFrontier;
+                        }
+                    }
+                }
+
+                localFrontier.Value = newLocalFrontier;
+            }
+        }
+
+        /// <summary>
+        /// Gets the <see cref="ISubscriptionDataSourceReader"/> for the specified source
+        /// </summary>
+        protected virtual ISubscriptionDataSourceReader GetSubscriptionDataSourceReader(SubscriptionDataSource source,
+            IDataCacheProvider dataCacheProvider,
+            SubscriptionDataConfig config,
+            DateTime date,
+            BaseData baseDataInstance,
+            IDataProvider dataProvider
+            )
+        {
+            return SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, true, baseDataInstance, dataProvider, _objectStore);
+        }
+
+        private bool SourceRequiresFastForward(SubscriptionDataSource source)
+        {
+            return source.TransportMedium == SubscriptionTransportMedium.LocalFile
+                || source.TransportMedium == SubscriptionTransportMedium.RemoteFile;
+        }
+
+        private static TimeSpan GetMinimumTimeBetweenCalls(TimeSpan increment, TimeSpan minimumInterval)
+        {
+            return TimeSpan.FromTicks(Math.Min(increment.Ticks, minimumInterval.Ticks));
+        }
+
+        private static TimeSpan GetMaximumDataAge(TimeSpan increment)
+        {
+            return TimeSpan.FromTicks(Math.Max(increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs

@@ -0,0 +1,166 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ISubscriptionEnumeratorFactory"/> that used the <see cref="SubscriptionDataReader"/>
+    /// </summary>
+    /// <remarks>Only used on backtesting by the <see cref="FileSystemDataFeed"/></remarks>
+    public class SubscriptionDataReaderSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory, IDisposable
+    {
+        private readonly IResultHandler _resultHandler;
+        private readonly IFactorFileProvider _factorFileProvider;
+        private readonly IDataCacheProvider _dataCacheProvider;
+        private readonly ConcurrentDictionary<Symbol, string> _numericalPrecisionLimitedWarnings;
+        private readonly int _numericalPrecisionLimitedWarningsMaxCount = 10;
+        private readonly ConcurrentDictionary<Symbol, string> _startDateLimitedWarnings;
+        private readonly int _startDateLimitedWarningsMaxCount = 10;
+        private readonly IMapFileProvider _mapFileProvider;
+        private readonly bool _enablePriceScaling;
+        private readonly IAlgorithm _algorithm;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class
+        /// </summary>
+        /// <param name="resultHandler">The result handler for the algorithm</param>
+        /// <param name="mapFileProvider">The map file provider</param>
+        /// <param name="factorFileProvider">The factor file provider</param>
+        /// <param name="cacheProvider">Provider used to get data when it is not present on disk</param>
+        /// <param name="algorithm">The algorithm instance to use</param>
+        /// <param name="enablePriceScaling">Applies price factor</param>
+        public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler,
+            IMapFileProvider mapFileProvider,
+            IFactorFileProvider factorFileProvider,
+            IDataCacheProvider cacheProvider,
+            IAlgorithm algorithm,
+            bool enablePriceScaling = true
+            )
+        {
+            _algorithm = algorithm;
+            _resultHandler = resultHandler;
+            _mapFileProvider = mapFileProvider;
+            _factorFileProvider = factorFileProvider;
+            _dataCacheProvider = cacheProvider;
+            _numericalPrecisionLimitedWarnings = new ConcurrentDictionary<Symbol, string>();
+            _startDateLimitedWarnings = new ConcurrentDictionary<Symbol, string>();
+            _enablePriceScaling = enablePriceScaling;
+        }
+
+        /// <summary>
+        /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
+        /// </summary>
+        /// <param name="request">The subscription request to be read</param>
+        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <returns>An enumerator reading the subscription request</returns>
+        public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
+        {
+            var dataReader = new SubscriptionDataReader(request.Configuration,
+                request,
+                _mapFileProvider,
+                _factorFileProvider,
+                _dataCacheProvider,
+                dataProvider,
+                _algorithm.ObjectStore);
+
+            dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
+            dataReader.StartDateLimited += (sender, args) =>
+            {
+                // Queue this warning into our dictionary to report on dispose
+                if (_startDateLimitedWarnings.Count <= _startDateLimitedWarningsMaxCount)
+                {
+                    _startDateLimitedWarnings.TryAdd(args.Symbol, args.Message);
+                }
+            };
+            dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
+            dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
+            dataReader.NumericalPrecisionLimited += (sender, args) =>
+            {
+                // Set a hard limit to keep this warning list from getting unnecessarily large
+                if (_numericalPrecisionLimitedWarnings.Count <= _numericalPrecisionLimitedWarningsMaxCount)
+                {
+                    _numericalPrecisionLimitedWarnings.TryAdd(args.Symbol, args.Message);
+                }
+            };
+
+            IEnumerator<BaseData> enumerator = dataReader;
+            if (LeanData.UseDailyStrictEndTimes(_algorithm.Settings, request, request.Configuration.Symbol, request.Configuration.Increment))
+            {
+                // before corporate events which might yield data and we synchronize both feeds
+                enumerator = new StrictDailyEndTimesEnumerator(enumerator, request.ExchangeHours, request.StartTimeLocal);
+            }
+
+            enumerator = CorporateEventEnumeratorFactory.CreateEnumerators(
+                enumerator,
+                request.Configuration,
+                _factorFileProvider,
+                dataReader,
+                _mapFileProvider,
+                request.StartTimeLocal,
+                request.EndTimeLocal,
+                _enablePriceScaling);
+
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            // Log our numerical precision limited warnings if any
+            if (!_numericalPrecisionLimitedWarnings.IsNullOrEmpty())
+            {
+                var message = "Due to numerical precision issues in the factor file, data for the following" +
+                    $" symbols was adjust to a later starting date: {string.Join(", ", _numericalPrecisionLimitedWarnings.Values.Take(_numericalPrecisionLimitedWarningsMaxCount))}";
+
+                // If we reached our max warnings count suggest that more may have been left out
+                if (_numericalPrecisionLimitedWarnings.Count >= _numericalPrecisionLimitedWarningsMaxCount)
+                {
+                    message += "...";
+                }
+
+                _resultHandler.DebugMessage(message);
+            }
+
+            // Log our start date adjustments because of map files
+            if (!_startDateLimitedWarnings.IsNullOrEmpty())
+            {
+                var message = "The starting dates for the following symbols have been adjusted to match their" +
+                    $" map files first date: {string.Join(", ", _startDateLimitedWarnings.Values.Take(_startDateLimitedWarningsMaxCount))}";
+
+                // If we reached our max warnings count suggest that more may have been left out
+                if (_startDateLimitedWarnings.Count >= _startDateLimitedWarningsMaxCount)
+                {
+                    message += "...";
+                }
+
+                _resultHandler.DebugMessage(message);
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/TimeTriggeredUniverseSubscriptionEnumeratorFactory.cs

@@ -0,0 +1,61 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ISubscriptionEnumeratorFactory"/> to emit
+    /// ticks based on <see cref="UserDefinedUniverse.GetTriggerTimes"/>, allowing universe
+    /// selection to fire at planned times.
+    /// </summary>
+    public class TimeTriggeredUniverseSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
+    {
+        private readonly ITimeTriggeredUniverse _universe;
+        private readonly MarketHoursDatabase _marketHoursDatabase;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="TimeTriggeredUniverseSubscriptionEnumeratorFactory"/> class
+        /// </summary>
+        /// <param name="universe">The user defined universe</param>
+        /// <param name="marketHoursDatabase">The market hours database</param>
+        public TimeTriggeredUniverseSubscriptionEnumeratorFactory(ITimeTriggeredUniverse universe, MarketHoursDatabase marketHoursDatabase)
+        {
+            _universe = universe;
+            _marketHoursDatabase = marketHoursDatabase;
+        }
+
+        /// <summary>
+        /// Creates an enumerator to read the specified request
+        /// </summary>
+        /// <param name="request">The subscription request to be read</param>
+        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <returns>An enumerator reading the subscription request</returns>
+        public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
+        {
+            return _universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase)
+                .Select(x => new Tick { Time = x, Symbol = request.Configuration.Symbol })
+                .GetEnumerator();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/FastForwardEnumerator.cs

@@ -0,0 +1,132 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using NodaTime;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides the ability to fast forward an enumerator based on the age of the data
+    /// </summary>
+    public class FastForwardEnumerator : IEnumerator<BaseData>
+    {
+        private BaseData _current;
+
+        private readonly DateTimeZone _timeZone;
+        private readonly TimeSpan _maximumDataAge;
+        private readonly ITimeProvider _timeProvider;
+        private readonly IEnumerator<BaseData> _enumerator;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FastForwardEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerator">The source enumerator</param>
+        /// <param name="timeProvider">A time provider used to determine age of data</param>
+        /// <param name="timeZone">The data's time zone</param>
+        /// <param name="maximumDataAge">The maximum age of data allowed</param>
+        public FastForwardEnumerator(IEnumerator<BaseData> enumerator, ITimeProvider timeProvider, DateTimeZone timeZone, TimeSpan maximumDataAge)
+        {
+            _enumerator = enumerator;
+            _timeProvider = timeProvider;
+            _timeZone = timeZone;
+            _maximumDataAge = maximumDataAge;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        public bool MoveNext()
+        {
+            // keep churning until recent data or null
+            while (_enumerator.MoveNext())
+            {
+                // we can't fast forward nulls or bad times
+                if (_enumerator.Current == null || _enumerator.Current.Time == DateTime.MinValue)
+                {
+                    _current = null;
+                    return true;
+                }
+
+                // make sure we never emit past data
+                if (_current != null && _current.EndTime > _enumerator.Current.EndTime)
+                {
+                    continue;
+                }
+
+                // comute the age of the data, if within limits we're done
+                var age = _timeProvider.GetUtcNow().ConvertFromUtc(_timeZone) - _enumerator.Current.EndTime;
+                if (age <= _maximumDataAge)
+                {
+                    _current = _enumerator.Current;
+                    return true;
+                }
+            }
+
+            // we've exhausted the underlying enumerator, iterator completed
+            _current = null;
+            return false;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+    }
+}

Engine/DataFeeds/Enumerators/FillForwardEnumerator.cs

@@ -0,0 +1,638 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Data.Consolidators;
+using QuantConnect.Data.Market;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// The FillForwardEnumerator wraps an existing base data enumerator and inserts extra 'base data' instances
+    /// on a specified fill forward resolution
+    /// </summary>
+    public class FillForwardEnumerator : IEnumerator<BaseData>
+    {
+        private DateTime? _delistedTime;
+        private BaseData _previous;
+        private bool _ended;
+        private bool _isFillingForward;
+        private bool _initialized;
+
+        /// <summary>
+        /// Whether to use strict daily end times
+        /// </summary>
+        protected bool UseStrictEndTime { get; }
+
+        private readonly TimeSpan _dataResolution;
+        private readonly DateTimeZone _dataTimeZone;
+        private readonly bool _isExtendedMarketHours;
+        private readonly DateTime _subscriptionStartTime;
+        private readonly DateTime _subscriptionEndTime;
+        private readonly CalendarInfo _subscriptionEndDataCalendar;
+        private readonly IEnumerator<BaseData> _enumerator;
+        private readonly IReadOnlyRef<TimeSpan> _fillForwardResolution;
+        private readonly bool _strictEndTimeIntraDayFillForward;
+
+        /// <summary>
+        /// The exchange used to determine when to insert fill forward data
+        /// </summary>
+        protected SecurityExchange Exchange { get; init; }
+
+        /// <summary>
+        /// A reference to the last point emitted for the subscription.
+        /// This is used to feed the last point of a previous enumerator in cases like concatenated enumerators.
+        /// For instance, if this enumerator is concatenated to a warm up one, we can use this to feed
+        /// the last point of the warm up enumerator to this one, so that it can use it to fill forward if
+        /// the first actual point of this enumerator is ahead of the subscription start time or the first market open after it.
+        /// </summary>
+        private LastPointTracker _lastPointTracker;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FillForwardEnumerator"/> class that accepts
+        /// a reference to the fill forward resolution, useful if the fill forward resolution is dynamic
+        /// and changing as the enumeration progresses
+        /// </summary>
+        /// <param name="enumerator">The source enumerator to be filled forward</param>
+        /// <param name="exchange">The exchange used to determine when to insert fill forward data</param>
+        /// <param name="fillForwardResolution">The resolution we'd like to receive data on</param>
+        /// <param name="isExtendedMarketHours">True to use the exchange's extended market hours, false to use the regular market hours</param>
+        /// <param name="subscriptionStartTime">The start time of the subscription</param>
+        /// <param name="subscriptionEndTime">The end time of the subscription, once passing this date the enumerator will stop</param>
+        /// <param name="dataResolution">The source enumerator's data resolution</param>
+        /// <param name="dataTimeZone">The time zone of the underlying source data. This is used for rounding calculations and
+        /// is NOT the time zone on the BaseData instances (unless of course data time zone equals the exchange time zone)</param>
+        /// <param name="dailyStrictEndTimeEnabled">True if daily strict end times are enabled</param>
+        /// <param name="dataType">The configuration data type this enumerator is for</param>
+        /// <param name="lastPointTracker">A reference to the last point emitted before this enumerator is first enumerated</param>
+        public FillForwardEnumerator(IEnumerator<BaseData> enumerator,
+            SecurityExchange exchange,
+            IReadOnlyRef<TimeSpan> fillForwardResolution,
+            bool isExtendedMarketHours,
+            DateTime subscriptionStartTime,
+            DateTime subscriptionEndTime,
+            TimeSpan dataResolution,
+            DateTimeZone dataTimeZone,
+            bool dailyStrictEndTimeEnabled,
+            Type dataType = null,
+            LastPointTracker lastPointTracker = null
+            )
+        {
+            _subscriptionStartTime = subscriptionStartTime;
+            _subscriptionEndTime = subscriptionEndTime;
+            Exchange = exchange;
+            _enumerator = enumerator;
+            _dataResolution = dataResolution;
+            _dataTimeZone = dataTimeZone;
+            _fillForwardResolution = fillForwardResolution;
+            _isExtendedMarketHours = isExtendedMarketHours;
+            _lastPointTracker = lastPointTracker;
+            UseStrictEndTime = dailyStrictEndTimeEnabled;
+            // OI data is fill-forwarded to the market close time when strict end times is enabled.
+            // Open interest data can arrive at any time and this would allow to synchronize it with trades and quotes when daily
+            // strict end times is enabled
+            _strictEndTimeIntraDayFillForward = dailyStrictEndTimeEnabled && dataType != null && dataType == typeof(OpenInterest);
+
+            // '_dataResolution' and '_subscriptionEndTime' are readonly they won't change, so lets calculate this once here since it's expensive.
+            // if UseStrictEndTime and also _strictEndTimeIntraDayFillForward, this is a subscription with data that is not adjusted
+            // for the strict end time (like open interest) but require fill forward to synchronize with other data.
+            // Use the non strict end time calendar for the last day of data so that all data for that date is emitted.
+            if (UseStrictEndTime && !_strictEndTimeIntraDayFillForward)
+            {
+                var lastDayCalendar = GetDailyCalendar(_subscriptionEndTime);
+                while (lastDayCalendar.End > _subscriptionEndTime)
+                {
+                    lastDayCalendar = GetDailyCalendar(lastDayCalendar.Start.AddDays(-1));
+                }
+                _subscriptionEndDataCalendar = lastDayCalendar;
+            }
+            else
+            {
+                _subscriptionEndDataCalendar = new (RoundDown(_subscriptionEndTime, _dataResolution), _dataResolution);
+            }
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current => Current;
+
+        private void Initialize()
+        {
+            if (_initialized)
+            {
+                return;
+            }
+
+            if (_lastPointTracker?.LastDataPoint != null)
+            {
+                // adjust the previous data point to the subscription start time to
+                // avoid emitting fill forward data before that
+                _previous = _lastPointTracker.LastDataPoint.Clone();
+                _previous.Time = _subscriptionStartTime - _dataResolution;
+            }
+
+            _initialized = true;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            Initialize();
+
+            if (_delistedTime.HasValue)
+            {
+                // don't fill forward after data after the delisted date
+                if (_previous == null || _previous.EndTime >= _delistedTime.Value)
+                {
+                    return false;
+                }
+            }
+
+            if (Current != null && Current.DataType != MarketDataType.Auxiliary)
+            {
+                // only set the _previous if the last item we emitted was NOT auxilliary data,
+                // since _previous is used for fill forward behavior
+                _previous = Current;
+            }
+
+            BaseData fillForward;
+
+            if (!_isFillingForward)
+            {
+                // if we're filling forward we don't need to move next since we haven't emitted _enumerator.Current yet
+                if (!_enumerator.MoveNext())
+                {
+                    _ended = true;
+                    if (_delistedTime.HasValue)
+                    {
+                        // don't fill forward delisted data
+                        return false;
+                    }
+
+                    // check to see if we ran out of data before the end of the subscription
+                    if (_previous == null || _previous.EndTime >= _subscriptionEndTime)
+                    {
+                        // we passed the end of subscription, we're finished
+                        return false;
+                    }
+
+                    // we can fill forward the rest of this subscription if required
+                    var endOfSubscription = (Current ?? _previous).Clone(true);
+                    endOfSubscription.Time = _subscriptionEndDataCalendar.Start;
+                    endOfSubscription.EndTime = endOfSubscription.Time + _subscriptionEndDataCalendar.Period;
+                    if (RequiresFillForwardData(_fillForwardResolution.Value, _previous, endOfSubscription, out fillForward))
+                    {
+                        // don't mark as filling forward so we come back into this block, subscription is done
+                        //_isFillingForward = true;
+                        Current = fillForward;
+                        return true;
+                    }
+
+                    // don't emit the last bar if the market isn't considered open!
+                    if (!Exchange.IsOpenDuringBar(endOfSubscription.Time, endOfSubscription.EndTime, _isExtendedMarketHours))
+                    {
+                        return false;
+                    }
+
+                    if (Current != null && Current.EndTime == endOfSubscription.EndTime
+                        // TODO this changes stats, why would the FF enumerator emit a data point beyoned the end time he was requested
+                        //|| endOfSubscription.EndTime > _subscriptionEndTime
+                        )
+                    {
+                        return false;
+                    }
+                    Current = endOfSubscription;
+                    return true;
+                }
+            }
+            // If we are filling forward and the underlying is null, let's MoveNext() as long as it didn't end.
+            // This only applies for live trading, so that the LiveFillForwardEnumerator does not stall whenever
+            // we generate a fill-forward bar. The underlying enumerator is advanced so that we don't get stuck
+            // in a cycle of generating infinite fill-forward bars.
+            else if (_enumerator.Current == null && !_ended)
+            {
+                _ended = _enumerator.MoveNext();
+            }
+
+            var underlyingCurrent = _enumerator.Current;
+            if (underlyingCurrent != null && underlyingCurrent.DataType == MarketDataType.Auxiliary)
+            {
+                var delisting = underlyingCurrent as Delisting;
+                if (delisting?.Type == DelistingType.Delisted)
+                {
+                    _delistedTime = delisting.EndTime;
+                }
+            }
+
+            if (_previous == null)
+            {
+                // first data point we dutifully emit without modification
+                Current = underlyingCurrent;
+                return true;
+            }
+
+            if (RequiresFillForwardData(_fillForwardResolution.Value, _previous, underlyingCurrent, out fillForward))
+            {
+                if (_previous.EndTime >= _subscriptionEndTime)
+                {
+                    // we passed the end of subscription, we're finished
+                    return false;
+                }
+                // we require fill forward data because the _enumerator.Current is too far in future
+                _isFillingForward = true;
+                Current = fillForward;
+                return true;
+            }
+
+            _isFillingForward = false;
+            Current = underlyingCurrent;
+            return true;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Determines whether or not fill forward is required, and if true, will produce the new fill forward data
+        /// </summary>
+        /// <param name="fillForwardResolution"></param>
+        /// <param name="previous">The last piece of data emitted by this enumerator</param>
+        /// <param name="next">The next piece of data on the source enumerator</param>
+        /// <param name="fillForward">When this function returns true, this will have a non-null value, null when the function returns false</param>
+        /// <returns>True when a new fill forward piece of data was produced and should be emitted by this enumerator</returns>
+        protected virtual bool RequiresFillForwardData(TimeSpan fillForwardResolution, BaseData previous, BaseData next, out BaseData fillForward)
+        {
+            // in live trading next can be null, in which case we create a potential FF bar and the live FF enumerator will decide what to do
+            var nextCalculatedEndTimeUtc = DateTime.MaxValue;
+            if (next != null)
+            {
+                // convert times to UTC for accurate comparisons and differences across DST changes
+                var previousTimeUtc = previous.Time.ConvertToUtc(Exchange.TimeZone);
+                var nextTimeUtc = next.Time.ConvertToUtc(Exchange.TimeZone);
+                var nextEndTimeUtc = next.EndTime.ConvertToUtc(Exchange.TimeZone);
+
+                if (nextEndTimeUtc < previousTimeUtc)
+                {
+                    if (_lastPointTracker == null || next.EndTime > _subscriptionStartTime)
+                    {
+                        // in some cases we might emit auxiliary data even before our actual start time, which can happen in some cases during warmup
+                        // where previous was initialized through the last point tracker, this point will be filtered out
+                        // but in any other case though let's log it, shouldn't happen
+                        Log.Error("FillForwardEnumerator received data out of order. Symbol: " + previous.Symbol.ID);
+                    }
+                    fillForward = null;
+                    return false;
+                }
+
+                // check to see if the gap between previous and next warrants fill forward behavior
+                if (!ShouldFillForward(previousTimeUtc, nextTimeUtc, fillForwardResolution))
+                {
+                    fillForward = null;
+                    return false;
+                }
+
+                // Double check!
+                // This might be the last FF bar before the next data point, and it might not be to be
+                // emitted because it will overlap with the next point.
+                // If the previous point was fill forwarded, its time might have been rounded down,
+                // we need to compare apples to apples.
+                // (e.g. daily bars with times != midnight and without strict end times)
+                var nextPeriod = nextEndTimeUtc - nextTimeUtc;
+                if (previous.IsFillForward && (!UseStrictEndTime || nextPeriod <= Time.OneHour))
+                {
+                    var roundedNextTimeUtc = RoundDown(next.Time, nextPeriod).ConvertToUtc(Exchange.TimeZone);
+                    if (!ShouldFillForward(previousTimeUtc, roundedNextTimeUtc, fillForwardResolution))
+                    {
+                        fillForward = null;
+                        return false;
+                    }
+                }
+
+                var period = _dataResolution;
+                if (UseStrictEndTime)
+                {
+                    // the period is not the data resolution (1 day) and can actually change dynamically, for example early close/late open
+                    period = next.EndTime - next.Time;
+                }
+                else if (next.Time == next.EndTime)
+                {
+                    // we merge corporate event data points (mapping, delisting, splits, dividend) which do not have
+                    // a period or resolution
+                    period = TimeSpan.Zero;
+                }
+                nextCalculatedEndTimeUtc = nextTimeUtc + period;
+            }
+
+            // every bar emitted MUST be of the data resolution.
+
+            // compute end times of the four potential fill forward scenarios
+            // 1. the next fill forward bar. 09:00-10:00 followed by 10:00-11:00 where 01:00 is the fill forward resolution
+            // 2. the next data resolution bar, same as above but with the data resolution instead
+            // 3. the next fill forward bar following the next market open, 15:00-16:00 followed by 09:00-10:00 the following open market day
+            // 4. the next data resolution bar following the next market open, same as above but with the data resolution instead
+
+            // the precedence for validation is based on the order of the end times, obviously if a potential match
+            // is before a later match, the earliest match should win.
+
+            foreach (var item in GetSortedReferenceDateIntervals(previous, fillForwardResolution, _dataResolution))
+            {
+                // issue GH 4925 , more description https://github.com/QuantConnect/Lean/pull/4941
+                // To build Time/EndTime we always use '+'/'-' dataResolution
+                // DataTime TZ = UTC -5; Exchange TZ = America/New York (-5/-4)
+                // Standard TimeZone    00:00:00 + 1 day = 1.00:00:00
+                // Daylight Time        01:00:00 + 1 day = 1.01:00:00
+
+                // daylight saving time starts/end at 2 a.m. on Sunday
+                // Having this information we find that the specific bar of Sunday
+                // Starts in one TZ (Standard TZ), but finishes in another (Daylight TZ) (consider winter => summer)
+                // During simple arithmetic operations like +/- we shift the time, but not the time zone
+                // which is sensitive for specific dates (daylight movement) if we are  in Exchange TimeZone, for example
+                // We have 00:00:00 + 1 day = 1.00:00:00, so both are in Standard TZ, but we expect endTime in Daylight, i.e. 1.01:00:00
+
+                // futher down double Convert (Exchange TZ => data TZ => Exchange TZ)
+                // allows us to calculate Time using it's own TZ (aka reapply)
+                // and don't rely on TZ of bar start/end time
+                // i.e. 00:00:00 + 1 day = 1.01:00:00, both start and end are in their own TZ
+                // it's interesting that NodaTime  consider next
+                // if time great or equal than 01:00 AM it's considered as "moved" (Standard, not Daylight)
+                // when time less than 01:00 AM it's considered as previous TZ (Standard, not Daylight)
+                // it's easy to fix this behavior by substract 1 tick  before first convert, and then return it back.
+                // so we work with 0:59:59.. AM instead.
+                // but now follow native behavior
+
+                // all above means, that all Time values, calculated using simple +/- operations
+                // sticks to original Time Zone, swallowing its own TZ and movement i.e.
+                // EndTime = Time + resolution, both Time and EndTime in the TZ of Time (Standard/Daylight)
+                // Time = EndTime - resolution, both Time and EndTime in the TZ of EndTime (Standard/Daylight)
+
+                // next.EndTime sticks to Time TZ,
+                // potentialBarEndTime should be calculated in the same way as bar.EndTime, i.e. Time + resolution
+                // round down doesn't make sense for daily data using strict times
+                var startTime = (UseStrictEndTime && item.Period > Time.OneHour) ? item.Start : RoundDown(item.Start, item.Period);
+                var potentialBarEndTime = startTime.ConvertToUtc(Exchange.TimeZone) + item.Period;
+
+                // to avoid duality it's necessary to compare potentialBarEndTime with
+                // next.EndTime calculated as Time + resolution,
+                // and both should be based on the same TZ (for example UTC)
+                if (potentialBarEndTime < nextCalculatedEndTimeUtc
+                    // let's fill forward based on previous (which isn't auxiliary) if next is auxiliary and they share the end time
+                    // we do allow emitting both an auxiliary data point and a Filled Forwared data for the same end time
+                    || next != null && next.DataType == MarketDataType.Auxiliary && potentialBarEndTime == nextCalculatedEndTimeUtc)
+                {
+                    // to check open hours we need to convert potential
+                    // bar EndTime into exchange time zone
+                    var potentialBarEndTimeInExchangeTZ =
+                        potentialBarEndTime.ConvertFromUtc(Exchange.TimeZone);
+                    var nextFillForwardBarStartTime = potentialBarEndTimeInExchangeTZ - item.Period;
+
+                    if (Exchange.IsOpenDuringBar(nextFillForwardBarStartTime, potentialBarEndTimeInExchangeTZ, _isExtendedMarketHours))
+                    {
+                        fillForward = previous.Clone(true);
+
+                        // bar are ALWAYS of the data resolution
+                        var expectedPeriod = _dataResolution;
+                        if (UseStrictEndTime)
+                        {
+                            // TODO: what about extended market hours
+                            // NOTE: Not using Exchange.Hours.RegularMarketDuration so we can handle things like early closes.
+
+                            // The earliest start time would be endTime - regularMarketDuration,
+                            // we use that as the potential time to get the exchange hours.
+                            // We don't use directly nextFillForwardBarStartTime because there might be cases where there are
+                            // adjacent extended and regular market hours segments that might cause the calendar start to be
+                            // in the previous date, and if it's an extended hours-only date like a Sunday for futures,
+                            // the market duration would be zero.
+                            var marketHoursDateTime = potentialBarEndTimeInExchangeTZ - Exchange.Hours.RegularMarketDuration;
+                            // That potential start is even before the calendar start, so we use the calendar start
+                            if (marketHoursDateTime < item.Start)
+                            {
+                                marketHoursDateTime = item.Start;
+                            }
+                            var marketHours = Exchange.Hours.GetMarketHours(marketHoursDateTime);
+                            expectedPeriod = marketHours.MarketDuration;
+                        }
+                        fillForward.Time = (potentialBarEndTime - expectedPeriod).ConvertFromUtc(Exchange.TimeZone);
+                        fillForward.EndTime = potentialBarEndTimeInExchangeTZ;
+                        return true;
+                    }
+                }
+                else
+                {
+                    break;
+                }
+            }
+
+            // the next is before the next fill forward time, so do nothing
+            fillForward = null;
+            return false;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private bool ShouldFillForward(DateTime previousTimeUtc, DateTime nextTimeUtc, TimeSpan fillForwardResolution)
+        {
+            var nextPreviousTimeUtcDelta = nextTimeUtc - previousTimeUtc;
+            return nextPreviousTimeUtcDelta > fillForwardResolution ||
+                nextPreviousTimeUtcDelta > _dataResolution ||
+                // even if there is no gap between the two data points, we still fill forward to ensure a FF bar is emitted at strict end time
+                _strictEndTimeIntraDayFillForward;
+        }
+
+        private IEnumerable<CalendarInfo> GetSortedReferenceDateIntervals(BaseData previous, TimeSpan fillForwardResolution, TimeSpan dataResolution)
+        {
+            if (fillForwardResolution < dataResolution)
+            {
+                return GetReferenceDateIntervals(previous.EndTime, fillForwardResolution, dataResolution);
+            }
+
+            if (fillForwardResolution > dataResolution)
+            {
+                return GetReferenceDateIntervals(previous.EndTime, dataResolution, fillForwardResolution);
+            }
+
+            return GetReferenceDateIntervals(previous.EndTime, fillForwardResolution);
+        }
+
+        /// <summary>
+        /// Get potential next fill forward bars.
+        /// </summary>
+        /// <remarks>Special case where fill forward resolution and data resolution are equal</remarks>
+        private IEnumerable<CalendarInfo> GetReferenceDateIntervals(DateTime previousEndTime, TimeSpan resolution)
+        {
+            // say daily bar goes from 9:30 to 16:00, if resolution is 1 day, IsOpenDuringBar can return true but it's not what we want
+            if (!UseStrictEndTime && Exchange.IsOpenDuringBar(previousEndTime, previousEndTime + resolution, _isExtendedMarketHours))
+            {
+                // if next in market us it
+                yield return new (previousEndTime, resolution);
+            }
+
+            if (UseStrictEndTime)
+            {
+                // If we're using strict end times for open interest data, for instance, the actual data comes at any time
+                // but we want to emit a ff point at market close. If extended market hours are enabled, and previousEndTime
+                // is Thursday after last segment open time, the daily calendar will be for Monday, because a next market open
+                // won't be found for Friday. So we use the Date of the previousEndTime to get calendar starting that day (Thursday)
+                // and ending the next one (Friday).
+                if (_strictEndTimeIntraDayFillForward)
+                {
+                    var firtMarketOpen = Exchange.Hours.GetNextMarketOpen(previousEndTime.Date, _isExtendedMarketHours);
+                    var firstCalendar = LeanData.GetDailyCalendar(firtMarketOpen, Exchange.Hours, false);
+
+                    if (firstCalendar.End > previousEndTime)
+                    {
+                        yield return firstCalendar;
+                    }
+                }
+
+                // now we can try the bar after next market open
+                var marketOpen = Exchange.Hours.GetNextMarketOpen(previousEndTime, false);
+                yield return GetDailyCalendar(marketOpen);
+            }
+            else
+            {
+                // now we can try the bar after next market open
+                var marketOpen = Exchange.Hours.GetNextMarketOpen(previousEndTime, _isExtendedMarketHours);
+                yield return new(marketOpen, resolution);
+            }
+        }
+
+        /// <summary>
+        /// Get potential next fill forward bars.
+        /// </summary>
+        private IEnumerable<CalendarInfo> GetReferenceDateIntervals(DateTime previousEndTime, TimeSpan smallerResolution, TimeSpan largerResolution)
+        {
+            List<CalendarInfo> result = null;
+            if (Exchange.IsOpenDuringBar(previousEndTime, previousEndTime + smallerResolution, _isExtendedMarketHours))
+            {
+                if (UseStrictEndTime)
+                {
+                    // case A
+                    result = new()
+                    {
+                        new(previousEndTime, smallerResolution)
+                    };
+                }
+                else
+                {
+                    // at the end of this method we perform an OrderBy which does not apply for this case because the consumer of this method
+                    // will perform a round down that will end up using an unexpected FF bar. This behavior is covered by tests
+                    yield return new (previousEndTime, smallerResolution);
+                }
+            }
+            result ??= new List<CalendarInfo>(4);
+
+            // we need to round down because previous end time could be of the smaller resolution, in data TZ!
+            if (UseStrictEndTime)
+            {
+                // case B: say smaller resolution (FF res) is 1 hour, larget resolution (daily data resolution) is 1 day
+                // For example for SPX we need to emit the daily FF bar from 8:30->15:15, even before the 'A' case above which would be 15->16 bar
+                var dailyCalendar = GetDailyCalendar(previousEndTime);
+                if (previousEndTime < (dailyCalendar.Start + dailyCalendar.Period))
+                {
+                    result.Add(new(dailyCalendar.Start, dailyCalendar.Period));
+                }
+            }
+            else
+            {
+                var start = RoundDown(previousEndTime, largerResolution);
+                if (Exchange.IsOpenDuringBar(start, start + largerResolution, _isExtendedMarketHours))
+                {
+                    result.Add(new(start, largerResolution));
+                }
+            }
+
+            // this is typically daily data being filled forward on a higher resolution
+            // since the previous bar was not in market hours then we can just fast forward
+            // to the next market open
+            var marketOpen = Exchange.Hours.GetNextMarketOpen(previousEndTime, _isExtendedMarketHours);
+            result.Add(new (marketOpen, smallerResolution));
+            if (UseStrictEndTime)
+            {
+                result.Add(GetDailyCalendar(Exchange.Hours.GetNextMarketOpen(previousEndTime, false)));
+            }
+
+            // we need to order them because they might not be in an incremental order and consumer expects them to be
+            foreach (var referenceDateInterval in result.OrderBy(interval => interval.Start + interval.Period))
+            {
+                yield return referenceDateInterval;
+            }
+        }
+
+        /// <summary>
+        /// We need to round down in data timezone.
+        /// For example GH issue 4392: Forex daily data, exchange tz time is 8PM, but time in data tz is 12AM
+        /// so rounding down on exchange tz will crop it, while rounding on data tz will return the same data point time.
+        /// Why are we even doing this? being able to determine the next valid data point for a resolution from a data point that might be in another resolution
+        /// </summary>
+        private DateTime RoundDown(DateTime value, TimeSpan interval)
+        {
+            return value.RoundDownInTimeZone(interval, Exchange.TimeZone, _dataTimeZone);
+        }
+
+        private CalendarInfo GetDailyCalendar(DateTime localReferenceTime)
+        {
+            // daily data does not have extended market hours, even if requested
+            // and it's times are always market hours if using strict end times see 'SetStrictEndTimes'
+            return LeanData.GetDailyCalendar(localReferenceTime, Exchange.Hours, extendedMarketHours: false);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/FilterEnumerator.cs

@@ -0,0 +1,96 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that allow applying a filtering function
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public class FilterEnumerator<T> : IEnumerator<T>
+    {
+        private readonly IEnumerator<T> _enumerator;
+        private readonly Func<T, bool> _filter;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to filter on</param>
+        /// <param name="filter">The filter to apply</param>
+        public FilterEnumerator(IEnumerator<T> enumerator, Func<T, bool> filter)
+        {
+            _enumerator = enumerator;
+            _filter = filter;
+        }
+
+        #region Implementation of IDisposable
+
+        /// <summary>
+        /// Disposes the FilterEnumerator
+        /// </summary>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        #endregion
+
+        #region Implementation of IEnumerator
+
+        /// <summary>
+        /// Moves the FilterEnumerator to the next item
+        /// </summary>
+        public bool MoveNext()
+        {
+            // run the enumerator until it passes the specified filter
+            while (_enumerator.MoveNext())
+            {
+                if (_filter(_enumerator.Current))
+                {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Resets the FilterEnumerator
+        /// </summary>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the current item in the FilterEnumerator
+        /// </summary>
+        public T Current
+        {
+            get { return _enumerator.Current; }
+        }
+
+        object IEnumerator.Current
+        {
+            get { return _enumerator.Current; }
+        }
+
+        #endregion
+    }
+}

Engine/DataFeeds/Enumerators/FrontierAwareEnumerator.cs

@@ -0,0 +1,149 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="IEnumerator{BaseData}"/> that will not emit
+    /// data ahead of the frontier as specified by an instance of <see cref="ITimeProvider"/>.
+    /// An instance of <see cref="TimeZoneOffsetProvider"/> is used to convert between UTC
+    /// and the data's native time zone
+    /// </summary>
+    public class FrontierAwareEnumerator : IEnumerator<BaseData>
+    {
+        private BaseData _current;
+        private bool _needsMoveNext = true;
+
+        private readonly ITimeProvider _timeProvider;
+        private readonly IEnumerator<BaseData> _enumerator;
+        private readonly TimeZoneOffsetProvider _offsetProvider;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FrontierAwareEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to make frontier aware</param>
+        /// <param name="timeProvider">The time provider used for resolving the current frontier time</param>
+        /// <param name="offsetProvider">An offset provider used for converting the frontier UTC time into the data's native time zone</param>
+        public FrontierAwareEnumerator(IEnumerator<BaseData> enumerator, ITimeProvider timeProvider, TimeZoneOffsetProvider offsetProvider)
+        {
+            _enumerator = enumerator;
+            _timeProvider = timeProvider;
+            _offsetProvider = offsetProvider;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            var underlyingCurrent = _enumerator.Current;
+            var frontier = _timeProvider.GetUtcNow();
+            var localFrontier = new DateTime(frontier.Ticks + _offsetProvider.GetOffsetTicks(frontier));
+
+            // if we moved next, but didn't emit, check to see if it's time to emit yet
+            if (!_needsMoveNext && underlyingCurrent != null)
+            {
+                if (underlyingCurrent.EndTime <= localFrontier)
+                {
+                    // we can now emit the underlyingCurrent as part of this time slice
+                    _current = underlyingCurrent;
+                    _needsMoveNext = true;
+                }
+                else
+                {
+                    // it's still not time to emit the underlyingCurrent, keep waiting for time to advance
+                    _current = null;
+                    _needsMoveNext = false;
+                }
+                return true;
+            }
+
+            // we've exhausted the underlying enumerator, iteration completed
+            if (_needsMoveNext && !_enumerator.MoveNext())
+            {
+                _needsMoveNext = true;
+                _current = null;
+                return false;
+            }
+
+            underlyingCurrent = _enumerator.Current;
+
+            if (underlyingCurrent != null && underlyingCurrent.EndTime <= localFrontier)
+            {
+                _needsMoveNext = true;
+                _current = underlyingCurrent;
+            }
+            else
+            {
+                _current = null;
+                _needsMoveNext = underlyingCurrent == null;
+            }
+
+            // technically we still need to return true since the iteration is not completed,
+            // however, Current may be null follow a true result here
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/ITradableDateEventProvider.cs

@@ -0,0 +1,49 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Interface for event providers for new tradable dates
+    /// </summary>
+    public interface ITradableDateEventProvider
+    {
+        /// <summary>
+        /// Called each time there is a new tradable day
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New corporate event if any</returns>
+        IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs);
+
+        /// <summary>
+        /// Initializes the event provider instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        void Initialize(SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime);
+    }
+}

Engine/DataFeeds/Enumerators/ITradableDatesNotifier.cs

@@ -0,0 +1,32 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Interface which will provide an event handler
+    /// who will be fired with each new tradable day
+    /// </summary>
+    public interface ITradableDatesNotifier
+    {
+        /// <summary>
+        /// Event fired when there is a new tradable date
+        /// </summary>
+        event EventHandler<NewTradableDateEventArgs> NewTradableDate;
+    }
+}

Engine/DataFeeds/Enumerators/LastPointTracker.cs

@@ -0,0 +1,43 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Tracks the last data point received by an enumerator.
+    /// </summary>
+    public class LastPointTracker
+    {
+        private BaseData _lastPoint;
+
+        /// <summary>
+        /// Tracks the last data point received by the enumerator.
+        /// </summary>
+        public BaseData LastDataPoint
+        {
+            get => _lastPoint;
+            set
+            {
+                if (value != null && !value.IsFillForward && value.DataType != MarketDataType.Auxiliary)
+                {
+                    _lastPoint = value;
+                }
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveAuxiliaryDataEnumerator.cs

@@ -0,0 +1,114 @@
+/*
+* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Auxiliary data enumerator that will trigger new tradable dates event accordingly
+    /// </summary>
+    public class LiveAuxiliaryDataEnumerator : AuxiliaryDataEnumerator
+    {
+        private DateTime _lastTime;
+        private ITimeProvider _timeProvider;
+        private SecurityCache _securityCache;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="tradableDateEventProviders">The tradable dates event providers</param>
+        /// <param name="startTime">Start date for the data request</param>
+        /// <param name="timeProvider">The time provider to use</param>
+        /// <param name="securityCache">The security cache</param>
+        public LiveAuxiliaryDataEnumerator(SubscriptionDataConfig config, IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider, ITradableDateEventProvider[] tradableDateEventProviders,
+            DateTime startTime,
+            ITimeProvider timeProvider,
+            SecurityCache securityCache)
+            // tradableDayNotifier: null -> we are going to trigger the new tradables events for the base implementation
+            : base(config, factorFileProvider, mapFileProvider, tradableDateEventProviders, tradableDayNotifier:null, startTime)
+        {
+            _securityCache = securityCache;
+            _timeProvider = timeProvider;
+
+            // initialize providers right away so mapping happens before we subscribe
+            Initialize();
+        }
+
+        /// <summary>
+        /// Moves the LiveAuxiliaryDataEnumerator to the next item
+        /// </summary>
+        public override bool MoveNext()
+        {
+            var currentDate = _timeProvider.GetUtcNow().ConvertFromUtc(Config.ExchangeTimeZone).Add(-Time.LiveAuxiliaryDataOffset).Date;
+            if (currentDate != _lastTime)
+            {
+                // when the date changes for the security we trigger a new tradable date event
+                var newDayEvent = new NewTradableDateEventArgs(currentDate, _securityCache.GetData(), Config.Symbol, null);
+
+                NewTradableDate(this, newDayEvent);
+                // update last time
+                _lastTime = currentDate;
+            }
+
+            return base.MoveNext();
+        }
+
+        /// <summary>
+        /// Helper method to create a new instance.
+        /// Knows which security types should create one and determines the appropriate delisting event provider to use
+        /// </summary>
+        public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider,
+            SecurityCache securityCache, IMapFileProvider mapFileProvider, IFactorFileProvider fileProvider, DateTime startTime,
+            out IEnumerator<BaseData> enumerator)
+        {
+            enumerator = null;
+            var securityType = dataConfig.SecurityType;
+            if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity)
+            {
+                var providers = new List<ITradableDateEventProvider>
+                {
+                    securityType == SecurityType.Equity
+                        ? new LiveDelistingEventProvider()
+                        : new DelistingEventProvider()
+                };
+
+                if (dataConfig.TickerShouldBeMapped())
+                {
+                    providers.Add(new LiveMappingEventProvider());
+                }
+
+                if (dataConfig.EmitSplitsAndDividends())
+                {
+                    providers.Add(new LiveDividendEventProvider());
+                    providers.Add(new LiveSplitEventProvider());
+                }
+
+                enumerator = new LiveAuxiliaryDataEnumerator(dataConfig, fileProvider, mapFileProvider,
+                    providers.ToArray(), startTime, timeProvider, securityCache);
+            }
+            return enumerator != null;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveAuxiliaryDataSynchronizingEnumerator.cs

@@ -0,0 +1,191 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Represents an enumerator capable of synchronizing live equity data enumerators in time.
+    /// This assumes that all enumerators have data time stamped in the same time zone.
+    /// </summary>
+    public class LiveAuxiliaryDataSynchronizingEnumerator : IEnumerator<BaseData>
+    {
+        private readonly ITimeProvider _timeProvider;
+        private readonly DateTimeZone _exchangeTimeZone;
+        private readonly List<IEnumerator<BaseData>> _auxDataEnumerators;
+        private readonly IEnumerator<BaseData> _tradeBarAggregator;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LiveAuxiliaryDataSynchronizingEnumerator"/> class
+        /// </summary>
+        /// <param name="timeProvider">The source of time used to gauge when this enumerator should emit extra bars when null data is returned from the source enumerator</param>
+        /// <param name="exchangeTimeZone">The time zone the raw data is time stamped in</param>
+        /// <param name="tradeBarAggregator">The trade bar aggregator enumerator</param>
+        /// <param name="auxDataEnumerators">The auxiliary data enumerators</param>
+        public LiveAuxiliaryDataSynchronizingEnumerator(ITimeProvider timeProvider, DateTimeZone exchangeTimeZone, IEnumerator<BaseData> tradeBarAggregator, List<IEnumerator<BaseData>> auxDataEnumerators)
+        {
+            _timeProvider = timeProvider;
+            _exchangeTimeZone = exchangeTimeZone;
+            _auxDataEnumerators = auxDataEnumerators;
+            _tradeBarAggregator = tradeBarAggregator;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns> true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.</returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
+        public bool MoveNext()
+        {
+            // use manual time provider from LiveTradingDataFeed
+            var frontierUtc = _timeProvider.GetUtcNow();
+
+            // check if any enumerator is ready to emit
+            if (DataPointEmitted(frontierUtc))
+                return true;
+
+            // advance enumerators with no current data
+            for (var i = 0; i < _auxDataEnumerators.Count; i++)
+            {
+                if (_auxDataEnumerators[i].Current == null)
+                {
+                    _auxDataEnumerators[i].MoveNext();
+                }
+            }
+            if (_tradeBarAggregator.Current == null) _tradeBarAggregator.MoveNext();
+
+            // check if any enumerator is ready to emit
+            if (DataPointEmitted(frontierUtc))
+                return true;
+
+            Current = null;
+
+            // IEnumerator contract dictates that we return true unless we're actually
+            // finished with the 'collection' and since this is live, we're never finished
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
+        public void Reset()
+        {
+            foreach (var auxDataEnumerator in _auxDataEnumerators)
+            {
+                auxDataEnumerator.Reset();
+            }
+            _tradeBarAggregator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>The element in the collection at the current position of the enumerator.</returns>
+        public BaseData Current { get; private set; }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>The current element in the collection.</returns>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            foreach (var auxDataEnumerator in _auxDataEnumerators)
+            {
+                auxDataEnumerator.DisposeSafely();
+            }
+            _tradeBarAggregator.DisposeSafely();
+        }
+
+        private bool DataPointEmitted(DateTime frontierUtc)
+        {
+            // we get the aux enumerator that has the smallest endTime if any
+            IEnumerator<BaseData> auxDataEnumerator = null;
+            for (var i = 0; i < _auxDataEnumerators.Count; i++)
+            {
+                var currentEnum = _auxDataEnumerators[i];
+                if (currentEnum.Current != null)
+                {
+                    if (auxDataEnumerator == null)
+                    {
+                        auxDataEnumerator = currentEnum;
+                    }
+                    else
+                    {
+                        auxDataEnumerator = auxDataEnumerator.Current.EndTime > currentEnum.Current.EndTime ? currentEnum : auxDataEnumerator;
+                    }
+                }
+            }
+
+            // check if any enumerator is ready to emit
+            if (auxDataEnumerator?.Current != null && _tradeBarAggregator.Current != null)
+            {
+                var auxDataEndTime = auxDataEnumerator.Current.EndTime.ConvertToUtc(_exchangeTimeZone);
+                var tradeBarEndTime = _tradeBarAggregator.Current.EndTime.ConvertToUtc(_exchangeTimeZone);
+                if (auxDataEndTime < tradeBarEndTime)
+                {
+                    if (auxDataEndTime <= frontierUtc)
+                    {
+                        Current = auxDataEnumerator.Current;
+                        auxDataEnumerator.MoveNext();
+                        return true;
+                    }
+                }
+                else
+                {
+                    if (tradeBarEndTime <= frontierUtc)
+                    {
+                        Current = _tradeBarAggregator.Current;
+                        _tradeBarAggregator.MoveNext();
+                        return true;
+                    }
+                }
+            }
+            else if (auxDataEnumerator?.Current != null)
+            {
+                var auxDataEndTime = auxDataEnumerator.Current.EndTime.ConvertToUtc(_exchangeTimeZone);
+                if (auxDataEndTime <= frontierUtc)
+                {
+                    Current = auxDataEnumerator.Current;
+                    auxDataEnumerator.MoveNext();
+                    return true;
+                }
+            }
+            else if (_tradeBarAggregator.Current != null)
+            {
+                var tradeBarEndTime = _tradeBarAggregator.Current.EndTime.ConvertToUtc(_exchangeTimeZone);
+                if (tradeBarEndTime <= frontierUtc)
+                {
+                    Current = _tradeBarAggregator.Current;
+                    _tradeBarAggregator.MoveNext();
+                    return true;
+                }
+            }
+
+            return false;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveDelistingEventProvider.cs

@@ -0,0 +1,54 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Delisting event provider implementation which will source the delisting date based on new map files
+    /// </summary>
+    public class LiveDelistingEventProvider : DelistingEventProvider
+    {
+        /// <summary>
+        /// Check for delistings
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New delisting event if any</returns>
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            var currentInstance = MapFile;
+            // refresh map file instance
+            InitializeMapFile();
+            var newInstance = MapFile;
+
+            if (currentInstance?.LastOrDefault()?.Date != newInstance?.LastOrDefault()?.Date)
+            {
+                // All future and option contracts sharing the same canonical symbol, share the same configuration too. Thus, in
+                // order to reduce logs, we log the configuration using the canonical symbol. See the optional parameter
+                // "overrideMessageFloodProtection" in Log.Trace() method for more information
+                var symbol = Config.Symbol.HasCanonical() ? Config.Symbol.Canonical.Value : Config.Symbol.Value;
+                Log.Trace($"LiveDelistingEventProvider({Config.ToString(symbol)}): new tradable date {eventArgs.Date:yyyyMMdd}. " +
+                    $"MapFile.LastDate Old: {currentInstance?.LastOrDefault()?.Date:yyyyMMdd} New: {newInstance?.LastOrDefault()?.Date:yyyyMMdd}");
+            }
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveDividendEventProvider.cs

@@ -0,0 +1,57 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    /// <remarks>Only special behavior is that it will refresh factor file on each new tradable date event</remarks>
+    public class LiveDividendEventProvider : DividendEventProvider
+    {
+        /// <summary>
+        /// Check for dividends and returns them
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New Dividend event if any</returns>
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            var currentInstance = FactorFile;
+            // refresh factor file instance
+            InitializeFactorFile();
+            var newInstance = FactorFile;
+
+            if (currentInstance?.Count() != newInstance?.Count())
+            {
+                // All future and option contracts sharing the same canonical symbol, share the same configuration too. Thus, in
+                // order to reduce logs, we log the configuration using the canonical symbol. See the optional parameter
+                // "overrideMessageFloodProtection" in Log.Trace() method for more information
+                var symbol = Config.Symbol.HasCanonical() ? Config.Symbol.Canonical.Value : Config.Symbol.Value;
+                Log.Trace($"LiveDividendEventProvider({Config.ToString(symbol)}): new tradable date {eventArgs.Date:yyyyMMdd}. " +
+                    $"New FactorFile: {!ReferenceEquals(currentInstance, newInstance)}. " +
+                    $"FactorFile.Count Old: {currentInstance?.Count()} New: {newInstance?.Count()}");
+            }
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveFillForwardEnumerator.cs

@@ -0,0 +1,157 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// An implementation of the <see cref="FillForwardEnumerator"/> that uses an <see cref="ITimeProvider"/>
+    /// to determine if a fill forward bar needs to be emitted
+    /// </summary>
+    public class LiveFillForwardEnumerator : FillForwardEnumerator
+    {
+        private readonly TimeSpan _dataResolution;
+        private readonly TimeSpan _underlyingTimeout;
+        private readonly ITimeProvider _timeProvider;
+
+        private TimeSpan _marketCloseTimeSpan;
+        private TimeSpan _marketOpenTimeSpan;
+        private DateTime _lastDate;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LiveFillForwardEnumerator"/> class that accepts
+        /// a reference to the fill forward resolution, useful if the fill forward resolution is dynamic
+        /// and changing as the enumeration progresses
+        /// </summary>
+        /// <param name="timeProvider">The source of time used to gauage when this enumerator should emit extra bars when
+        /// null data is returned from the source enumerator</param>
+        /// <param name="enumerator">The source enumerator to be filled forward</param>
+        /// <param name="exchange">The exchange used to determine when to insert fill forward data</param>
+        /// <param name="fillForwardResolution">The resolution we'd like to receive data on</param>
+        /// <param name="isExtendedMarketHours">True to use the exchange's extended market hours, false to use the regular market hours</param>
+        /// <param name="subscriptionStartTime">The start time of the subscription</param>
+        /// <param name="subscriptionEndTime">The end time of the subscription, once passing this date the enumerator will stop</param>
+        /// <param name="dataResolution">The source enumerator's data resolution</param>
+        /// <param name="dataTimeZone">Time zone of the underlying source data</param>
+        /// <param name="dailyStrictEndTimeEnabled">True if daily strict end times are enabled</param>
+        /// <param name="dataType">The configuration data type this enumerator is for</param>
+        /// <param name="lastPointTracker">A reference to the last point emitted before this enumerator is first enumerated</param>
+        public LiveFillForwardEnumerator(ITimeProvider timeProvider, IEnumerator<BaseData> enumerator, SecurityExchange exchange, IReadOnlyRef<TimeSpan> fillForwardResolution,
+            bool isExtendedMarketHours, DateTime subscriptionStartTime, DateTime subscriptionEndTime, Resolution dataResolution, DateTimeZone dataTimeZone, bool dailyStrictEndTimeEnabled,
+            Type dataType = null, LastPointTracker lastPointTracker = null)
+            : base(enumerator, exchange, fillForwardResolution, isExtendedMarketHours, subscriptionStartTime, subscriptionEndTime, dataResolution.ToTimeSpan(), dataTimeZone,
+                  dailyStrictEndTimeEnabled, dataType, lastPointTracker)
+        {
+            _timeProvider = timeProvider;
+            _dataResolution = dataResolution.ToTimeSpan();
+            _underlyingTimeout = GetMaximumDataTimeout(dataResolution);
+        }
+
+        /// <summary>
+        /// Determines whether or not fill forward is required, and if true, will produce the new fill forward data
+        /// </summary>
+        /// <param name="fillForwardResolution"></param>
+        /// <param name="previous">The last piece of data emitted by this enumerator</param>
+        /// <param name="next">The next piece of data on the source enumerator, this may be null</param>
+        /// <param name="fillForward">When this function returns true, this will have a non-null value, null when the function returns false</param>
+        /// <returns>True when a new fill forward piece of data was produced and should be emitted by this enumerator</returns>
+        protected override bool RequiresFillForwardData(TimeSpan fillForwardResolution, BaseData previous, BaseData next, out BaseData fillForward)
+        {
+            if (base.RequiresFillForwardData(fillForwardResolution, previous, next, out fillForward))
+            {
+                var underlyingTimeout = TimeSpan.Zero;
+                if (fillForwardResolution >= _dataResolution && ShouldWaitForData(fillForward))
+                {
+                    // we enforce the underlying FF timeout when the FF resolution matches it or is bigger, not the other way round, for example:
+                    // this is a daily enumerator and FF resolution is second, we are expected to emit a bar every second, we can't wait until the timeout each time
+                    underlyingTimeout = _underlyingTimeout;
+                }
+
+                var nextEndTimeUtc = (fillForward.EndTime + underlyingTimeout).ConvertToUtc(Exchange.TimeZone);
+                if (next != null || nextEndTimeUtc <= _timeProvider.GetUtcNow())
+                {
+                    // we FF if next is here but in the future or next has not come yet and we've wait enough time
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Helper method to determine if we should wait for data before emitting a fill forward bar.
+        /// We only wait for data if the fill forward bar is either in the market open or close time.
+        /// </summary>
+        private bool ShouldWaitForData(BaseData fillForward)
+        {
+            if (fillForward.Symbol.SecurityType != SecurityType.Equity || Exchange.Hours.IsMarketAlwaysOpen)
+            {
+                return false;
+            }
+
+            // Update market open and close daily
+            if (_lastDate != fillForward.EndTime.Date ||
+                // Update market open and close for days with multiple sessions, e.g. early close and then late open
+                fillForward.Time.TimeOfDay > _marketCloseTimeSpan)
+            {
+                _lastDate = fillForward.EndTime.Date;
+                var marketOpen = Exchange.Hours.GetNextMarketOpen(_lastDate, false);
+                var marketClose = Exchange.Hours.GetNextMarketClose(_lastDate, false);
+
+                if (_dataResolution == Time.OneHour || (_dataResolution == Time.OneDay && !UseStrictEndTime))
+                {
+                    marketOpen = marketOpen.RoundDown(_dataResolution);
+                    marketClose = marketClose.RoundUp(_dataResolution);
+                }
+
+                _marketOpenTimeSpan = marketOpen.TimeOfDay;
+                _marketCloseTimeSpan = marketClose.TimeOfDay;
+            }
+
+            // we only wait for data if the fill forward bar is not in the market open or close time
+            return fillForward.Time.TimeOfDay == _marketOpenTimeSpan || fillForward.EndTime.TimeOfDay == _marketCloseTimeSpan;
+        }
+
+        /// <summary>
+        /// Helper method to know how much we should wait before fill forwarding a bar in live trading
+        /// </summary>
+        /// <remarks>This allows us to create bars taking into account the market auction close and open official prices. Also it will
+        /// allow data providers which might have some delay on creating the bars on their end, to be consumed correctly, when available, by Lean</remarks>
+        public static TimeSpan GetMaximumDataTimeout(Resolution resolution)
+        {
+            switch (resolution)
+            {
+                case Resolution.Tick:
+                    return TimeSpan.Zero;
+                case Resolution.Second:
+                    return TimeSpan.FromSeconds(0.9);
+                case Resolution.Minute:
+                    return TimeSpan.FromMinutes(0.9);
+                case Resolution.Hour:
+                    return TimeSpan.FromMinutes(10);
+                case Resolution.Daily:
+                    return TimeSpan.FromMinutes(10);
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveMappingEventProvider.cs

@@ -0,0 +1,52 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    /// <remarks>Only special behavior is that it will refresh map file on each new tradable date event</remarks>
+    public class LiveMappingEventProvider : MappingEventProvider
+    {
+        /// <summary>
+        /// Check for new mappings
+        /// </summary>
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            var currentInstance = MapFile;
+            // refresh map file instance
+            InitializeMapFile();
+            var newInstance = MapFile;
+
+            // All future and option contracts sharing the same canonical symbol, share the same configuration too. Thus, in
+            // order to reduce logs, we log the configuration using the canonical symbol. See the optional parameter
+            // "overrideMessageFloodProtection" in Log.Trace() method for more information
+            var symbol = Config.Symbol.HasCanonical() ? Config.Symbol.Canonical.Value : Config.Symbol.Value;
+            Log.Trace($"LiveMappingEventProvider({Config.ToString(symbol)}): new tradable date {eventArgs.Date:yyyyMMdd}. " +
+                $"New MapFile: {!ReferenceEquals(currentInstance, newInstance)}. " +
+                $"MapFile.Count Old: {currentInstance?.Count()} New: {newInstance?.Count()}");
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveSplitEventProvider.cs

@@ -0,0 +1,57 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    /// <remarks>Only special behavior is that it will refresh factor file on each new tradable date event</remarks>
+    public class LiveSplitEventProvider : SplitEventProvider
+    {
+        /// <summary>
+        /// Check for dividends and returns them
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New Dividend event if any</returns>
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            var currentInstance = FactorFile;
+            // refresh factor file instance
+            InitializeFactorFile();
+            var newInstance = FactorFile;
+
+            if(currentInstance?.Count() != newInstance?.Count())
+            {
+                // All future and option contracts sharing the same canonical symbol, share the same configuration too. Thus, in
+                // order to reduce logs, we log the configuration using the canonical symbol. See the optional parameter
+                // "overrideMessageFloodProtection" in Log.Trace() method for more information
+                var symbol = Config.Symbol.HasCanonical() ? Config.Symbol.Canonical.Value : Config.Symbol.Value;
+                Log.Trace($"LiveSplitEventProvider({Config.ToString(symbol)}): new tradable date {eventArgs.Date:yyyyMMdd}. " +
+                    $"New FactorFile: {!ReferenceEquals(currentInstance, newInstance)}. " +
+                    $"FactorFile.Count Old: {currentInstance?.Count()} New: {newInstance?.Count()}");
+            }
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveSubscriptionEnumerator.cs

@@ -0,0 +1,120 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using System.Collections;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that will subscribe through the provided data queue handler and refresh the subscription if any mapping occurs
+    /// </summary>
+    public class LiveSubscriptionEnumerator : IEnumerator<BaseData>
+    {
+        private BaseData _current;
+        private readonly Symbol _requestedSymbol;
+        private SubscriptionDataConfig _currentConfig;
+        private IEnumerator<BaseData> _previousEnumerator;
+        private IEnumerator<BaseData> _underlyingEnumerator;
+
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
+        public BaseData Current => _current;
+
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public LiveSubscriptionEnumerator(SubscriptionDataConfig dataConfig, IDataQueueHandler dataQueueHandler, EventHandler handler, Func<SubscriptionDataConfig, bool> isExpired)
+        {
+            _requestedSymbol = dataConfig.Symbol;
+            _underlyingEnumerator = dataQueueHandler.SubscribeWithMapping(dataConfig, handler, isExpired, out _currentConfig);
+
+            // for any mapping event we will re subscribe
+            dataConfig.NewSymbol += (_, _) =>
+            {
+                dataQueueHandler.Unsubscribe(_currentConfig);
+                _previousEnumerator = _underlyingEnumerator;
+
+                var oldSymbol = _currentConfig.Symbol;
+                _underlyingEnumerator = dataQueueHandler.SubscribeWithMapping(dataConfig, handler, isExpired, out _currentConfig);
+
+                Log.Trace($"LiveSubscriptionEnumerator({_requestedSymbol}): " +
+                    $"resubscribing old: '{oldSymbol.Value}' new '{_currentConfig.Symbol.Value}'");
+            };
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element.
+        /// </summary>
+        public bool MoveNext()
+        {
+            if (_previousEnumerator != null)
+            {
+                // if previous is set we dispose of it here since we are the consumers of it
+                _previousEnumerator.DisposeSafely();
+                _previousEnumerator = null;
+            }
+
+            var result = _underlyingEnumerator.MoveNext();
+            if (result)
+            {
+                _current = _underlyingEnumerator.Current;
+            }
+            else
+            {
+                _current = null;
+            }
+
+            if (_current != null && _current.Symbol != _requestedSymbol)
+            {
+                // if we've done some mapping at this layer let's clone the underlying and set the requested symbol,
+                // don't trust the IDQH implementations for data uniqueness, since the configuration could be shared
+                _current = _current.Clone();
+                _current.Symbol = _requestedSymbol;
+            }
+
+            return result;
+        }
+
+        /// <summary>
+        /// Reset the IEnumeration
+        /// </summary>
+        public void Reset()
+        {
+            _underlyingEnumerator.Reset();
+        }
+
+        /// <summary>
+        /// Disposes of the used enumerators
+        /// </summary>
+        public void Dispose()
+        {
+            _previousEnumerator.DisposeSafely();
+            _underlyingEnumerator.DisposeSafely();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/MappingEventProvider.cs

@@ -0,0 +1,102 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    public class MappingEventProvider : ITradableDateEventProvider
+    {
+        private IMapFileProvider _mapFileProvider;
+
+        /// <summary>
+        /// The associated configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; private set; }
+
+        /// <summary>
+        /// The current instance being used
+        /// </summary>
+        protected MapFile MapFile { get; private set; }
+
+        /// <summary>
+        /// Initializes this instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public virtual void Initialize(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime)
+        {
+            _mapFileProvider = mapFileProvider;
+            Config = config;
+            InitializeMapFile();
+
+            if (MapFile.HasData(startTime.Date))
+            {
+                // initialize mapped symbol using request start date
+                Config.MappedSymbol = MapFile.GetMappedSymbol(startTime.Date, Config.MappedSymbol, Config.DataMappingMode);
+            }
+        }
+
+        /// <summary>
+        /// Check for new mappings
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New mapping event if any</returns>
+        public virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            if (Config.Symbol == eventArgs.Symbol
+                && MapFile.HasData(eventArgs.Date))
+            {
+                var old = Config.MappedSymbol;
+                var newSymbol = MapFile.GetMappedSymbol(eventArgs.Date, Config.MappedSymbol, Config.DataMappingMode);
+                Config.MappedSymbol = newSymbol;
+
+                // check to see if the symbol was remapped
+                if (old != Config.MappedSymbol)
+                {
+                    var changed = new SymbolChangedEvent(
+                        Config.Symbol,
+                        eventArgs.Date,
+                        old,
+                        Config.MappedSymbol);
+                    yield return changed;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the map file to use
+        /// </summary>
+        protected void InitializeMapFile()
+        {
+            MapFile = _mapFileProvider.ResolveMapFile(Config);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/NewDataAvailableEventArgs.cs

@@ -0,0 +1,32 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event args for when a new data point is ready to be emitted
+    /// </summary>
+    public class NewDataAvailableEventArgs : EventArgs
+    {
+        /// <summary>
+        /// The new data point
+        /// </summary>
+        public IBaseData DataPoint { get; set; }
+    }
+}

Engine/DataFeeds/Enumerators/PriceScaleFactorEnumerator.cs

@@ -0,0 +1,138 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// This enumerator will update the <see cref="SubscriptionDataConfig.PriceScaleFactor"/> when required
+    /// and adjust the raw <see cref="BaseData"/> prices based on the provided <see cref="SubscriptionDataConfig"/>.
+    /// Assumes the prices of the provided <see cref="IEnumerator"/> are in raw mode.
+    /// </summary>
+    public class PriceScaleFactorEnumerator : IEnumerator<BaseData>
+    {
+        private readonly IEnumerator<BaseData> _rawDataEnumerator;
+        private readonly SubscriptionDataConfig _config;
+        private readonly IFactorFileProvider _factorFileProvider;
+        private DateTime _nextTradableDate;
+        private IFactorProvider _factorFile;
+        private bool _liveMode;
+        private DateTime? _endDate;
+
+        /// <summary>
+        /// Explicit interface implementation for <see cref="Current"/>
+        /// </summary>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Last read <see cref="BaseData"/> object from this type and source
+        /// </summary>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Creates a new instance of the <see cref="PriceScaleFactorEnumerator"/>.
+        /// </summary>
+        /// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/> to enumerate for.
+        /// Will determine the <see cref="DataNormalizationMode"/> to use.</param>
+        /// <param name="factorFileProvider">The <see cref="IFactorFileProvider"/> instance to use</param>
+        /// <param name="liveMode">True, is this is a live mode data stream</param>
+        /// <param name="endDate">The enumerator end date</param>
+        /// <remarks>
+        /// For <see cref="DataNormalizationMode.ScaledRaw"/> normalization mode,
+        /// the prices are scaled to the prices on the <paramref name="endDate"/>
+        /// </remarks>
+        public PriceScaleFactorEnumerator(
+            IEnumerator<BaseData> rawDataEnumerator,
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            bool liveMode = false,
+            DateTime? endDate = null)
+        {
+            _config = config;
+            _liveMode = liveMode;
+            _nextTradableDate = DateTime.MinValue;
+            _rawDataEnumerator = rawDataEnumerator;
+            _factorFileProvider = factorFileProvider;
+            _endDate = endDate;
+        }
+
+        /// <summary>
+        /// Dispose of the underlying enumerator.
+        /// </summary>
+        public void Dispose()
+        {
+            _rawDataEnumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// True if the enumerator was successfully advanced to the next element;
+        /// False if the enumerator has passed the end of the collection.
+        /// </returns>
+        public bool MoveNext()
+        {
+            var underlyingReturnValue = _rawDataEnumerator.MoveNext();
+            Current = _rawDataEnumerator.Current;
+
+            if (underlyingReturnValue
+                && Current != null
+                && _factorFileProvider != null
+                && _config.DataNormalizationMode != DataNormalizationMode.Raw)
+            {
+                var priceScaleFrontier = Current.GetUpdatePriceScaleFrontier();
+                if (priceScaleFrontier >= _nextTradableDate)
+                {
+                    _factorFile = _factorFileProvider.Get(_config.Symbol);
+                    _config.PriceScaleFactor = _factorFile.GetPriceScale(priceScaleFrontier.Date, _config.DataNormalizationMode, _config.ContractDepthOffset, _config.DataMappingMode, _endDate);
+
+                    // update factor files every day
+                    _nextTradableDate = priceScaleFrontier.Date.AddDays(1);
+                    if (_liveMode)
+                    {
+                        // in live trading we add a offset to make sure new factor files are available
+                        _nextTradableDate = _nextTradableDate.Add(Time.LiveAuxiliaryDataOffset);
+                    }
+                }
+
+                Current = Current.Normalize(_config.PriceScaleFactor, _config.DataNormalizationMode, _config.SumOfDividends);
+            }
+
+            return underlyingReturnValue;
+        }
+
+        /// <summary>
+        /// Reset the IEnumeration
+        /// </summary>
+        /// <remarks>Not used</remarks>
+        public void Reset()
+        {
+            throw new NotImplementedException("Reset method not implemented. Assumes loop will only be used once.");
+        }
+    }
+}

Engine/DataFeeds/Enumerators/QuoteBarFillForwardEnumerator.cs

@@ -0,0 +1,113 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// The QuoteBarFillForwardEnumerator wraps an existing base data enumerator
+    /// If the current QuoteBar has null Bid and/or Ask bars, it copies them from the previous QuoteBar
+    /// </summary>
+    public class QuoteBarFillForwardEnumerator : IEnumerator<BaseData>
+    {
+        private QuoteBar _previous;
+        private readonly IEnumerator<BaseData> _enumerator;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FillForwardEnumerator"/> class
+        /// </summary>
+        public QuoteBarFillForwardEnumerator(IEnumerator<BaseData> enumerator)
+        {
+            _enumerator = enumerator;
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception>
+        public bool MoveNext()
+        {
+            if (!_enumerator.MoveNext()) return false;
+
+            var bar = _enumerator.Current as QuoteBar;
+            if (bar != null)
+            {
+                if (_previous != null)
+                {
+                    if (bar.Bid == null)
+                    {
+                        bar.Bid = _previous.Bid;
+                    }
+
+                    if (bar.Ask == null)
+                    {
+                        bar.Ask = _previous.Ask;
+                    }
+                }
+
+                _previous = bar;
+            }
+
+            Current = _enumerator.Current;
+
+            return true;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/RateLimitEnumerator.cs

@@ -0,0 +1,128 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides augmentation of how often an enumerator can be called. Time is measured using
+    /// an <see cref="ITimeProvider"/> instance and calls to the underlying enumerator are limited
+    /// to a minimum time between each call.
+    /// </summary>
+    public class RateLimitEnumerator<T> : IEnumerator<T>
+    {
+        private T _current;
+        private DateTime _lastCallTime;
+
+        private readonly ITimeProvider _timeProvider;
+        private readonly IEnumerator<T> _enumerator;
+        private readonly TimeSpan _minimumTimeBetweenCalls;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RateLimitEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to place rate limits on</param>
+        /// <param name="timeProvider">Time provider used for determing the time between calls</param>
+        /// <param name="minimumTimeBetweenCalls">The minimum time allowed between calls to the underlying enumerator</param>
+        public RateLimitEnumerator(IEnumerator<T> enumerator, ITimeProvider timeProvider, TimeSpan minimumTimeBetweenCalls)
+        {
+            _enumerator = enumerator;
+            _timeProvider = timeProvider;
+            _minimumTimeBetweenCalls = minimumTimeBetweenCalls;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            // determine time since last successful call, do this on units of the minimum time
+            // this will give us nice round emit times
+            var currentTime = _timeProvider.GetUtcNow().RoundDown(_minimumTimeBetweenCalls);
+            var timeBetweenCalls = currentTime - _lastCallTime;
+
+            // if within limits, patch it through to move next
+            if (timeBetweenCalls >= _minimumTimeBetweenCalls)
+            {
+                if (!_enumerator.MoveNext())
+                {
+                    // our underlying is finished
+                    _current = default(T);
+                    return false;
+                }
+
+                // only update last call time on non rate limited requests
+                _lastCallTime = currentTime;
+                _current = _enumerator.Current;
+            }
+            else
+            {
+                // we've been rate limitted
+                _current = default(T);
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/RefreshEnumerator.cs

@@ -0,0 +1,135 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using QuantConnect.Util;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="IEnumerator{T}"/> that will
+    /// always return true via MoveNext.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public class RefreshEnumerator<T> : IEnumerator<T>
+    {
+        private T _current;
+        private IEnumerator<T> _enumerator;
+        private readonly Func<IEnumerator<T>> _enumeratorFactory;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RefreshEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="enumeratorFactory">Enumerator factory used to regenerate the underlying
+        /// enumerator when it ends</param>
+        public RefreshEnumerator(Func<IEnumerator<T>> enumeratorFactory)
+        {
+            _enumeratorFactory = enumeratorFactory;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            if (_enumerator == null)
+            {
+                _enumerator = _enumeratorFactory.Invoke();
+            }
+
+            var moveNext = false;
+            try
+            {
+                moveNext = _enumerator.MoveNext();
+                if (moveNext)
+                {
+                    _current = _enumerator.Current;
+                }
+            }
+            catch (IOException exception)
+            {
+                // we will ignore stale file handle exceptions and retry instead, enumerator will be refreshed
+                if (exception.Message == null || !exception.Message.Contains("Stale file handle", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    throw;
+                }
+            }
+
+            if (!moveNext)
+            {
+                _enumerator.DisposeSafely();
+
+                _enumerator = null;
+                _current = default(T);
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            if (_enumerator != null)
+            {
+                _enumerator.Reset();
+            }
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            if (_enumerator != null)
+            {
+                _enumerator.Dispose();
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/ScannableEnumerator.cs

@@ -0,0 +1,174 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Data.Consolidators;
+using System;
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// An implementation of <see cref="IEnumerator{T}"/> that relies on "consolidated" data
+    /// </summary>
+    /// <typeparam name="T">The item type yielded by the enumerator</typeparam>
+    public class ScannableEnumerator<T> : IEnumerator<T> where T : class, IBaseData
+    {
+        private T _current;
+        private bool _consolidated;
+        private bool _isPeriodBase;
+        private bool _validateInputType;
+        private Type _consolidatorInputType;
+        private readonly DateTimeZone _timeZone;
+        private readonly ConcurrentQueue<T> _queue;
+        private readonly ITimeProvider _timeProvider;
+        private readonly EventHandler _newDataAvailableHandler;
+        private readonly IDataConsolidator _consolidator;
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current => _current;
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ScannableEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="consolidator">Consolidator taking BaseData updates and firing events containing new 'consolidated' data</param>
+        /// <param name="timeZone">The time zone the raw data is time stamped in</param>
+        /// <param name="timeProvider">The time provider instance used to determine when bars are completed and can be emitted</param>
+        /// <param name="newDataAvailableHandler">The event handler for a new available data point</param>
+        /// <param name="isPeriodBased">The consolidator is period based, this will enable scanning on <see cref="MoveNext"/></param>
+        public ScannableEnumerator(IDataConsolidator consolidator, DateTimeZone timeZone, ITimeProvider timeProvider, EventHandler newDataAvailableHandler, bool isPeriodBased = true)
+        {
+            _timeZone = timeZone;
+            _timeProvider = timeProvider;
+            _consolidator = consolidator;
+            _isPeriodBase = isPeriodBased;
+            _queue = new ConcurrentQueue<T>();
+            _consolidatorInputType = consolidator.InputType;
+            _validateInputType = _consolidatorInputType != typeof(BaseData);
+            _newDataAvailableHandler = newDataAvailableHandler ?? ((s, e) => { });
+
+            _consolidator.DataConsolidated += DataConsolidatedHandler;
+        }
+
+        /// <summary>
+        /// Updates the consolidator
+        /// </summary>
+        /// <param name="data">The data to consolidate</param>
+        public void Update(T data)
+        {
+            // if the input type of the consolidator isn't generic we validate it's correct before sending it in
+            if (_validateInputType && data.GetType() != _consolidatorInputType)
+            {
+                return;
+            }
+
+            if (_isPeriodBase)
+            {
+                // we only need to lock if it's period base since the move next call could trigger a scan
+                lock (_consolidator)
+                {
+                    _consolidator.Update(data);
+                }
+            }
+            else
+            {
+                _consolidator.Update(data);
+            }
+        }
+
+        /// <summary>
+        /// Enqueues the new data into this enumerator
+        /// </summary>
+        /// <param name="data">The data to be enqueued</param>
+        private void Enqueue(T data)
+        {
+            _queue.Enqueue(data);
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            if (!_queue.TryDequeue(out _current) && _isPeriodBase)
+            {
+                _consolidated = false;
+                lock (_consolidator)
+                {
+                    // if there is a working bar we will try to pull it out if the time is right, each consolidator knows when it's right
+                    var localTime = _timeProvider.GetUtcNow().ConvertFromUtc(_timeZone);
+                    _consolidator.Scan(localTime);
+                }
+
+                if (_consolidated)
+                {
+                    _queue.TryDequeue(out _current);
+                }
+            }
+
+            // even if we don't have data to return, we haven't technically
+            // passed the end of the collection, so always return true until
+            // the enumerator is explicitly disposed or ended
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _consolidator.DataConsolidated -= DataConsolidatedHandler;
+        }
+
+        private void DataConsolidatedHandler(object sender, IBaseData data)
+        {
+            var dataPoint = data as T;
+            _consolidated = true;
+            Enqueue(dataPoint);
+            _newDataAvailableHandler(sender, new NewDataAvailableEventArgs { DataPoint = dataPoint });
+        }
+    }
+}

Engine/DataFeeds/Enumerators/ScheduledEnumerator.cs

@@ -0,0 +1,197 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NodaTime;
+using QuantConnect.Data;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// This enumerator will filter out data of the underlying enumerator based on a provided schedule.
+    /// Will respect the schedule above the data, meaning will let older data through if the underlying provides none for the schedule date
+    /// </summary>
+    public class ScheduledEnumerator : IEnumerator<BaseData>
+    {
+        private readonly IEnumerator<BaseData> _underlyingEnumerator;
+        private readonly IEnumerator<DateTime> _scheduledTimes;
+        private readonly ITimeProvider _frontierTimeProvider;
+        private readonly DateTimeZone _scheduleTimeZone;
+        private BaseData _underlyingCandidateDataPoint;
+        private bool _scheduledTimesEnded;
+
+        /// <summary>
+        /// The current data point
+        /// </summary>
+        public BaseData Current { get; private set; }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="underlyingEnumerator">The underlying enumerator to filter</param>
+        /// <param name="scheduledTimes">The scheduled times to emit new data points</param>
+        /// <param name="frontierTimeProvider"></param>
+        /// <param name="scheduleTimeZone"></param>
+        /// <param name="startTime">the underlying request start time</param>
+        public ScheduledEnumerator(IEnumerator<BaseData> underlyingEnumerator,
+            IEnumerable<DateTime> scheduledTimes,
+            ITimeProvider frontierTimeProvider,
+            DateTimeZone scheduleTimeZone,
+            DateTime startTime)
+        {
+            _scheduleTimeZone = scheduleTimeZone;
+            _frontierTimeProvider = frontierTimeProvider;
+            _underlyingEnumerator = underlyingEnumerator;
+            _scheduledTimes = scheduledTimes.GetEnumerator();
+            // move our schedule enumerator to current start time
+            MoveScheduleForward(startTime);
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns> True if the enumerator was successfully advanced to the next element;
+        /// false if the enumerator has passed the end of the collection.
+        /// </returns>
+        public bool MoveNext()
+        {
+            if (_scheduledTimesEnded)
+            {
+                Current = null;
+                return false;
+            }
+
+            // lets get our candidate data point to emit
+            if (_underlyingCandidateDataPoint == null)
+            {
+                if (_underlyingEnumerator.Current != null && _underlyingEnumerator.Current.EndTime <= _scheduledTimes.Current)
+                {
+                    _underlyingCandidateDataPoint = _underlyingEnumerator.Current;
+                }
+                else if (Current != null)
+                {
+                    // we will keep the last data point, even if we already emitted it, there could be a case where the user has a schedule in a
+                    // period where there's not new data (or it's far in the future) so let's just FF the previous point
+                    _underlyingCandidateDataPoint = Current.Clone(fillForward: true);
+                }
+            }
+
+            // lets try to get a better candidate
+            if (_underlyingEnumerator.Current == null
+                || _underlyingEnumerator.Current.EndTime < _scheduledTimes.Current)
+            {
+                bool pullAgain;
+                do
+                {
+                    pullAgain = false;
+                    if (!_underlyingEnumerator.MoveNext())
+                    {
+                        if (_underlyingCandidateDataPoint != null)
+                        {
+                            // if we still have a candidate wait till we emit him before stopping
+                            break;
+                        }
+                        Current = null;
+                        return false;
+                    }
+
+                    if (_underlyingEnumerator.Current != null)
+                    {
+                        if (_underlyingEnumerator.Current.EndTime <= _scheduledTimes.Current)
+                        {
+                            // lets try again
+                            pullAgain = true;
+                            // we got another data point which is a newer candidate to emit so let use it instead
+                            // and drop the previous
+                            _underlyingCandidateDataPoint = _underlyingEnumerator.Current;
+                        }
+                        else if (_underlyingCandidateDataPoint == null)
+                        {
+                            // this is the first data point we got and it's After our schedule, let's move our schedule forward
+                            _underlyingCandidateDataPoint = _underlyingEnumerator.Current;
+                            MoveScheduleForward();
+                        }
+                    }
+                } while (pullAgain);
+            }
+
+            if (_underlyingCandidateDataPoint != null
+            // if we are at or past the schedule time we try to emit, in backtest this emits right away, since time is data driven, in live though
+            // we don't emit right away because the underlying might provide us with a newer data point
+                && _scheduledTimes.Current.ConvertToUtc(_scheduleTimeZone) <= GetUtcNow())
+            {
+                Current = _underlyingCandidateDataPoint;
+                // we align the data endtime with the schedule, we respect the schedule above the data time. In backtesting,
+                // time is driven by the data, so let's make sure we emit at the scheduled time even if the data is older
+                Current.EndTime = _scheduledTimes.Current;
+                if (Current.Time > Current.EndTime)
+                {
+                    Current.Time = _scheduledTimes.Current;
+                }
+
+                MoveScheduleForward();
+                _underlyingCandidateDataPoint = null;
+                return true;
+            }
+
+            Current = null;
+            return true;
+        }
+
+        /// <summary>
+        /// Resets the underlying enumerator
+        /// </summary>
+        public void Reset()
+        {
+            _underlyingEnumerator.Reset();
+        }
+
+        /// <summary>
+        /// Disposes of the underlying enumerator
+        /// </summary>
+        public void Dispose()
+        {
+            _scheduledTimes.Dispose();
+            _underlyingEnumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Available in live trading only, in backtesting frontier is driven and sycned already by the data itself
+        /// so we can't hold data here based on it
+        /// </summary>
+        private DateTime GetUtcNow()
+        {
+            if (_frontierTimeProvider != null)
+            {
+                return _frontierTimeProvider.GetUtcNow();
+            }
+            return DateTime.MaxValue;
+        }
+
+        private void MoveScheduleForward(DateTime? frontier = null)
+        {
+            do
+            {
+                _scheduledTimesEnded = !_scheduledTimes.MoveNext();
+            }
+            while (!_scheduledTimesEnded && frontier.HasValue && _scheduledTimes.Current < frontier.Value);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SortEnumerator.cs

@@ -0,0 +1,113 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides an enumerator for sorting collections of <see cref="BaseData"/> objects based on a specified property.
+    /// The sorting occurs lazily, only when enumeration begins.
+    /// </summary>
+    /// <typeparam name="TKey">The type of the key used for sorting.</typeparam>
+    public sealed class SortEnumerator<TKey> : IEnumerator<BaseData>, IDisposable
+    {
+        private readonly IEnumerable<BaseData> _data;
+#pragma warning disable CA2213 // call csutom DisposeSafely() in Dispose()
+        private IEnumerator<BaseData> _sortedEnumerator;
+#pragma warning restore CA2213 // call csutom DisposeSafely() in Dispose()
+        private readonly Func<BaseData, TKey> _keySelector;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SortEnumerator{TKey}"/> class.
+        /// </summary>
+        /// <param name="data">The collection of <see cref="BaseData"/> to enumerate over.</param>
+        /// <param name="keySelector">A function that defines the key to sort by. Defaults to sorting by <see cref="BaseData.EndTime"/>.</param>
+        public SortEnumerator(IEnumerable<BaseData> data, Func<BaseData, TKey> keySelector = null)
+        {
+            _data = data;
+            _sortedEnumerator = GetSortedData().GetEnumerator();
+            _keySelector = keySelector ??= baseData => (TKey)(object)baseData.EndTime;
+        }
+
+        /// <summary>
+        /// Static method to wrap an enumerable with the sort enumerator.
+        /// </summary>
+        /// <param name="preSorted">Indicates if the data is pre-sorted.</param>
+        /// <param name="data">The data to be wrapped into the enumerator.</param>
+        /// <returns>An enumerator over the <see cref="BaseData"/>.</returns>
+        public static IEnumerator<BaseData> TryWrapSortEnumerator(bool preSorted, IEnumerable<BaseData> data)
+        {
+            return preSorted ? new SortEnumerator<TKey>(data) : data.GetEnumerator();
+        }
+
+        /// <summary>
+        /// Lazily retrieves the sorted data.
+        /// </summary>
+        /// <returns>An enumerable collection of <see cref="BaseData"/>.</returns>
+        private IEnumerable<BaseData> GetSortedData()
+        {
+            foreach (var item in _data.OrderBy(_keySelector))
+            {
+                yield return item;
+            }
+        }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Gets the current <see cref="BaseData"/> element in the collection.
+        /// </summary>
+        public BaseData Current
+        {
+            get => _sortedEnumerator.Current;
+        }
+
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// <c>true</c> if the enumerator was successfully advanced to the next element; 
+        /// <c>false</c> if the enumerator has passed the end of the collection.
+        /// </returns>
+        public bool MoveNext()
+        {
+            return _sortedEnumerator.MoveNext();
+        }
+
+        /// <summary>
+        /// Resets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        public void Reset()
+        {
+            _sortedEnumerator = null;
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="SortEnumerator{TKey}"/> and suppresses finalization.
+        /// </summary>
+        public void Dispose()
+        {
+            _sortedEnumerator?.DisposeSafely();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SplitEventProvider.cs

@@ -0,0 +1,121 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="Split"/> events
+    /// </summary>
+    public class SplitEventProvider : ITradableDateEventProvider
+    {
+        // we set the split factor when we encounter a split in the factor file
+        // and on the next trading day we use this data to produce the split instance
+        private decimal? _splitFactor;
+        private decimal _referencePrice;
+        private IFactorFileProvider _factorFileProvider;
+        private MapFile _mapFile;
+
+        /// <summary>
+        /// The current instance being used
+        /// </summary>
+        protected CorporateFactorProvider FactorFile { get; private set; }
+
+        /// <summary>
+        /// The associated configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; private set; }
+
+        /// <summary>
+        /// Initializes this instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public void Initialize(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime)
+        {
+            Config = config;
+            _factorFileProvider = factorFileProvider;
+            _mapFile = mapFileProvider.ResolveMapFile(Config);
+            InitializeFactorFile();
+        }
+
+        /// <summary>
+        /// Check for new splits
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New split event if any</returns>
+        public virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            if (Config.Symbol == eventArgs.Symbol
+                && FactorFile != null
+                && _mapFile.HasData(eventArgs.Date))
+            {
+                var factor = _splitFactor;
+                if (factor != null)
+                {
+                    var close = _referencePrice;
+                    if (close == 0)
+                    {
+                        throw new InvalidOperationException($"Zero reference price for {Config.Symbol} split at {eventArgs.Date}");
+                    }
+
+                    _splitFactor = null;
+                    _referencePrice = 0;
+                    yield return new Split(
+                        eventArgs.Symbol,
+                        eventArgs.Date,
+                        close,
+                        factor.Value,
+                        SplitType.SplitOccurred);
+                }
+
+                decimal splitFactor;
+                decimal referencePrice;
+                if (FactorFile.HasSplitEventOnNextTradingDay(eventArgs.Date, out splitFactor, out referencePrice))
+                {
+                    _splitFactor = splitFactor;
+                    _referencePrice = referencePrice;
+                    yield return new Split(
+                        eventArgs.Symbol,
+                        eventArgs.Date,
+                        eventArgs.LastRawPrice ?? 0,
+                        splitFactor,
+                        SplitType.Warning);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the factor file to use
+        /// </summary>
+        protected void InitializeFactorFile()
+        {
+            FactorFile = _factorFileProvider.Get(Config.Symbol) as CorporateFactorProvider;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/StrictDailyEndTimesEnumerator.cs

@@ -0,0 +1,95 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using System.Collections;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that will handle adjusting daily strict end times if appropriate
+    /// </summary>
+    public class StrictDailyEndTimesEnumerator : IEnumerator<BaseData>
+    {
+        private readonly DateTime _localStartTime;
+        private readonly SecurityExchangeHours _securityExchange;
+        private readonly IEnumerator<BaseData> _underlying;
+
+        /// <summary>
+        /// Current value of the enumerator
+        /// </summary>
+        public BaseData Current { get; private set; }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public StrictDailyEndTimesEnumerator(IEnumerator<BaseData> underlying, SecurityExchangeHours securityExchangeHours, DateTime localStartTime)
+        {
+            _underlying = underlying;
+            _localStartTime = localStartTime;
+            _securityExchange = securityExchangeHours;
+        }
+
+        /// <summary>
+        /// Move to the next date
+        /// </summary>
+        public bool MoveNext()
+        {
+            Current = null;
+            bool result;
+            do
+            {
+                result = _underlying.MoveNext();
+                if (!result || !LeanData.UseDailyStrictEndTimes(_underlying.Current?.GetType()))
+                {
+                    break;
+                }
+
+                // before setting the strict daily end times, let's clone it because underlying enumerator (SubscriptionDataReader) might be using it
+                var pontentialNewBar = _underlying.Current.Clone();
+                if (LeanData.SetStrictEndTimes(pontentialNewBar, _securityExchange) && pontentialNewBar.EndTime >= _localStartTime)
+                {
+                    Current = pontentialNewBar;
+                    break;
+                }
+            }
+            while (true);
+
+            return result;
+        }
+
+        /// <summary>
+        /// Reset the enumerator
+        /// </summary>
+        public void Reset()
+        {
+            _underlying.Reset();
+        }
+
+        /// <summary>
+        /// Dispose the enumerator
+        /// </summary>
+        public void Dispose()
+        {
+            _underlying.Dispose();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SubscriptionDataEnumerator.cs

@@ -0,0 +1,106 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// An <see cref="IEnumerator{SubscriptionData}"/> which wraps an existing <see cref="IEnumerator{BaseData}"/>.
+    /// </summary>
+    /// <remarks>Using this class is important, versus directly yielding, because we setup the <see cref="Dispose"/> chain</remarks>
+    public class SubscriptionDataEnumerator : IEnumerator<SubscriptionData>
+    {
+        private readonly IEnumerator<BaseData> _enumerator;
+        private readonly SubscriptionDataConfig _configuration;
+        private readonly SecurityExchangeHours _exchangeHours;
+        private readonly TimeZoneOffsetProvider _offsetProvider;
+        private readonly bool _isUniverse;
+        private readonly bool _dailyStrictEndTimeEnabled;
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        public SubscriptionData Current { get; private set; }
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="configuration">The subscription's configuration</param>
+        /// <param name="exchangeHours">The security's exchange hours</param>
+        /// <param name="offsetProvider">The subscription's time zone offset provider</param>
+        /// <param name="enumerator">The underlying data enumerator</param>
+        /// <param name="isUniverse">The subscription is a universe subscription</param>
+        /// <returns>A subscription data enumerator</returns>
+        public SubscriptionDataEnumerator(SubscriptionDataConfig configuration,
+            SecurityExchangeHours exchangeHours,
+            TimeZoneOffsetProvider offsetProvider,
+            IEnumerator<BaseData> enumerator,
+            bool isUniverse,
+            bool dailyStrictEndTimeEnabled)
+        {
+            _enumerator = enumerator;
+            _offsetProvider = offsetProvider;
+            _exchangeHours = exchangeHours;
+            _configuration = configuration;
+            _isUniverse = isUniverse;
+            _dailyStrictEndTimeEnabled = dailyStrictEndTimeEnabled;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>True if the enumerator was successfully advanced to the next element;
+        /// False if the enumerator has passed the end of the collection.</returns>
+        public bool MoveNext()
+        {
+            var result = _enumerator.MoveNext();
+            if (result)
+            {
+                // Use our config filter to see if we should emit this
+                // This currently catches Auxiliary data that we don't want to emit
+                if (_enumerator.Current != null && !_configuration.ShouldEmitData(_enumerator.Current, _isUniverse))
+                {
+                    // We shouldn't emit this data, so we will MoveNext() again.
+                    return MoveNext();
+                }
+
+                Current = SubscriptionData.Create(_dailyStrictEndTimeEnabled, _configuration, _exchangeHours, _offsetProvider, _enumerator.Current, _configuration.DataNormalizationMode);
+            }
+            return result;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SubscriptionFilterEnumerator.cs

@@ -0,0 +1,195 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+using QuantConnect.Securities.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Implements a wrapper around a base data enumerator to provide a final filtering step
+    /// </summary>
+    public class SubscriptionFilterEnumerator : IEnumerator<BaseData>
+    {
+        /// <summary>
+        /// Fired when there's an error executing a user's data filter
+        /// </summary>
+        public event EventHandler<Exception> DataFilterError;
+
+        private readonly bool _liveMode;
+        private readonly Security _security;
+        private readonly DateTime _endTime;
+        private readonly bool _extendedMarketHours;
+        private readonly SecurityExchangeHours _exchangeHours;
+        private readonly ISecurityDataFilter _dataFilter;
+        private readonly IEnumerator<BaseData> _enumerator;
+
+        /// <summary>
+        /// Convenience method to wrap the enumerator and attach the data filter event to log and alery users of errors
+        /// </summary>
+        /// <param name="resultHandler">Result handler reference used to send errors</param>
+        /// <param name="enumerator">The source enumerator to be wrapped</param>
+        /// <param name="security">The security who's data is being enumerated</param>
+        /// <param name="endTime">The end time of the subscription</param>
+        /// <param name="extendedMarketHours">True if extended market hours are enabled</param>
+        /// <param name="liveMode">True if live mode</param>
+        /// <param name="securityExchangeHours">The security exchange hours instance to use</param>
+        /// <returns>A new instance of the <see cref="SubscriptionFilterEnumerator"/> class that has had it's <see cref="DataFilterError"/>
+        /// event subscribed to to send errors to the result handler</returns>
+        public static SubscriptionFilterEnumerator WrapForDataFeed(IResultHandler resultHandler, IEnumerator<BaseData> enumerator, Security security, DateTime endTime, bool extendedMarketHours, bool liveMode,
+            SecurityExchangeHours securityExchangeHours)
+        {
+            var filter = new SubscriptionFilterEnumerator(enumerator, security, endTime, extendedMarketHours, liveMode, securityExchangeHours);
+            filter.DataFilterError += (sender, exception) =>
+            {
+                Log.Error(exception, "WrapForDataFeed");
+                resultHandler.RuntimeError("Runtime error applying data filter. Assuming filter pass: " + exception.Message, exception.StackTrace);
+            };
+            return filter;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SubscriptionFilterEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerator">The source enumerator to be wrapped</param>
+        /// <param name="security">The security containing an exchange and data filter</param>
+        /// <param name="endTime">The end time of the subscription</param>
+        /// <param name="extendedMarketHours">True if extended market hours are enabled</param>
+        /// <param name="liveMode">True if live mode</param>
+        /// <param name="securityExchangeHours">The security exchange hours instance to use</param>
+        public SubscriptionFilterEnumerator(IEnumerator<BaseData> enumerator, Security security, DateTime endTime, bool extendedMarketHours, bool liveMode, SecurityExchangeHours securityExchangeHours)
+        {
+            _liveMode = liveMode;
+            _enumerator = enumerator;
+            _security = security;
+            _endTime = endTime;
+            _exchangeHours = securityExchangeHours;
+            _dataFilter = _security.DataFilter;
+            _extendedMarketHours = extendedMarketHours;
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            while (_enumerator.MoveNext())
+            {
+                var current = _enumerator.Current;
+                if (current != null)
+                {
+                    try
+                    {
+                        // execute user data filters
+                        if (current.DataType != MarketDataType.Auxiliary && !_dataFilter.Filter(_security, current))
+                        {
+                            continue;
+                        }
+                    }
+                    catch (Exception err)
+                    {
+                        OnDataFilterError(err);
+                        continue;
+                    }
+
+                    // verify that the bar is within the exchange's market hours
+                    if (current.DataType != MarketDataType.Auxiliary && !_exchangeHours.IsOpen(current.Time, current.EndTime, _extendedMarketHours))
+                    {
+                        if (_liveMode && !current.IsFillForward)
+                        {
+                            // TODO: replace for setting security.RealTimePrice not to modify security cache data directly
+                            _security.SetMarketPrice(current);
+                        }
+                        continue;
+                    }
+
+                    // make sure we haven't passed the end
+                    if (current.Time > _endTime)
+                    {
+                        return false;
+                    }
+                }
+
+                Current = current;
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Event invocated for the <see cref="DataFilterError"/> event
+        /// </summary>
+        /// <param name="exception">The exception that was thrown when trying to perform data filtering</param>
+        private void OnDataFilterError(Exception exception)
+        {
+            var handler = DataFilterError;
+            if (handler != null) handler(this, exception);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SynchronizingBaseDataEnumerator.cs

@@ -0,0 +1,55 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using QuantConnect.Data;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Represents an enumerator capable of synchronizing other base data enumerators in time.
+    /// This assumes that all enumerators have data time stamped in the same time zone
+    /// </summary>
+    public class SynchronizingBaseDataEnumerator : SynchronizingEnumerator<BaseData>
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingBaseDataEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        public SynchronizingBaseDataEnumerator(params IEnumerator<BaseData>[] enumerators)
+            : this((IEnumerable<IEnumerator>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingBaseDataEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        public SynchronizingBaseDataEnumerator(IEnumerable<IEnumerator> enumerators) : base((IEnumerable<IEnumerator<BaseData>>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Gets the Timestamp for the data
+        /// </summary>
+        protected override DateTime GetInstanceTime(BaseData instance)
+        {
+            return instance.EndTime;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SynchronizingEnumerator.cs

@@ -0,0 +1,204 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Represents an enumerator capable of synchronizing other enumerators of type T in time.
+    /// This assumes that all enumerators have data time stamped in the same time zone
+    /// </summary>
+    public abstract class SynchronizingEnumerator<T> : IEnumerator<T>
+    {
+        private IEnumerator<T> _syncer;
+        private readonly IEnumerator<T>[] _enumerators;
+
+        /// <summary>
+        /// Gets the Timestamp for the data
+        /// </summary>
+        protected abstract DateTime GetInstanceTime(T instance);
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        /// <remark>The type of data we want, for example, <see cref="BaseData"/> or <see cref="Slice"/>, ect...</remark>
+        protected SynchronizingEnumerator(params IEnumerator<T>[] enumerators)
+            : this ((IEnumerable<IEnumerator<T>>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        /// <remark>The type of data we want, for example, <see cref="BaseData"/> or <see cref="Slice"/>, ect...</remark>
+        protected SynchronizingEnumerator(IEnumerable<IEnumerator<T>> enumerators)
+        {
+            _enumerators = enumerators.ToArray();
+            _syncer = GetSynchronizedEnumerator(_enumerators);
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception>
+        public bool MoveNext()
+        {
+            var moveNext =  _syncer.MoveNext();
+            Current = moveNext ? _syncer.Current : default(T);
+            return moveNext;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception>
+        public void Reset()
+        {
+            foreach (var enumerator in _enumerators)
+            {
+                enumerator.Reset();
+            }
+            // don't call syncer.reset since the impl will just throw
+            _syncer = GetSynchronizedEnumerator(_enumerators);
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            foreach (var enumerator in _enumerators)
+            {
+                enumerator.Dispose();
+            }
+            _syncer.Dispose();
+        }
+
+        /// <summary>
+        /// Synchronization system for the enumerator:
+        /// </summary>
+        /// <param name="enumerators"></param>
+        /// <returns></returns>
+        private IEnumerator<T> GetSynchronizedEnumerator(IEnumerator<T>[] enumerators)
+        {
+            return GetBruteForceMethod(enumerators);
+        }
+
+        /// <summary>
+        /// Brute force implementation for synchronizing the enumerator.
+        /// Will remove enumerators returning false to the call to MoveNext.
+        /// Will not remove enumerators with Current Null returning true to the call to MoveNext
+        /// </summary>
+        private IEnumerator<T> GetBruteForceMethod(IEnumerator<T>[] enumerators)
+        {
+            var ticks = DateTime.MaxValue.Ticks;
+            var collection = new HashSet<IEnumerator<T>>();
+            foreach (var enumerator in enumerators)
+            {
+                if (enumerator.MoveNext())
+                {
+                    if (enumerator.Current != null)
+                    {
+                        ticks = Math.Min(ticks, GetInstanceTime(enumerator.Current).Ticks);
+                    }
+                    collection.Add(enumerator);
+                }
+                else
+                {
+                    enumerator.Dispose();
+                }
+            }
+
+            var frontier = new DateTime(ticks);
+            var toRemove = new List<IEnumerator<T>>();
+            while (collection.Count > 0)
+            {
+                var nextFrontierTicks = DateTime.MaxValue.Ticks;
+                foreach (var enumerator in collection)
+                {
+                    while (enumerator.Current == null || GetInstanceTime(enumerator.Current) <= frontier)
+                    {
+                        if (enumerator.Current != null)
+                        {
+                            yield return enumerator.Current;
+                        }
+                        if (!enumerator.MoveNext())
+                        {
+                            toRemove.Add(enumerator);
+                            break;
+                        }
+                        if (enumerator.Current == null)
+                        {
+                            break;
+                        }
+                    }
+
+                    if (enumerator.Current != null)
+                    {
+                        nextFrontierTicks = Math.Min(nextFrontierTicks, GetInstanceTime(enumerator.Current).Ticks);
+                    }
+                }
+
+                if (toRemove.Count > 0)
+                {
+                    foreach (var enumerator in toRemove)
+                    {
+                        collection.Remove(enumerator);
+                    }
+                    toRemove.Clear();
+                }
+
+                frontier = new DateTime(nextFrontierTicks);
+                if (frontier == DateTime.MaxValue)
+                {
+                    break;
+                }
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SynchronizingSliceEnumerator.cs

@@ -0,0 +1,55 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using QuantConnect.Data;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Represents an enumerator capable of synchronizing other slice enumerators in time.
+    /// This assumes that all enumerators have data time stamped in the same time zone
+    /// </summary>
+    public class SynchronizingSliceEnumerator : SynchronizingEnumerator<Slice>
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingSliceEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        public SynchronizingSliceEnumerator(params IEnumerator<Slice>[] enumerators)
+            : this((IEnumerable<IEnumerator>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingSliceEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        public SynchronizingSliceEnumerator(IEnumerable<IEnumerator> enumerators) : base((IEnumerable<IEnumerator<Slice>>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Gets the Timestamp for the data
+        /// </summary>
+        protected override DateTime GetInstanceTime(Slice instance)
+        {
+            return instance.UtcTime;
+        }
+    }
+}

Engine/DataFeeds/FileSystemDataFeed.cs

@@ -0,0 +1,307 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Data.Fundamental;
+using QuantConnect.Data.Market;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Logging;
+using QuantConnect.Packets;
+using QuantConnect.Securities;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Historical datafeed stream reader for processing files on a local disk.
+    /// </summary>
+    /// <remarks>Filesystem datafeeds are incredibly fast</remarks>
+    public class FileSystemDataFeed : IDataFeed
+    {
+        private IAlgorithm _algorithm;
+        private ITimeProvider _timeProvider;
+        private IResultHandler _resultHandler;
+        private IMapFileProvider _mapFileProvider;
+        private IFactorFileProvider _factorFileProvider;
+        private IDataProvider _dataProvider;
+        private IDataCacheProvider _cacheProvider;
+        private SubscriptionCollection _subscriptions;
+        private MarketHoursDatabase _marketHoursDatabase;
+        private SubscriptionDataReaderSubscriptionEnumeratorFactory _subscriptionFactory;
+
+        /// <summary>
+        /// Flag indicating the hander thread is completely finished and ready to dispose.
+        /// </summary>
+        public bool IsActive { get; private set; }
+
+        /// <summary>
+        /// Initializes the data feed for the specified job and algorithm
+        /// </summary>
+        public virtual void Initialize(IAlgorithm algorithm,
+            AlgorithmNodePacket job,
+            IResultHandler resultHandler,
+            IMapFileProvider mapFileProvider,
+            IFactorFileProvider factorFileProvider,
+            IDataProvider dataProvider,
+            IDataFeedSubscriptionManager subscriptionManager,
+            IDataFeedTimeProvider dataFeedTimeProvider,
+            IDataChannelProvider dataChannelProvider)
+        {
+            _algorithm = algorithm;
+            _resultHandler = resultHandler;
+            _mapFileProvider = mapFileProvider;
+            _factorFileProvider = factorFileProvider;
+            _dataProvider = dataProvider;
+            _timeProvider = dataFeedTimeProvider.FrontierTimeProvider;
+            _subscriptions = subscriptionManager.DataFeedSubscriptions;
+            _cacheProvider = new ZipDataCacheProvider(dataProvider, isDataEphemeral: false);
+            _subscriptionFactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(
+                _resultHandler,
+                _mapFileProvider,
+                _factorFileProvider,
+                _cacheProvider,
+                algorithm,
+                enablePriceScaling: false);
+
+            IsActive = true;
+            _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
+        }
+
+        /// <summary>
+        /// Creates a file based data enumerator for the given subscription request
+        /// </summary>
+        /// <remarks>Protected so it can be used by the <see cref="LiveTradingDataFeed"/> to warmup requests</remarks>
+        protected IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, Resolution? fillForwardResolution = null,
+            LastPointTracker lastPointTracker = null, bool isWarmUp = false)
+        {
+            return request.IsUniverseSubscription ? CreateUniverseEnumerator(request) : CreateDataEnumerator(request, fillForwardResolution, lastPointTracker, isWarmUp);
+        }
+
+        private IEnumerator<BaseData> CreateDataEnumerator(SubscriptionRequest request, Resolution? fillForwardResolution, LastPointTracker lastPointTracker, bool isWarmUp)
+        {
+            // ReSharper disable once PossibleMultipleEnumeration
+            var enumerator = _subscriptionFactory.CreateEnumerator(request, _dataProvider);
+            enumerator = ConfigureEnumerator(request, false, enumerator, fillForwardResolution, lastPointTracker, isWarmUp);
+
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Creates a new subscription to provide data for the specified security.
+        /// </summary>
+        /// <param name="request">Defines the subscription to be added, including start/end times the universe and security</param>
+        /// <returns>The created <see cref="Subscription"/> if successful, null otherwise</returns>
+        public virtual Subscription CreateSubscription(SubscriptionRequest request)
+        {
+            IEnumerator<BaseData> enumerator;
+            if(_algorithm.IsWarmingUp)
+            {
+                var pivotTimeUtc = _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
+
+                var lastPointTracker = new LastPointTracker();
+
+                var warmupRequest = new SubscriptionRequest(request, endTimeUtc: pivotTimeUtc,
+                    configuration: new SubscriptionDataConfig(request.Configuration, resolution: _algorithm.Settings.WarmupResolution));
+                IEnumerator<BaseData> warmupEnumerator = null;
+                if (warmupRequest.TradableDaysInDataTimeZone.Any()
+                    // since we change the resolution, let's validate it's still valid configuration (example daily equity quotes are not!)
+                    && LeanData.IsValidConfiguration(warmupRequest.Configuration.SecurityType, warmupRequest.Configuration.Resolution, warmupRequest.Configuration.TickType))
+                {
+                    // let them overlap a day if possible to avoid data gaps since each request will FFed it's own since they are different resolutions
+                    pivotTimeUtc = Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours,
+                        _algorithm.StartDate.ConvertTo(_algorithm.TimeZone, request.Security.Exchange.TimeZone),
+                        Time.OneDay,
+                        1,
+                        false,
+                        warmupRequest.Configuration.DataTimeZone,
+                        LeanData.UseDailyStrictEndTimes(_algorithm.Settings, request, request.Security.Symbol, Time.OneDay))
+                        .ConvertToUtc(request.Security.Exchange.TimeZone);
+                    if (pivotTimeUtc < warmupRequest.StartTimeUtc)
+                    {
+                        pivotTimeUtc = warmupRequest.StartTimeUtc;
+                    }
+
+                    warmupEnumerator = CreateEnumerator(warmupRequest, _algorithm.Settings.WarmupResolution, lastPointTracker, true);
+                    // don't let future data past
+                    warmupEnumerator = new FilterEnumerator<BaseData>(warmupEnumerator, data => data == null || data.EndTime <= warmupRequest.EndTimeLocal);
+                }
+
+                var normalEnumerator = CreateEnumerator(new SubscriptionRequest(request, startTimeUtc: pivotTimeUtc), lastPointTracker: lastPointTracker);
+                // don't let pre start data pass, since we adjust start so they overlap 1 day let's not let this data pass, we just want it for fill forwarding after the target start
+                // this is also useful to drop any initial selection point which was already emitted during warmup
+                normalEnumerator = new FilterEnumerator<BaseData>(normalEnumerator, data => data == null || data.EndTime >= warmupRequest.EndTimeLocal);
+
+                // after the warmup enumerator we concatenate the 'normal' one
+                enumerator = new ConcatEnumerator(true, warmupEnumerator, normalEnumerator);
+            }
+            else
+            {
+                enumerator = CreateEnumerator(request);
+            }
+
+            enumerator = AddScheduleWrapper(request, enumerator, null);
+
+            return SubscriptionUtils.CreateAndScheduleWorker(request, enumerator, _factorFileProvider, true, _algorithm.Settings.DailyPreciseEndTime);
+        }
+
+        /// <summary>
+        /// Removes the subscription from the data feed, if it exists
+        /// </summary>
+        /// <param name="subscription">The subscription to remove</param>
+        public virtual void RemoveSubscription(Subscription subscription)
+        {
+        }
+
+        /// <summary>
+        /// Creates a universe enumerator from the Subscription request, the underlying enumerator func and the fill forward resolution (in some cases)
+        /// </summary>
+        protected IEnumerator<BaseData> CreateUniverseEnumerator(SubscriptionRequest request)
+        {
+            ISubscriptionEnumeratorFactory factory = _subscriptionFactory;
+            if (request.Universe is ITimeTriggeredUniverse)
+            {
+                factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, _marketHoursDatabase);
+            }
+            else if (request.Configuration.Type == typeof(FundamentalUniverse))
+            {
+                factory = new BaseDataCollectionSubscriptionEnumeratorFactory(_algorithm.ObjectStore);
+            }
+
+            // define our data enumerator
+            var enumerator = factory.CreateEnumerator(request, _dataProvider);
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Returns a scheduled enumerator from the given arguments. It can also return the given underlying enumerator
+        /// </summary>
+        protected IEnumerator<BaseData> AddScheduleWrapper(SubscriptionRequest request, IEnumerator<BaseData> underlying, ITimeProvider timeProvider)
+        {
+            if (!request.IsUniverseSubscription || !request.Universe.UniverseSettings.Schedule.Initialized)
+            {
+                return underlying;
+            }
+
+            var schedule = request.Universe.UniverseSettings.Schedule.Get(request.StartTimeLocal, request.EndTimeLocal);
+            if (schedule != null)
+            {
+                return new ScheduledEnumerator(underlying, schedule, timeProvider, request.Configuration.ExchangeTimeZone, request.StartTimeLocal);
+            }
+            return underlying;
+        }
+
+        /// <summary>
+        /// Send an exit signal to the thread.
+        /// </summary>
+        public virtual void Exit()
+        {
+            if (IsActive)
+            {
+                IsActive = false;
+                Log.Trace("FileSystemDataFeed.Exit(): Start. Setting cancellation token...");
+                _subscriptionFactory?.DisposeSafely();
+                _cacheProvider.DisposeSafely();
+                Log.Trace("FileSystemDataFeed.Exit(): Exit Finished.");
+            }
+        }
+
+        /// <summary>
+        /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
+        /// </summary>
+        protected IEnumerator<BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator<BaseData> enumerator, Resolution? fillForwardResolution, LastPointTracker lastPointTracker, bool isWarmUpEnumerator = false)
+        {
+            if (aggregate)
+            {
+                enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
+            }
+
+            enumerator = TryAddFillForwardEnumerator(request, enumerator, request.Configuration.FillDataForward, fillForwardResolution, lastPointTracker);
+
+            // optionally apply exchange/user filters
+            if (request.Configuration.IsFilteredSubscription)
+            {
+                enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security,
+                    request.EndTimeLocal, request.Configuration.ExtendedMarketHours, false, request.ExchangeHours);
+            }
+
+            enumerator = ConfigureLastPointTracker(enumerator, lastPointTracker, isWarmUpEnumerator);
+
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Configures the enumerator to track the last data point, if requested, and if this is a warmup enumerator
+        /// </summary>
+        protected IEnumerator<BaseData> ConfigureLastPointTracker(IEnumerator<BaseData> enumerator, LastPointTracker lastPointTracker, bool isWarmUpEnumerator)
+        {
+            if (lastPointTracker != null && isWarmUpEnumerator)
+            {
+                enumerator = new FilterEnumerator<BaseData>(enumerator,
+                    data =>
+                    {
+                        lastPointTracker.LastDataPoint = data;
+                        return true;
+                    });
+            }
+
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Will add a fill forward enumerator if requested
+        /// </summary>
+        protected IEnumerator<BaseData> TryAddFillForwardEnumerator(SubscriptionRequest request, IEnumerator<BaseData> enumerator, bool fillForward, Resolution? fillForwardResolution, LastPointTracker lastPointTracker = null)
+        {
+            // optionally apply fill forward logic, but never for tick data
+            if (fillForward && request.Configuration.Resolution != Resolution.Tick)
+            {
+                // copy forward Bid/Ask bars for QuoteBars
+                if (request.Configuration.Type == typeof(QuoteBar))
+                {
+                    enumerator = new QuoteBarFillForwardEnumerator(enumerator);
+                }
+
+                var fillForwardSpan = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
+                if (fillForwardResolution != null && fillForwardResolution != Resolution.Tick)
+                {
+                    // if we are giving a FFspan we use it instead of the collection based one. This is useful during warmup when the warmup resolution has been set
+                    fillForwardSpan = Ref.Create(fillForwardResolution.Value.ToTimeSpan());
+                }
+
+                // Pass the security exchange hours explicitly to avoid using the ones in the request, since
+                // those could be different. e.g. when requests are created for open interest data the exchange
+                // hours are set to always open to avoid OI data being filtered out due to the exchange being closed.
+                // This way we allow OI data to be fill-forwarded to the market close time when strict end times is enabled,
+                // so that OI data is available at the same time as trades and quotes.
+                var useDailyStrictEndTimes = LeanData.UseDailyStrictEndTimes(_algorithm.Settings, request, request.Configuration.Symbol,
+                    request.Configuration.Increment, request.Security.Exchange.Hours);
+                enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, fillForwardSpan,
+                    request.Configuration.ExtendedMarketHours, request.StartTimeLocal, request.EndTimeLocal, request.Configuration.Increment,
+                    request.Configuration.DataTimeZone, useDailyStrictEndTimes, request.Configuration.Type, lastPointTracker);
+            }
+
+            return enumerator;
+        }
+    }
+}

Engine/DataFeeds/FillForwardResolutionChangedEvent.cs

@@ -0,0 +1,36 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Helper class for fill forward resolution change events
+    /// </summary>
+    public class FillForwardResolutionChangedEvent
+    {
+        /// <summary>
+        /// The old fill forward time span
+        /// </summary>
+        public TimeSpan Old { get; set; }
+
+        /// <summary>
+        /// The new fill forward time span
+        /// </summary>
+        public TimeSpan New { get; set; }
+    }
+}

Engine/DataFeeds/IDataFeed.cs

@@ -0,0 +1,70 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.ComponentModel.Composition;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Packets;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Datafeed interface for creating custom datafeed sources.
+    /// </summary>
+    [InheritedExport(typeof(IDataFeed))]
+    public interface IDataFeed
+    {
+        /// <summary>
+        /// Public flag indicator that the thread is still busy.
+        /// </summary>
+        bool IsActive
+        {
+            get;
+        }
+
+        /// <summary>
+        /// Initializes the data feed for the specified job and algorithm
+        /// </summary>
+        void Initialize(IAlgorithm algorithm,
+            AlgorithmNodePacket job,
+            IResultHandler resultHandler,
+            IMapFileProvider mapFileProvider,
+            IFactorFileProvider factorFileProvider,
+            IDataProvider dataProvider,
+            IDataFeedSubscriptionManager subscriptionManager,
+            IDataFeedTimeProvider dataFeedTimeProvider,
+            IDataChannelProvider dataChannelProvider);
+
+        /// <summary>
+        /// Creates a new subscription to provide data for the specified security.
+        /// </summary>
+        /// <param name="request">Defines the subscription to be added, including start/end times the universe and security</param>
+        /// <returns>The created <see cref="Subscription"/> if successful, null otherwise</returns>
+        Subscription CreateSubscription(SubscriptionRequest request);
+
+        /// <summary>
+        /// Removes the subscription from the data feed, if it exists
+        /// </summary>
+        /// <param name="subscription">The subscription to remove</param>
+        void RemoveSubscription(Subscription subscription);
+
+        /// <summary>
+        /// External controller calls to signal a terminate of the thread.
+        /// </summary>
+        void Exit();
+    }
+}

Engine/DataFeeds/IDataFeedSubscriptionManager.cs

@@ -0,0 +1,64 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// DataFeedSubscriptionManager interface will manage the subscriptions for the Data Feed
+    /// </summary>
+    public interface IDataFeedSubscriptionManager
+    {
+        /// <summary>
+        /// Event fired when a new subscription is added
+        /// </summary>
+        event EventHandler<Subscription> SubscriptionAdded;
+
+        /// <summary>
+        /// Event fired when an existing subscription is removed
+        /// </summary>
+        event EventHandler<Subscription> SubscriptionRemoved;
+
+        /// <summary>
+        /// Gets the data feed subscription collection
+        /// </summary>
+        SubscriptionCollection DataFeedSubscriptions { get; }
+
+        /// <summary>
+        /// Get the universe selection instance
+        /// </summary>
+        UniverseSelection UniverseSelection { get; }
+
+        /// <summary>
+        /// Removes the <see cref="Subscription"/>, if it exists
+        /// </summary>
+        /// <param name="configuration">The <see cref="SubscriptionDataConfig"/> of the subscription to remove</param>
+        /// <param name="universe">Universe requesting to remove <see cref="Subscription"/>.
+        /// Default value, null, will remove all universes</param>
+        /// <returns>True if the subscription was successfully removed, false otherwise</returns>
+        bool RemoveSubscription(SubscriptionDataConfig configuration, Universe universe = null);
+
+        /// <summary>
+        /// Adds a new <see cref="Subscription"/> to provide data for the specified security.
+        /// </summary>
+        /// <param name="request">Defines the <see cref="SubscriptionRequest"/> to be added</param>
+        /// <returns>True if the subscription was created and added successfully, false otherwise</returns>
+        bool AddSubscription(SubscriptionRequest request);
+    }
+}

Engine/DataFeeds/IDataFeedTimeProvider.cs

@@ -0,0 +1,34 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Reduced interface which exposes required <see cref="ITimeProvider"/> for <see cref="IDataFeed"/> implementations
+    /// </summary>
+    public interface IDataFeedTimeProvider
+    {
+        /// <summary>
+        /// Continuous UTC time provider
+        /// </summary>
+        ITimeProvider TimeProvider { get; }
+
+        /// <summary>
+        /// Time provider which returns current UTC frontier time
+        /// </summary>
+        ITimeProvider FrontierTimeProvider { get; }
+    }
+}

Engine/DataFeeds/IDataManager.cs

@@ -0,0 +1,29 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// IDataManager is the engines view of the Data Manager.
+    /// </summary>
+    public interface IDataManager
+    {
+        /// <summary>
+        /// Get the universe selection instance
+        /// </summary>
+        UniverseSelection UniverseSelection { get; }
+    }
+}

Engine/DataFeeds/ISubscriptionDataSourceReader.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Represents a type responsible for accepting an input <see cref="SubscriptionDataSource"/>
+    /// and returning an enumerable of the source's <see cref="BaseData"/>
+    /// </summary>
+    public interface ISubscriptionDataSourceReader
+    {
+        /// <summary>
+        /// Event fired when the specified source is considered invalid, this may
+        /// be from a missing file or failure to download a remote source
+        /// </summary>
+        event EventHandler<InvalidSourceEventArgs> InvalidSource;
+
+        /// <summary>
+        /// Reads the specified <paramref name="source"/>
+        /// </summary>
+        /// <param name="source">The source to be read</param>
+        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
+        IEnumerable<BaseData> Read(SubscriptionDataSource source);
+    }
+}

Engine/DataFeeds/ISubscriptionSynchronizer.cs

@@ -0,0 +1,41 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Provides the ability to synchronize subscriptions into time slices
+    /// </summary>
+    public interface ISubscriptionSynchronizer
+    {
+        /// <summary>
+        /// Event fired when a subscription is finished
+        /// </summary>
+        event EventHandler<Subscription> SubscriptionFinished;
+
+        /// <summary>
+        /// Syncs the specified subscriptions. The frontier time used for synchronization is
+        /// managed internally and dependent upon previous synchronization operations.
+        /// </summary>
+        /// <param name="subscriptions">The subscriptions to sync</param>
+        /// <param name="cancellationToken">The cancellation token to stop enumeration</param>
+        IEnumerable<TimeSlice> Sync(IEnumerable<Subscription> subscriptions, CancellationToken cancellationToken);
+    }
+}

Engine/DataFeeds/ISynchronizer.cs

@@ -0,0 +1,32 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections.Generic;
+using System.Threading;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Interface which provides the data to stream to the algorithm
+    /// </summary>
+    public interface ISynchronizer
+    {
+        /// <summary>
+        /// Returns an enumerable which provides the data to stream to the algorithm
+        /// </summary>
+        IEnumerable<TimeSlice> StreamData(CancellationToken cancellationToken);
+    }
+}

Engine/DataFeeds/IndexSubscriptionDataSourceReader.cs

@@ -0,0 +1,121 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// This <see cref="ISubscriptionDataSourceReader"/> implementation supports
+    /// the <see cref="FileFormat.Index"/> and <see cref="IndexedBaseData"/> types.
+    /// Handles the layer of indirection for the index data source and forwards
+    /// the target source to the corresponding <see cref="ISubscriptionDataSourceReader"/>
+    /// </summary>
+    public class IndexSubscriptionDataSourceReader : BaseSubscriptionDataSourceReader
+    {
+        private readonly SubscriptionDataConfig _config;
+        private readonly DateTime _date;
+        private IDataProvider _dataProvider;
+        private readonly IndexedBaseData _factory;
+
+        /// <summary>
+        /// Creates a new instance of this <see cref="ISubscriptionDataSourceReader"/>
+        /// </summary>
+        public IndexSubscriptionDataSourceReader(IDataCacheProvider dataCacheProvider,
+            SubscriptionDataConfig config,
+            DateTime date,
+            bool isLiveMode,
+            IDataProvider dataProvider,
+            IObjectStore objectStore)
+        : base(dataCacheProvider, isLiveMode, objectStore)
+        {
+            _config = config;
+            _date = date;
+            _dataProvider = dataProvider;
+            _factory = config.Type.GetBaseDataInstance() as IndexedBaseData;
+            if (_factory == null)
+            {
+                throw new ArgumentException($"{nameof(IndexSubscriptionDataSourceReader)} should be used" +
+                                            $"with a data type which implements {nameof(IndexedBaseData)}");
+            }
+        }
+
+        /// <summary>
+        /// Reads the specified <paramref name="source"/>
+        /// </summary>
+        /// <param name="source">The source to be read</param>
+        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
+        public override IEnumerable<BaseData> Read(SubscriptionDataSource source)
+        {
+            // handles zip or text files
+            using (var reader = CreateStreamReader(source))
+            {
+                // if the reader doesn't have data then we're done with this subscription
+                if (reader == null || reader.EndOfStream)
+                {
+                    OnInvalidSource(source, new Exception($"The reader was empty for source: ${source.Source}"));
+                    yield break;
+                }
+
+                // while the reader has data
+                while (!reader.EndOfStream)
+                {
+                    // read a line and pass it to the base data factory
+                    var line = reader.ReadLine();
+                    if (line.IsNullOrEmpty())
+                    {
+                        continue;
+                    }
+
+                    SubscriptionDataSource dataSource;
+                    try
+                    {
+                        dataSource = _factory.GetSourceForAnIndex(_config, _date, line, IsLiveMode);
+                    }
+                    catch
+                    {
+                        OnInvalidSource(source, new Exception("Factory.GetSourceForAnIndex() failed to return a valid source"));
+                        yield break;
+                    }
+
+                    if (dataSource != null)
+                    {
+                        var dataReader = SubscriptionDataSourceReader.ForSource(
+                            dataSource,
+                            DataCacheProvider,
+                            _config,
+                            _date,
+                            IsLiveMode,
+                            _factory,
+                            _dataProvider,
+                            ObjectStore);
+
+                        var enumerator = dataReader.Read(dataSource).GetEnumerator();
+                        while (enumerator.MoveNext())
+                        {
+                            yield return enumerator.Current;
+                        }
+                        enumerator.DisposeSafely();
+                    }
+                }
+            }
+        }
+    }
+}

Engine/DataFeeds/InternalSubscriptionManager.cs

@@ -0,0 +1,137 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Class in charge of handling Leans internal subscriptions
+    /// </summary>
+    public class InternalSubscriptionManager
+    {
+        private readonly Dictionary<Symbol, List<SubscriptionRequest>> _subscriptionRequests;
+        private readonly Resolution _resolution;
+        private readonly IAlgorithm _algorithm;
+
+        /// <summary>
+        /// Event fired when a new internal subscription request is to be added
+        /// </summary>
+        public EventHandler<SubscriptionRequest> Added { get; set; }
+
+        /// <summary>
+        /// Event fired when an existing internal subscription should be removed
+        /// </summary>
+        public EventHandler<SubscriptionRequest> Removed { get; set; }
+
+        /// <summary>
+        /// Creates a new instances
+        /// </summary>
+        /// <param name="algorithm">The associated algorithm</param>
+        /// <param name="resolution">The resolution to use for the internal subscriptions</param>
+        public InternalSubscriptionManager(IAlgorithm algorithm, Resolution resolution)
+        {
+            _algorithm = algorithm;
+            _resolution = resolution;
+            _subscriptionRequests = new Dictionary<Symbol, List<SubscriptionRequest>>();
+        }
+
+        /// <summary>
+        /// Notifies about a removed subscription request
+        /// </summary>
+        /// <param name="request">The removed subscription request</param>
+        public void AddedSubscriptionRequest(SubscriptionRequest request)
+        {
+            if (PreFilter(request))
+            {
+                var lowResolution = request.Configuration.Resolution > Resolution.Minute;
+                List<SubscriptionRequest> internalRequests;
+                var existing = _subscriptionRequests.TryGetValue(request.Configuration.Symbol, out internalRequests);
+                var alreadyInternal = existing && internalRequests.Any(internalRequest => internalRequest.Configuration.Type == request.Configuration.Type
+                    && request.Configuration.TickType == internalRequest.Configuration.TickType);
+
+                if (lowResolution && !alreadyInternal)
+                {
+                    // low resolution subscriptions we will add internal Resolution.Minute subscriptions
+                    // if we don't already have this symbol added
+                    var config = new SubscriptionDataConfig(request.Configuration, resolution: _resolution, isInternalFeed: true, extendedHours: true, isFilteredSubscription: false);
+                    var startTimeUtc = request.StartTimeUtc;
+                    if (_algorithm.IsWarmingUp)
+                    {
+                        // during warmup in live trading do not add these internal subscription until the algorithm starts
+                        // these subscription are only added for realtime price in low resolution subscriptions which isn't required for warmup
+                        startTimeUtc = DateTime.UtcNow;
+                    }
+                    var internalRequest = new SubscriptionRequest(false, null, request.Security, config, startTimeUtc, request.EndTimeUtc);
+                    if (existing)
+                    {
+                        _subscriptionRequests[request.Configuration.Symbol].Add(internalRequest);
+                    }
+                    else
+                    {
+                        _subscriptionRequests[request.Configuration.Symbol] = new List<SubscriptionRequest>{ internalRequest };
+                    }
+                    Added?.Invoke(this, internalRequest);
+                }
+                else if (!lowResolution && alreadyInternal)
+                {
+                    _subscriptionRequests.Remove(request.Configuration.Symbol);
+                    // the user added a higher resolution configuration, we can remove the internal we added
+                    foreach (var subscriptionRequest in internalRequests)
+                    {
+                        Removed?.Invoke(this, subscriptionRequest);
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Notifies about an added subscription request
+        /// </summary>
+        /// <param name="request">The added subscription request</param>
+        public void RemovedSubscriptionRequest(SubscriptionRequest request)
+        {
+            if (PreFilter(request) && _subscriptionRequests.ContainsKey(request.Configuration.Symbol))
+            {
+                var userConfigs = _algorithm.SubscriptionManager.SubscriptionDataConfigService
+                    .GetSubscriptionDataConfigs(request.Configuration.Symbol).ToList();
+
+                if (userConfigs.Count == 0 || userConfigs.Any(config => config.Resolution <= Resolution.Minute))
+                {
+                    var requests = _subscriptionRequests[request.Configuration.Symbol];
+                    _subscriptionRequests.Remove(request.Configuration.Symbol);
+                    // if we had a config and the user no longer has a config for this symbol we remove the internal subscription
+                    foreach (var subscriptionRequest in requests)
+                    {
+                        Removed?.Invoke(this, subscriptionRequest);
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// True for for live trading, non internal, non universe subscriptions, non custom data subscriptions
+        /// </summary>
+        private bool PreFilter(SubscriptionRequest request)
+        {
+            return _algorithm.LiveMode && !request.Configuration.IsInternalFeed && !request.IsUniverseSubscription && !request.Configuration.IsCustomData;
+        }
+    }
+}

Engine/DataFeeds/InvalidSourceEventArgs.cs

@@ -0,0 +1,53 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Event arguments for the <see cref="ISubscriptionDataSourceReader.InvalidSource"/> event
+    /// </summary>
+    public sealed class InvalidSourceEventArgs : EventArgs
+    {
+        /// <summary>
+        /// Gets the source that was considered invalid
+        /// </summary>
+        public SubscriptionDataSource Source
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the exception that was encountered
+        /// </summary>
+        public Exception Exception
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="InvalidSourceEventArgs"/> class
+        /// </summary>
+        /// <param name="source">The source that was considered invalid</param>
+        /// <param name="exception">The exception that was encountered</param>
+        public InvalidSourceEventArgs(SubscriptionDataSource source, Exception exception)
+        {
+            Source = source;
+            Exception = exception;
+        }
+    }
+}

Engine/DataFeeds/LiveFutureChainProvider.cs

@@ -0,0 +1,62 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// An implementation of <see cref="IFutureChainProvider"/> that fetches the list of contracts
+    /// from an external source
+    /// </summary>
+    public class LiveFutureChainProvider : BacktestingFutureChainProvider
+    {
+        /// <summary>
+        /// Gets the list of future contracts for a given underlying symbol
+        /// </summary>
+        /// <param name="symbol">The underlying symbol</param>
+        /// <param name="date">The date for which to request the future chain (only used in backtesting)</param>
+        /// <returns>The list of future contracts</returns>
+        public override IEnumerable<Symbol> GetFutureContractList(Symbol symbol, DateTime date)
+        {
+            var result = Enumerable.Empty<Symbol>();
+            try
+            {
+                result = base.GetFutureContractList(symbol, date);
+            }
+            catch (Exception ex)
+            {
+                // this shouldn't happen but just in case let's log it
+                Log.Error(ex);
+            }
+
+            bool yielded = false;
+            foreach (var symbols in result)
+            {
+                yielded = true;
+                yield return symbols;
+            }
+
+            if (!yielded)
+            {
+                throw new NotImplementedException("LiveFutureChainProvider.GetFutureContractList() has not been implemented yet.");
+            }
+        }
+    }
+}

Engine/DataFeeds/LiveOptionChainProvider.cs

@@ -0,0 +1,386 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Net;
+using System.Linq;
+using System.Net.Http;
+using Newtonsoft.Json;
+using System.Threading;
+using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Securities.Future;
+using QuantConnect.Securities.FutureOption;
+using QuantConnect.Securities.FutureOption.Api;
+using System.Net.Http.Headers;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// An implementation of <see cref="IOptionChainProvider"/> that fetches the list of contracts
+    /// from the Options Clearing Corporation (OCC) website
+    /// </summary>
+    public class LiveOptionChainProvider : BacktestingOptionChainProvider
+    {
+        private static readonly HttpClient _client;
+        private static readonly DateTime _epoch = new DateTime(1970, 1, 1);
+
+        private static RateGate _cmeRateGate;
+
+        private const string CMESymbolReplace = "{{SYMBOL}}";
+        private const string CMEProductCodeReplace = "{{PRODUCT_CODE}}";
+        private const string CMEProductExpirationReplace = "{{PRODUCT_EXPIRATION}}";
+
+        private const string CMEProductSlateURL = "https://www.cmegroup.com/CmeWS/mvc/ProductSlate/V2/List?pageNumber=1&sortAsc=false&sortField=rank&searchString=" + CMESymbolReplace + "&pageSize=5";
+        private const string CMEOptionsTradeDateAndExpirations = "https://www.cmegroup.com/CmeWS/mvc/Settlements/Options/TradeDateAndExpirations/" + CMEProductCodeReplace;
+        private const string CMEOptionChainQuotesURL = "https://www.cmegroup.com/CmeWS/mvc/Quotes/Option/" + CMEProductCodeReplace + "/G/" + CMEProductExpirationReplace + "/ALL?_=";
+
+        private const int MaxDownloadAttempts = 5;
+
+        /// <summary>
+        /// Static constructor for the <see cref="LiveOptionChainProvider"/> class
+        /// </summary>
+        static LiveOptionChainProvider()
+        {
+            // The OCC website now requires at least TLS 1.1 for API requests.
+            // NET 4.5.2 and below does not enable these more secure protocols by default, so we add them in here
+            ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
+
+            _client = new HttpClient(new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate });
+            _client.DefaultRequestHeaders.Connection.Add("keep-alive");
+            _client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*", 0.8));
+            _client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0");
+            _client.DefaultRequestHeaders.AcceptLanguage.Add(new StringWithQualityHeaderValue("en-US", 0.5));
+        }
+
+        /// <summary>
+        /// Gets the option chain associated with the underlying Symbol
+        /// </summary>
+        /// <param name="symbol">The option or the underlying symbol to get the option chain for.
+        /// Providing the option allows targetting an option ticker different than the default e.g. SPXW</param>
+        /// <param name="date">The date to ask for the option contract list for</param>
+        /// <returns>Option chain</returns>
+        /// <exception cref="ArgumentException">Option underlying Symbol is not Future or Equity</exception>
+        public override IEnumerable<Symbol> GetOptionContractList(Symbol symbol, DateTime date)
+        {
+            HashSet<Symbol> result = null;
+            try
+            {
+                result = base.GetOptionContractList(symbol, date).ToHashSet();
+            }
+            catch (Exception ex)
+            {
+                result = new();
+                // this shouldn't happen but just in case let's log it
+                Log.Error(ex);
+            }
+
+            // during warmup we rely on the backtesting provider, but as we get closer to current time let's join the data with our live chain sources
+            if (date.Date >= DateTime.UtcNow.Date.AddDays(-5) || result.Count == 0)
+            {
+                var underlyingSymbol = symbol;
+                if (symbol.SecurityType.IsOption())
+                {
+                    // we were given the option
+                    underlyingSymbol = symbol.Underlying;
+                }
+
+                if (underlyingSymbol.SecurityType == SecurityType.Equity || underlyingSymbol.SecurityType == SecurityType.Index)
+                {
+                    var expectedOptionTicker = underlyingSymbol.Value;
+                    if (underlyingSymbol.SecurityType == SecurityType.Index)
+                    {
+                        expectedOptionTicker = symbol.ID.Symbol;
+                    }
+
+                    // Source data from TheOCC if we're trading equity or index options
+                    foreach (var optionSymbol in GetEquityIndexOptionContractList(underlyingSymbol, expectedOptionTicker).Where(symbol => !IsContractExpired(symbol, date)))
+                    {
+                        result.Add(optionSymbol);
+                    }
+                }
+                else if (underlyingSymbol.SecurityType == SecurityType.Future)
+                {
+                    // We get our data from CME if we're trading future options
+                    foreach (var optionSymbol in GetFutureOptionContractList(underlyingSymbol, date).Where(symbol => !IsContractExpired(symbol, date)))
+                    {
+                        result.Add(optionSymbol);
+                    }
+                }
+                else
+                {
+                    throw new ArgumentException("Option Underlying SecurityType is not supported. Supported types are: Equity, Index, Future");
+                }
+            }
+
+            foreach (var optionSymbol in result)
+            {
+                yield return optionSymbol;
+            }
+        }
+
+        private IEnumerable<Symbol> GetFutureOptionContractList(Symbol futureContractSymbol, DateTime date)
+        {
+            var symbols = new List<Symbol>();
+            var retries = 0;
+            var maxRetries = 5;
+
+            // rate gate will start a timer in the background, so let's avoid it we if don't need it
+            _cmeRateGate ??= new RateGate(1, TimeSpan.FromSeconds(0.5));
+
+            while (++retries <= maxRetries)
+            {
+                try
+                {
+                    _cmeRateGate.WaitToProceed();
+
+                    var productResponse = _client.GetAsync(CMEProductSlateURL.Replace(CMESymbolReplace, futureContractSymbol.ID.Symbol))
+                        .SynchronouslyAwaitTaskResult();
+
+                    productResponse.EnsureSuccessStatusCode();
+
+                    var productResults = JsonConvert.DeserializeObject<CMEProductSlateV2ListResponse>(productResponse.Content
+                        .ReadAsStringAsync()
+                        .SynchronouslyAwaitTaskResult());
+
+                    productResponse.Dispose();
+
+                    // We want to gather the future product to get the future options ID
+                    var futureProductId = productResults.Products.Where(p => p.Globex == futureContractSymbol.ID.Symbol && p.GlobexTraded && p.Cleared == "Futures")
+                        .Select(p => p.Id)
+                        .Single();
+
+
+                    var optionsTradesAndExpiries = CMEOptionsTradeDateAndExpirations.Replace(CMEProductCodeReplace, futureProductId.ToStringInvariant());
+
+                    _cmeRateGate.WaitToProceed();
+
+                    var optionsTradesAndExpiriesResponse = _client.GetAsync(optionsTradesAndExpiries).SynchronouslyAwaitTaskResult();
+                    optionsTradesAndExpiriesResponse.EnsureSuccessStatusCode();
+
+                    var tradesAndExpiriesResponse = JsonConvert.DeserializeObject<List<CMEOptionsTradeDatesAndExpiration>>(optionsTradesAndExpiriesResponse.Content
+                        .ReadAsStringAsync()
+                        .SynchronouslyAwaitTaskResult());
+
+                    optionsTradesAndExpiriesResponse.Dispose();
+
+                    // For now, only support American options on CME
+                    var selectedOption = tradesAndExpiriesResponse
+                        .FirstOrDefault(x => !x.Daily && !x.Weekly && !x.Sto && x.OptionType == "AME");
+
+                    if (selectedOption == null)
+                    {
+                        Log.Error($"LiveOptionChainProvider.GetFutureOptionContractList(): Found no matching future options for contract {futureContractSymbol}");
+                        yield break;
+                    }
+
+                    // Gather the month code and the year's last number to query the next API, which expects an expiration as `<MONTH_CODE><YEAR_LAST_NUMBER>`
+                    var expiryFunction = FuturesExpiryFunctions.FuturesExpiryFunction(futureContractSymbol.Canonical);
+
+                    var futureContractExpiration = selectedOption.Expirations
+                        .Select(x => new KeyValuePair<CMEOptionsExpiration, DateTime>(x, expiryFunction(new DateTime(x.Expiration.Year, x.Expiration.Month, 1))))
+                        .FirstOrDefault(x => x.Value.Year == futureContractSymbol.ID.Date.Year && x.Value.Month == futureContractSymbol.ID.Date.Month)
+                        .Key;
+
+                    if (futureContractExpiration == null)
+                    {
+                        Log.Error($"LiveOptionChainProvider.GetFutureOptionContractList(): Found no future options with matching expiry year and month for contract {futureContractSymbol}");
+                        yield break;
+                    }
+
+                    var futureContractMonthCode = futureContractExpiration.Expiration.Code;
+
+                    _cmeRateGate.WaitToProceed();
+
+                    // Subtract one day from now for settlement API since settlement may not be available for today yet
+                    var optionChainQuotesResponseResult = _client.GetAsync(CMEOptionChainQuotesURL
+                        .Replace(CMEProductCodeReplace, selectedOption.ProductId.ToStringInvariant())
+                        .Replace(CMEProductExpirationReplace, futureContractMonthCode)
+                        + Math.Floor((DateTime.UtcNow - _epoch).TotalMilliseconds).ToStringInvariant());
+
+                    optionChainQuotesResponseResult.Result.EnsureSuccessStatusCode();
+
+                    var futureOptionChain = JsonConvert.DeserializeObject<CMEOptionChainQuotes>(optionChainQuotesResponseResult.Result.Content
+                        .ReadAsStringAsync()
+                        .SynchronouslyAwaitTaskResult())
+                        .Quotes
+                        .DistinctBy(s => s.StrikePrice)
+                        .ToList();
+
+                    optionChainQuotesResponseResult.Dispose();
+
+                    // Each CME contract can have arbitrary scaling applied to the strike price, so we normalize it to the
+                    // underlying's price via static entries.
+                    var optionStrikePriceScaleFactor = CMEStrikePriceScalingFactors.GetScaleFactor(futureContractSymbol);
+                    var canonicalOption = Symbol.CreateOption(
+                        futureContractSymbol,
+                        futureContractSymbol.ID.Market,
+                        futureContractSymbol.SecurityType.DefaultOptionStyle(),
+                        default(OptionRight),
+                        default(decimal),
+                        SecurityIdentifier.DefaultDate);
+
+                    foreach (var optionChainEntry in futureOptionChain)
+                    {
+                        var futureOptionExpiry = FuturesOptionsExpiryFunctions.GetFutureOptionExpiryFromFutureExpiry(futureContractSymbol, canonicalOption);
+                        var scaledStrikePrice = optionChainEntry.StrikePrice / optionStrikePriceScaleFactor;
+
+                        // Calls and puts share the same strike, create two symbols per each to avoid iterating twice.
+                        symbols.Add(Symbol.CreateOption(
+                            futureContractSymbol,
+                            futureContractSymbol.ID.Market,
+                            OptionStyle.American,
+                            OptionRight.Call,
+                            scaledStrikePrice,
+                            futureOptionExpiry));
+
+                        symbols.Add(Symbol.CreateOption(
+                            futureContractSymbol,
+                            futureContractSymbol.ID.Market,
+                            OptionStyle.American,
+                            OptionRight.Put,
+                            scaledStrikePrice,
+                            futureOptionExpiry));
+                    }
+
+                    break;
+                }
+                catch (HttpRequestException err)
+                {
+                    if (retries != maxRetries)
+                    {
+                        Log.Error(err, $"Failed to retrieve futures options chain from CME, retrying ({retries} / {maxRetries})");
+                        continue;
+                    }
+
+                    Log.Error(err, $"Failed to retrieve futures options chain from CME, returning empty result ({retries} / {retries})");
+                }
+            }
+
+            foreach (var symbol in symbols)
+            {
+                yield return symbol;
+            }
+        }
+
+        /// <summary>
+        /// Gets the list of option contracts for a given underlying equity symbol
+        /// </summary>
+        /// <param name="symbol">The underlying symbol</param>
+        /// <param name="expectedOptionTicker">The expected option ticker</param>
+        /// <returns>The list of option contracts</returns>
+        private static IEnumerable<Symbol> GetEquityIndexOptionContractList(Symbol symbol, string expectedOptionTicker)
+        {
+            var attempt = 1;
+            IEnumerable<Symbol> contracts;
+
+            while (true)
+            {
+                try
+                {
+                    Log.Trace($"LiveOptionChainProvider.GetOptionContractList(): Fetching option chain for option {expectedOptionTicker} underlying {symbol.Value} [Attempt {attempt}]");
+
+                    contracts = FindOptionContracts(symbol, expectedOptionTicker);
+                    break;
+                }
+                catch (WebException exception)
+                {
+                    Log.Error(exception);
+
+                    if (++attempt > MaxDownloadAttempts)
+                    {
+                        throw;
+                    }
+
+                    Thread.Sleep(1000);
+                }
+            }
+
+            return contracts;
+        }
+
+        /// <summary>
+        /// Retrieve the list of option contracts for an underlying symbol from the OCC website
+        /// </summary>
+        private static IEnumerable<Symbol> FindOptionContracts(Symbol underlyingSymbol, string expectedOptionTicker)
+        {
+            var symbols = new List<Symbol>();
+
+            // use QC url to bypass TLS issues with Mono pre-4.8 version
+            var url = "https://www.quantconnect.com/api/v2/theocc/series-search?symbolType=U&symbol=" + underlyingSymbol.Value;
+
+            // download the text file
+            var fileContent = _client.DownloadData(url);
+
+            // read the lines, skipping the headers
+            var lines = fileContent.Split(new[] { "\r\n" }, StringSplitOptions.None).Skip(7);
+
+            // Example of a line:
+            // SPY		2021	03	26	190	000	C P 	0	612	360000000
+
+            // avoid being sensitive to case
+            expectedOptionTicker = expectedOptionTicker.LazyToUpper();
+
+            var optionStyle = underlyingSymbol.SecurityType.DefaultOptionStyle();
+
+            // parse the lines, creating the Lean option symbols
+            foreach (var line in lines)
+            {
+                var fields = line.Split('\t');
+
+                var ticker = fields[0].Trim();
+                if (ticker != expectedOptionTicker)
+                {
+                    // skip undesired options. For example SPX underlying has SPX & SPXW option tickers
+                    continue;
+                }
+
+                var expiryDate = new DateTime(fields[2].ToInt32(), fields[3].ToInt32(), fields[4].ToInt32());
+                var strike = (fields[5] + "." + fields[6]).ToDecimal();
+
+                foreach (var right in fields[7].Trim().Split(' '))
+                {
+                    OptionRight? targetRight = null;
+
+                    if (right.Equals("C", StringComparison.OrdinalIgnoreCase))
+                    {
+                        targetRight = OptionRight.Call;
+                    }
+                    else if (right.Equals("P", StringComparison.OrdinalIgnoreCase))
+                    {
+                        targetRight = OptionRight.Put;
+                    }
+
+                    if (targetRight.HasValue)
+                    {
+                        symbols.Add(Symbol.CreateOption(
+                            underlyingSymbol,
+                            expectedOptionTicker,
+                            underlyingSymbol.ID.Market,
+                            optionStyle,
+                            targetRight.Value,
+                            strike,
+                            expiryDate));
+                    }
+                }
+            }
+
+            return symbols;
+        }
+    }
+}

Engine/DataFeeds/LiveSynchronizer.cs

@@ -0,0 +1,237 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using QuantConnect.Configuration;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Logging;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Implementation of the <see cref="ISynchronizer"/> interface which provides the mechanism to stream live data to the algorithm
+    /// </summary>
+    public class LiveSynchronizer : Synchronizer
+    {
+        /// <summary>
+        /// Consumer batching timeout in ms
+        /// </summary>
+        public static readonly int BatchingDelay = Config.GetInt("consumer-batching-timeout-ms");
+
+        private ITimeProvider _timeProvider;
+        private LiveTimeProvider _frontierTimeProvider;
+        private RealTimeScheduleEventService _realTimeScheduleEventService;
+        private readonly ManualResetEventSlim _newLiveDataEmitted = new ManualResetEventSlim(false);
+
+        /// <summary>
+        /// Continuous UTC time provider
+        /// </summary>
+        public override ITimeProvider TimeProvider => _timeProvider;
+
+        /// <summary>
+        /// Initializes the instance of the Synchronizer class
+        /// </summary>
+        public override void Initialize(
+            IAlgorithm algorithm,
+            IDataFeedSubscriptionManager dataFeedSubscriptionManager,
+            PerformanceTrackingTool performanceTrackingTool)
+        {
+            base.Initialize(algorithm, dataFeedSubscriptionManager, performanceTrackingTool);
+
+            // the time provider, is the real time provider
+            _timeProvider = GetTimeProvider();
+            _frontierTimeProvider = new LiveTimeProvider(realTime: TimeProvider);
+            // the synchronizer will use our '_frontierTimeProvider' which initially during warmup will be using
+            // the base time provider which is the subscription based time provider (like backtesting)
+            // once wawrmup finishes it will start using the realtime provider
+            SubscriptionSynchronizer.SetTimeProvider(_frontierTimeProvider);
+
+            // attach event handlers to subscriptions
+            dataFeedSubscriptionManager.SubscriptionAdded += (sender, subscription) =>
+            {
+                subscription.NewDataAvailable += OnSubscriptionNewDataAvailable;
+            };
+
+            dataFeedSubscriptionManager.SubscriptionRemoved += (sender, subscription) =>
+            {
+                subscription.NewDataAvailable -= OnSubscriptionNewDataAvailable;
+            };
+
+            _realTimeScheduleEventService = new RealTimeScheduleEventService(new RealTimeProvider());
+            // this schedule event will be our time pulse
+            _realTimeScheduleEventService.NewEvent += (sender, args) => _newLiveDataEmitted.Set();
+        }
+
+        /// <summary>
+        /// Returns an enumerable which provides the data to stream to the algorithm
+        /// </summary>
+        public override IEnumerable<TimeSlice> StreamData(CancellationToken cancellationToken)
+        {
+            PostInitialize();
+
+            var shouldSendExtraEmptyPacket = false;
+            var nextEmit = DateTime.MinValue;
+            var lastLoopStart = DateTime.UtcNow;
+
+            var enumerator = SubscriptionSynchronizer
+                .Sync(SubscriptionManager.DataFeedSubscriptions, cancellationToken)
+                .GetEnumerator();
+
+            var previousWasTimePulse = false;
+            while (!cancellationToken.IsCancellationRequested)
+            {
+                var now = DateTime.UtcNow;
+                if (!previousWasTimePulse)
+                {
+                    if (!_newLiveDataEmitted.IsSet
+                        // we warmup as fast as we can even if no new data point is available
+                        && !Algorithm.IsWarmingUp)
+                    {
+                        // if we just crossed into the next second let's loop again, we will flush any consolidator bar
+                        // else we will wait to be notified by the subscriptions or our scheduled event service every second
+                        if (lastLoopStart.Second == now.Second)
+                        {
+                            _realTimeScheduleEventService.ScheduleEvent(TimeSpan.FromMilliseconds(GetPulseDueTime(now)), now);
+                            _newLiveDataEmitted.Wait();
+                        }
+                    }
+                    _newLiveDataEmitted.Reset();
+                }
+
+                lastLoopStart = now;
+
+                TimeSlice timeSlice;
+                try
+                {
+                    if (!enumerator.MoveNext())
+                    {
+                        // the enumerator ended
+                        break;
+                    }
+
+                    timeSlice = enumerator.Current;
+                }
+                catch (Exception err)
+                {
+                    // notify the algorithm about the error, so it can be reported to the user
+                    Algorithm.SetRuntimeError(err, "LiveSynchronizer");
+                    shouldSendExtraEmptyPacket = true;
+                    break;
+                }
+
+                // check for cancellation
+                if (timeSlice == null || cancellationToken.IsCancellationRequested) break;
+
+                if (ShouldEmitWarmupEndPulse(timeSlice))
+                {
+                    yield return TimeSliceFactory.CreateTimePulse(WarmupEndUtc);
+                }
+
+                var frontierUtc = FrontierTimeProvider.GetUtcNow();
+                // emit on data or if we've elapsed a full second since last emit or there are security changes
+                if (timeSlice.SecurityChanges != SecurityChanges.None
+                    || timeSlice.IsTimePulse
+                    || timeSlice.Data.Count != 0
+                    || frontierUtc >= nextEmit)
+                {
+                    previousWasTimePulse = timeSlice.IsTimePulse;
+                    yield return timeSlice;
+
+                    // ignore if time pulse because we will emit a slice with the same time just after this one
+                    if (!timeSlice.IsTimePulse)
+                    {
+                        // force emitting every second since the data feed is
+                        // the heartbeat of the application
+                        nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond);
+                    }
+                }
+            }
+
+            if (shouldSendExtraEmptyPacket)
+            {
+                // send last empty packet list before terminating,
+                // so the algorithm manager has a chance to detect the runtime error
+                // and exit showing the correct error instead of a timeout
+                nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond);
+                if (!cancellationToken.IsCancellationRequested)
+                {
+                    var timeSlice = TimeSliceFactory.Create(
+                        nextEmit,
+                        new List<DataFeedPacket>(),
+                        SecurityChanges.None,
+                        new Dictionary<Universe, BaseDataCollection>());
+                    yield return timeSlice;
+                }
+            }
+
+            enumerator.DisposeSafely();
+            Log.Trace("LiveSynchronizer.GetEnumerator(): Exited thread.");
+        }
+
+        /// <summary>
+        /// Free resources
+        /// </summary>
+        public override void Dispose()
+        {
+            _newLiveDataEmitted.Set();
+            _newLiveDataEmitted?.DisposeSafely();
+            _realTimeScheduleEventService?.DisposeSafely();
+        }
+
+        /// <summary>
+        /// Gets the <see cref="ITimeProvider"/> to use. By default this will load the
+        /// <see cref="RealTimeProvider"/> for live mode, else <see cref="SubscriptionFrontierTimeProvider"/>
+        /// </summary>
+        /// <returns>The <see cref="ITimeProvider"/> to use</returns>
+        protected override ITimeProvider GetTimeProvider()
+        {
+            return RealTimeProvider.Instance;
+        }
+
+        /// <summary>
+        /// Performs additional initialization steps after algorithm initialization
+        /// </summary>
+        protected override void PostInitialize()
+        {
+            base.PostInitialize();
+            _frontierTimeProvider.Initialize(base.GetTimeProvider());
+            WarmupEndUtc = TimeProvider.GetUtcNow();
+        }
+
+        /// <summary>
+        /// Will return the amount of milliseconds that are missing for the next time pulse
+        /// </summary>
+        protected virtual int GetPulseDueTime(DateTime now)
+        {
+            // let's wait until the next second starts
+            return 1000 - now.Millisecond + BatchingDelay;
+        }
+
+        /// <summary>
+        /// Trigger new data event
+        /// </summary>
+        /// <param name="sender">Sender of the event</param>
+        /// <param name="args">Event information</param>
+        protected virtual void OnSubscriptionNewDataAvailable(object sender, EventArgs args)
+        {
+            _newLiveDataEmitted.Set();
+        }
+    }
+}

Engine/DataFeeds/LiveTimeProvider.cs

@@ -0,0 +1,79 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Live time provide which supports an initial warmup period using the given time provider <see cref="SubscriptionFrontierTimeProvider"/>, used by the <see cref="LiveSynchronizer"/>
+    /// </summary>
+    public class LiveTimeProvider : ITimeProvider
+    {
+        private DateTime _previous;
+        private readonly DateTime _liveStart;
+        private readonly ITimeProvider _realTime;
+        private ITimeProvider _warmupTimeProvider;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="realTime">Real time provider</param>
+        public LiveTimeProvider(ITimeProvider realTime)
+        {
+            _realTime = realTime;
+            _liveStart = _realTime.GetUtcNow();
+        }
+
+        /// <summary>
+        /// Fully initializes this instance providing the initial warmup time provider to use
+        /// </summary>
+        /// <param name="warmupTimeProvider">The warmup provider to use</param>
+        public void Initialize(ITimeProvider warmupTimeProvider)
+        {
+            _warmupTimeProvider = warmupTimeProvider;
+        }
+
+        /// <summary>
+        /// Gets the current time in UTC
+        /// </summary>
+        /// <returns>The current time in UTC</returns>
+        public DateTime GetUtcNow()
+        {
+            if(ReferenceEquals(_realTime, _warmupTimeProvider))
+            {
+                // warmup ended
+                return _realTime.GetUtcNow();
+            }
+
+            if (_warmupTimeProvider == null)
+            {
+                // don't let any live data point through until we are fully initialized
+                return Time.BeginningOfTime;
+            }
+
+            var newTime = _warmupTimeProvider.GetUtcNow();
+            if (_previous == newTime || newTime >= _liveStart)
+            {
+                // subscription time provider swap ended, start using live
+                _warmupTimeProvider = _realTime;
+                return GetUtcNow();
+            }
+            _previous = newTime;
+            return newTime;
+        }
+    }
+}

Engine/DataFeeds/LiveTradingDataFeed.cs

@@ -0,0 +1,613 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Packets;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Configuration;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Data.Custom.Tiingo;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
+using QuantConnect.Data.Fundamental;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="IDataFeed"/> that is designed to deal with
+    /// live, remote data sources
+    /// </summary>
+    public class LiveTradingDataFeed : FileSystemDataFeed
+    {
+        private static readonly int MaximumWarmupHistoryDaysLookBack = Config.GetInt("maximum-warmup-history-days-look-back", 5);
+
+        private LiveNodePacket _job;
+
+        // used to get current time
+        private ITimeProvider _timeProvider;
+        private IAlgorithm _algorithm;
+        private ITimeProvider _frontierTimeProvider;
+        private IDataProvider _dataProvider;
+        private IMapFileProvider _mapFileProvider;
+        private IDataQueueHandler _dataQueueHandler;
+        private BaseDataExchange _customExchange;
+        private SubscriptionCollection _subscriptions;
+        private IFactorFileProvider _factorFileProvider;
+        private IDataChannelProvider _channelProvider;
+        private readonly HashSet<string> _unsupportedConfigurations = new();
+
+        // in live trading we delay scheduled universe selection to 8 am NY, but NY goes from -4/-5 UTC time, so we adjust it
+        private static ReferenceWrapper<DateTime> _lastUtcDateShiftUpdate;
+        private static ReferenceWrapper<TimeSpan> _scheduledUniverseUtcTimeShift;
+
+        /// <summary>
+        /// Public flag indicator that the thread is still busy.
+        /// </summary>
+        public bool IsActive
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Initializes the data feed for the specified job and algorithm
+        /// </summary>
+        public override void Initialize(IAlgorithm algorithm,
+            AlgorithmNodePacket job,
+            IResultHandler resultHandler,
+            IMapFileProvider mapFileProvider,
+            IFactorFileProvider factorFileProvider,
+            IDataProvider dataProvider,
+            IDataFeedSubscriptionManager subscriptionManager,
+            IDataFeedTimeProvider dataFeedTimeProvider,
+            IDataChannelProvider dataChannelProvider)
+        {
+            if (!(job is LiveNodePacket))
+            {
+                throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
+            }
+
+            _algorithm = algorithm;
+            _job = (LiveNodePacket)job;
+            _timeProvider = dataFeedTimeProvider.TimeProvider;
+            _dataProvider = dataProvider;
+            _mapFileProvider = mapFileProvider;
+            _factorFileProvider = factorFileProvider;
+            _channelProvider = dataChannelProvider;
+            _frontierTimeProvider = dataFeedTimeProvider.FrontierTimeProvider;
+            _customExchange = GetBaseDataExchange();
+            _subscriptions = subscriptionManager.DataFeedSubscriptions;
+
+            _dataQueueHandler = GetDataQueueHandler();
+            _dataQueueHandler?.SetJob(_job);
+
+            // run the custom data exchange
+            _customExchange.Start();
+
+            IsActive = true;
+
+            base.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, subscriptionManager, dataFeedTimeProvider, dataChannelProvider);
+        }
+
+        /// <summary>
+        /// Creates a new subscription to provide data for the specified security.
+        /// </summary>
+        /// <param name="request">Defines the subscription to be added, including start/end times the universe and security</param>
+        /// <returns>The created <see cref="Subscription"/> if successful, null otherwise</returns>
+        public override Subscription CreateSubscription(SubscriptionRequest request)
+        {
+            Subscription subscription = null;
+            try
+            {
+                // create and add the subscription to our collection
+                subscription = request.IsUniverseSubscription
+                    ? CreateUniverseSubscription(request)
+                    : CreateDataSubscription(request);
+            }
+            catch (Exception err)
+            {
+                Log.Error(err, $"CreateSubscription(): Failed configuration: '{request.Configuration}'");
+                // kill the algorithm, this shouldn't happen
+                _algorithm.SetRuntimeError(err, $"Failed to subscribe to {request.Configuration.Symbol}");
+            }
+
+            return subscription;
+        }
+
+        /// <summary>
+        /// Removes the subscription from the data feed, if it exists
+        /// </summary>
+        /// <param name="subscription">The subscription to remove</param>
+        public override void RemoveSubscription(Subscription subscription)
+        {
+            var symbol = subscription.Configuration.Symbol;
+
+            // remove the subscriptions
+            if (!_channelProvider.ShouldStreamSubscription(subscription.Configuration))
+            {
+                _customExchange.RemoveEnumerator(symbol);
+            }
+            else
+            {
+                _dataQueueHandler.UnsubscribeWithMapping(subscription.Configuration);
+            }
+        }
+
+        /// <summary>
+        /// External controller calls to signal a terminate of the thread.
+        /// </summary>
+        public override void Exit()
+        {
+            if (IsActive)
+            {
+                IsActive = false;
+                Log.Trace("LiveTradingDataFeed.Exit(): Start. Setting cancellation token...");
+                if (_dataQueueHandler is DataQueueHandlerManager manager)
+                {
+                    manager.UnsupportedConfiguration -= HandleUnsupportedConfigurationEvent;
+                }
+                _customExchange?.Stop();
+                Log.Trace("LiveTradingDataFeed.Exit(): Exit Finished.");
+
+                base.Exit();
+            }
+        }
+
+        /// <summary>
+        /// Gets the <see cref="IDataQueueHandler"/> to use by default <see cref="DataQueueHandlerManager"/>
+        /// </summary>
+        /// <remarks>Useful for testing</remarks>
+        /// <returns>The loaded <see cref="IDataQueueHandler"/></returns>
+        protected virtual IDataQueueHandler GetDataQueueHandler()
+        {
+            var result = new DataQueueHandlerManager(_algorithm.Settings);
+            result.UnsupportedConfiguration += HandleUnsupportedConfigurationEvent;
+            return result;
+        }
+
+        /// <summary>
+        /// Gets the <see cref="BaseDataExchange"/> to use
+        /// </summary>
+        /// <remarks>Useful for testing</remarks>
+        protected virtual BaseDataExchange GetBaseDataExchange()
+        {
+            return new BaseDataExchange("CustomDataExchange") { SleepInterval = 100 };
+        }
+
+        /// <summary>
+        /// Creates a new subscription for the specified security
+        /// </summary>
+        /// <param name="request">The subscription request</param>
+        /// <returns>A new subscription instance of the specified security</returns>
+        private Subscription CreateDataSubscription(SubscriptionRequest request)
+        {
+            Subscription subscription = null;
+
+            // let's keep track of the last point we got from the file based enumerator and start our history enumeration from this point
+            // this is much more efficient since these duplicated points will be dropped by the filter righ away causing memory usage spikes
+            var lastPointTracker = new LastPointTracker();
+
+            var localStartTime = request.StartTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
+            var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
+            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
+
+            IEnumerator<BaseData> enumerator = null;
+            if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
+            {
+                if (!Tiingo.IsAuthCodeSet)
+                {
+                    // we're not using the SubscriptionDataReader, so be sure to set the auth token here
+                    Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
+                }
+
+                var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _algorithm.ObjectStore);
+                var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
+
+                var enqueable = new EnqueueableEnumerator<BaseData>();
+                _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack, handleData: data =>
+                {
+                    enqueable.Enqueue(data);
+
+                    subscription?.OnNewDataAvailable();
+                });
+
+                enumerator = enqueable;
+            }
+            else
+            {
+                var auxEnumerators = new List<IEnumerator<BaseData>>();
+
+                if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, request.Security.Cache, _mapFileProvider,
+                    _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
+                {
+                    auxEnumerators.Add(auxDataEnumator);
+                }
+
+                EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
+                enumerator = Subscribe(request.Configuration, handler, IsExpired);
+
+                if (auxEnumerators.Count > 0)
+                {
+                    enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
+                }
+            }
+
+            // scale prices before 'SubscriptionFilterEnumerator' since it updates securities realtime price
+            // and before fill forwarding so we don't happen to apply twice the factor
+            if (request.Configuration.PricesShouldBeScaled(liveMode: true))
+            {
+                enumerator = new PriceScaleFactorEnumerator(
+                    enumerator,
+                    request.Configuration,
+                    _factorFileProvider,
+                    liveMode: true);
+            }
+
+            if (request.Configuration.FillDataForward)
+            {
+                var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
+                // Pass the security exchange hours explicitly to avoid using the ones in the request, since
+                // those could be different. e.g. when requests are created for open interest data the exchange
+                // hours are set to always open to avoid OI data being filtered out due to the exchange being closed.
+                var useDailyStrictEndTimes = LeanData.UseDailyStrictEndTimes(_algorithm.Settings, request, request.Configuration.Symbol, request.Configuration.Increment, request.Security.Exchange.Hours);
+
+                enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution,
+                    request.Configuration.ExtendedMarketHours, localStartTime, localEndTime, request.Configuration.Resolution, request.Configuration.DataTimeZone,
+                    useDailyStrictEndTimes, request.Configuration.Type, lastPointTracker);
+            }
+
+            // make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
+            enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
+
+            // define market hours and user filters to incoming data after the frontier enumerator so during warmup we avoid any realtime data making it's way into the securities
+            if (request.Configuration.IsFilteredSubscription)
+            {
+                enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
+            }
+
+            enumerator = GetWarmupEnumerator(request, enumerator, lastPointTracker);
+
+            var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider,
+                enumerator, request.IsUniverseSubscription, _algorithm.Settings.DailyPreciseEndTime);
+            subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
+
+            return subscription;
+        }
+
+        /// <summary>
+        /// Helper method to determine if the symbol associated with the requested configuration is expired or not
+        /// </summary>
+        /// <remarks>This is useful during warmup where we can be requested to add some already expired asset. We want to skip sending it
+        /// to our live <see cref="_dataQueueHandler"/> instance to avoid explosions. But we do want to add warmup enumerators</remarks>
+        private bool IsExpired(SubscriptionDataConfig dataConfig)
+        {
+            var mapFile = _mapFileProvider.ResolveMapFile(dataConfig);
+            var delistingDate = dataConfig.Symbol.GetDelistingDate(mapFile);
+            return _timeProvider.GetUtcNow().Date > delistingDate.ConvertToUtc(dataConfig.ExchangeTimeZone);
+        }
+
+        private IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler, Func<SubscriptionDataConfig, bool> isExpired)
+        {
+            return new LiveSubscriptionEnumerator(dataConfig, _dataQueueHandler, newDataAvailableHandler, isExpired);
+        }
+
+        /// <summary>
+        /// Creates a new subscription for universe selection
+        /// </summary>
+        /// <param name="request">The subscription request</param>
+        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
+        {
+            Subscription subscription = null;
+
+            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class
+
+            // grab the relevant exchange hours
+            var config = request.Universe.Configuration;
+            var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
+            var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
+
+            IEnumerator<BaseData> enumerator = null;
+            if (request.Universe is ITimeTriggeredUniverse timeTriggered)
+            {
+                Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
+
+                // spoof a tick on the requested interval to trigger the universe selection function
+                var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
+                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
+
+                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
+                if (request.Universe is not UserDefinedUniverse)
+                {
+                    var enqueueable = new EnqueueableEnumerator<BaseData>();
+                    _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
+                    enumerator = enqueueable;
+                }
+            }
+            else if (config.Type.IsAssignableTo(typeof(ETFConstituentUniverse)) ||
+                config.Type.IsAssignableTo(typeof(FundamentalUniverse)) ||
+                request.Universe is OptionChainUniverse ||
+                request.Universe is FuturesChainUniverse)
+            {
+                Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating {config.Type.Name} universe: {config.Symbol.ID}");
+
+                // Will try to pull data from the data folder every 10min, file with yesterdays date.
+                // If lean is started today it will trigger initial coarse universe selection
+                var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
+                    _algorithm.ObjectStore,
+                    // we adjust time to the previous tradable date
+                    time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone, _algorithm.Settings.DailyPreciseEndTime),
+                    TimeSpan.FromMinutes(10)
+                );
+                var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
+
+                // aggregates each coarse data point into a single BaseDataCollection
+                var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, true);
+                var enqueable = new EnqueueableEnumerator<BaseData>();
+                _customExchange.AddEnumerator(config.Symbol, aggregator, handleData: data =>
+                {
+                    enqueable.Enqueue(data);
+                    subscription?.OnNewDataAvailable();
+                });
+
+                enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
+                    // advance time if before 23pm or after 5am and not on Saturdays
+                    time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
+            }
+            else
+            {
+                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);
+
+                var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _algorithm.ObjectStore);
+                var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
+                enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
+
+                var enqueueable = new EnqueueableEnumerator<BaseData>();
+                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
+                enumerator = enqueueable;
+            }
+
+            enumerator = AddScheduleWrapper(request, enumerator, new PredicateTimeProvider(_frontierTimeProvider, (currentUtcDateTime) => {
+                // will only let time advance after it's passed the live time shift frontier
+                return currentUtcDateTime.TimeOfDay > GetScheduledUniverseUtcTimeShift(currentUtcDateTime);
+            }));
+
+            enumerator = GetWarmupEnumerator(request, enumerator);
+
+            // create the subscription
+            var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider,
+                enumerator, request.IsUniverseSubscription, _algorithm.Settings.DailyPreciseEndTime);
+            subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
+
+            return subscription;
+        }
+
+        public static TimeSpan GetScheduledUniverseUtcTimeShift(DateTime currentUtcDateTime)
+        {
+            var currentDate = currentUtcDateTime.Date;
+            if (currentDate != _lastUtcDateShiftUpdate?.Value)
+            {
+                // on every date change, we will update the scheduled time shift to be 8am NY time, which is 12-13 UTC depending on DST
+                _lastUtcDateShiftUpdate = new(currentDate);
+                _scheduledUniverseUtcTimeShift = new(currentDate.ConvertFromUtc(TimeZones.NewYork).Date.AddHours(8).ConvertToUtc(TimeZones.NewYork).TimeOfDay
+                    // some minor randomness
+                    + TimeSpan.FromSeconds(DateTime.UtcNow.Second));
+            }
+            return _scheduledUniverseUtcTimeShift.Value;
+        }
+
+        /// <summary>
+        /// Build and apply the warmup enumerators when required
+        /// </summary>
+        private IEnumerator<BaseData> GetWarmupEnumerator(SubscriptionRequest request, IEnumerator<BaseData> liveEnumerator, LastPointTracker lastPointTracker = null)
+        {
+            if (_algorithm.IsWarmingUp)
+            {
+                var warmupRequest = new SubscriptionRequest(request, endTimeUtc: _timeProvider.GetUtcNow(),
+                    // we will not fill forward each warmup enumerators separately but concatenated bellow
+                    configuration: new SubscriptionDataConfig(request.Configuration, fillForward: false,
+                    resolution: _algorithm.Settings.WarmupResolution));
+                if (warmupRequest.TradableDaysInDataTimeZone.Any()
+                    // make sure there is at least room for a single bar of the requested resolution, else can cause issues with some history providers
+                    // this could happen when we create some internal subscription whose start time is 'Now', which we don't really want to warmup
+                    && warmupRequest.EndTimeUtc - warmupRequest.StartTimeUtc >= warmupRequest.Configuration.Resolution.ToTimeSpan()
+                    // since we change the resolution, let's validate it's still valid configuration (example daily equity quotes are not!)
+                    && LeanData.IsValidConfiguration(warmupRequest.Configuration.SecurityType, warmupRequest.Configuration.Resolution, warmupRequest.Configuration.TickType))
+                {
+                    // since we will source data locally and from the history provider, let's limit the history request size
+                    // by setting a start date respecting the 'MaximumWarmupHistoryDaysLookBack'
+                    var historyWarmup = warmupRequest;
+                    var warmupHistoryStartDate = warmupRequest.EndTimeUtc.AddDays(-MaximumWarmupHistoryDaysLookBack);
+                    if (warmupHistoryStartDate > warmupRequest.StartTimeUtc)
+                    {
+                        historyWarmup = new SubscriptionRequest(warmupRequest, startTimeUtc: warmupHistoryStartDate);
+                    }
+
+                    lastPointTracker ??= new LastPointTracker();
+
+                    var synchronizedWarmupEnumerator = TryAddFillForwardEnumerator(warmupRequest,
+                        // we concatenate the file based and history based warmup enumerators, dropping duplicate time stamps
+                        new ConcatEnumerator(true, GetFileBasedWarmupEnumerator(warmupRequest), GetHistoryWarmupEnumerator(historyWarmup, lastPointTracker)) { CanEmitNull = false },
+                        // if required by the original request, we will fill forward the Synced warmup data
+                        request.Configuration.FillDataForward,
+                        _algorithm.Settings.WarmupResolution);
+                    synchronizedWarmupEnumerator = ConfigureLastPointTracker(synchronizedWarmupEnumerator, lastPointTracker, isWarmUpEnumerator: true);
+                    synchronizedWarmupEnumerator = AddScheduleWrapper(warmupRequest, synchronizedWarmupEnumerator, null);
+
+                    // don't let future data past. We let null pass because that's letting the next enumerator know we've ended because we always return true in live
+                    synchronizedWarmupEnumerator = new FilterEnumerator<BaseData>(synchronizedWarmupEnumerator, data => data == null || data.EndTime <= warmupRequest.EndTimeLocal);
+
+                    // the order here is important, concat enumerator will keep the last enumerator given and dispose of the rest
+                    liveEnumerator = new ConcatEnumerator(true, synchronizedWarmupEnumerator, liveEnumerator);
+                }
+            }
+            return liveEnumerator;
+        }
+
+        /// <summary>
+        /// File based warmup enumerator
+        /// </summary>
+        private IEnumerator<BaseData> GetFileBasedWarmupEnumerator(SubscriptionRequest warmup)
+        {
+            IEnumerator<BaseData> result = null;
+            try
+            {
+                var enumerator = CreateEnumerator(warmup);
+                if (warmup.Configuration.PricesShouldBeScaled())
+                {
+                    enumerator = new PriceScaleFactorEnumerator(enumerator, warmup.Configuration, _factorFileProvider);
+                }
+                result = new FilterEnumerator<BaseData>(enumerator,
+                    // don't let future data past, nor fill forward, that will be handled after merging with the history request response
+                    data => data == null || data.EndTime < warmup.EndTimeLocal && !data.IsFillForward);
+            }
+            catch (Exception e)
+            {
+                Log.Error(e, $"File based warmup: {warmup.Configuration}");
+            }
+            return result;
+        }
+
+        /// <summary>
+        /// History based warmup enumerator
+        /// </summary>
+        private IEnumerator<BaseData> GetHistoryWarmupEnumerator(SubscriptionRequest warmup, LastPointTracker lastPointTracker)
+        {
+            IEnumerator<BaseData> result;
+            if (warmup.IsUniverseSubscription)
+            {
+                // we ignore the fill forward time span argument because we will fill forwared the concatenated file and history based enumerators next in the stack
+                result = CreateUniverseEnumerator(warmup);
+            }
+            else
+            {
+                // we create an enumerable of which we get the enumerator to defer the creation of the history request until the file based enumeration ended
+                // and potentially the 'lastPointTracker' is available to adjust our start time
+                result = new[] { warmup }.SelectMany(_ =>
+                {
+                    var startTimeUtc = warmup.StartTimeUtc;
+                    if (lastPointTracker != null && lastPointTracker.LastDataPoint != null)
+                    {
+                        var lastPointExchangeTime = lastPointTracker.LastDataPoint.Time;
+                        if (warmup.Configuration.Resolution == Resolution.Daily)
+                        {
+                            // time could be 9.30 for example using strict daily end times, but we just want the date in this case
+                            lastPointExchangeTime = lastPointExchangeTime.Date;
+                        }
+
+                        var utcLastPointTime = lastPointExchangeTime.ConvertToUtc(warmup.ExchangeHours.TimeZone);
+                        if (utcLastPointTime > startTimeUtc)
+                        {
+                            if (Log.DebuggingEnabled)
+                            {
+                                Log.Debug($"LiveTradingDataFeed.GetHistoryWarmupEnumerator(): Adjusting history warmup start time to {utcLastPointTime} from {startTimeUtc} for {warmup.Configuration}");
+                            }
+                            startTimeUtc = utcLastPointTime;
+                        }
+                    }
+                    var historyRequest = new Data.HistoryRequest(warmup.Configuration, warmup.ExchangeHours, startTimeUtc, warmup.EndTimeUtc);
+                    try
+                    {
+                        return _algorithm.HistoryProvider.GetHistory(new[] { historyRequest }, _algorithm.TimeZone).Select(slice =>
+                        {
+                            try
+                            {
+                                var data = slice.Get(historyRequest.DataType);
+                                return (BaseData)data[warmup.Configuration.Symbol];
+                            }
+                            catch (Exception e)
+                            {
+                                Log.Error(e, $"History warmup: {warmup.Configuration}");
+                            }
+                            return null;
+                        });
+                    }
+                    catch
+                    {
+                        // some history providers could throw if they do not support a type
+                    }
+                    return Enumerable.Empty<BaseData>();
+                }).GetEnumerator();
+            }
+
+            return new FilterEnumerator<BaseData>(result,
+                // don't let future data past, nor fill forward, that will be handled after merging with the file based enumerator
+                data => data == null || data.EndTime < warmup.EndTimeLocal && !data.IsFillForward);
+        }
+
+        /// <summary>
+        /// Will wrap the provided enumerator with a <see cref="FrontierAwareEnumerator"/>
+        /// using a <see cref="PredicateTimeProvider"/> that will advance time based on the provided
+        /// function
+        /// </summary>
+        /// <remarks>Won't advance time if now.Hour is bigger or equal than 23pm, less or equal than 5am or Saturday.
+        /// This is done to prevent universe selection occurring in those hours so that the subscription changes
+        /// are handled correctly.</remarks>
+        private IEnumerator<BaseData> GetConfiguredFrontierAwareEnumerator(
+            IEnumerator<BaseData> enumerator,
+            TimeZoneOffsetProvider tzOffsetProvider,
+            Func<DateTime, bool> customStepEvaluator)
+        {
+            var stepTimeProvider = new PredicateTimeProvider(_frontierTimeProvider, customStepEvaluator);
+
+            return new FrontierAwareEnumerator(enumerator, stepTimeProvider, tzOffsetProvider);
+        }
+
+        private IDataQueueUniverseProvider GetUniverseProvider(SecurityType securityType)
+        {
+            if (_dataQueueHandler is not IDataQueueUniverseProvider or DataQueueHandlerManager { HasUniverseProvider: false })
+            {
+                throw new NotSupportedException($"The DataQueueHandler does not support {securityType}.");
+            }
+            return (IDataQueueUniverseProvider)_dataQueueHandler;
+        }
+
+        private void HandleUnsupportedConfigurationEvent(object _, SubscriptionDataConfig config)
+        {
+            if (_algorithm != null)
+            {
+                lock (_unsupportedConfigurations)
+                {
+                    var key = $"{config.Symbol.ID.Market} {config.Symbol.ID.SecurityType} {config.Type.Name}";
+                    if (_unsupportedConfigurations.Add(key))
+                    {
+                        Log.Trace($"LiveTradingDataFeed.HandleUnsupportedConfigurationEvent(): detected unsupported configuration: {config}");
+
+                        _algorithm.Debug($"Warning: {key} data not supported. Please consider reviewing the data providers selection.");
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Overrides methods of the base data exchange implementation
+        /// </summary>
+        private class EnumeratorHandler : BaseDataExchange.EnumeratorHandler
+        {
+            public EnumeratorHandler(Symbol symbol, IEnumerator<BaseData> enumerator, EnqueueableEnumerator<BaseData> enqueueable)
+                : base(symbol, enumerator, handleData: enqueueable.Enqueue)
+            {
+                EnumeratorFinished += (_, _) => enqueueable.Stop();
+            }
+        }
+    }
+}

Engine/DataFeeds/ManualTimeProvider.cs

@@ -0,0 +1,102 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NodaTime;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ITimeProvider"/> that can be
+    /// manually advanced through time
+    /// </summary>
+    public class ManualTimeProvider : ITimeProvider
+    {
+        private volatile ReferenceWrapper<DateTime> _currentTime;
+        private readonly DateTimeZone _setCurrentTimeTimeZone;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ManualTimeProvider"/>
+        /// </summary>
+        /// <param name="setCurrentTimeTimeZone">Specify to use this time zone when calling <see cref="SetCurrentTime"/>,
+        /// leave null for the default of <see cref="TimeZones.Utc"/></param>
+        public ManualTimeProvider(DateTimeZone setCurrentTimeTimeZone = null)
+        {
+            _setCurrentTimeTimeZone = setCurrentTimeTimeZone ?? TimeZones.Utc;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ManualTimeProvider"/> class
+        /// </summary>
+        /// <param name="currentTime">The current time in the specified time zone, if the time zone is
+        /// null then the time is interpreted as being in <see cref="TimeZones.Utc"/></param>
+        /// <param name="setCurrentTimeTimeZone">Specify to use this time zone when calling <see cref="SetCurrentTime"/>,
+        /// leave null for the default of <see cref="TimeZones.Utc"/></param>
+        public ManualTimeProvider(DateTime currentTime, DateTimeZone setCurrentTimeTimeZone = null)
+            : this(setCurrentTimeTimeZone)
+        {
+            _currentTime = new ReferenceWrapper<DateTime>(currentTime.ConvertToUtc(_setCurrentTimeTimeZone));
+        }
+
+        /// <summary>
+        /// Gets the current time in UTC
+        /// </summary>
+        /// <returns>The current time in UTC</returns>
+        public DateTime GetUtcNow()
+        {
+            return _currentTime.Value;
+        }
+
+        /// <summary>
+        /// Sets the current time interpreting the specified time as a UTC time
+        /// </summary>
+        /// <param name="time">The current time in UTC</param>
+        public void SetCurrentTimeUtc(DateTime time)
+        {
+            _currentTime = new ReferenceWrapper<DateTime>(time);
+        }
+
+        /// <summary>
+        /// Sets the current time interpeting the specified time as a local time
+        /// using the time zone used at instatiation.
+        /// </summary>
+        /// <param name="time">The local time to set the current time time, will be
+        /// converted into UTC</param>
+        public void SetCurrentTime(DateTime time)
+        {
+            SetCurrentTimeUtc(time.ConvertToUtc(_setCurrentTimeTimeZone));
+        }
+
+        /// <summary>
+        /// Advances the current time by the specified span
+        /// </summary>
+        /// <param name="span">The amount of time to advance the current time by</param>
+        public void Advance(TimeSpan span)
+        {
+            _currentTime = new ReferenceWrapper<DateTime>(_currentTime.Value + span);
+        }
+
+        /// <summary>
+        /// Advances the current time by the specified number of seconds
+        /// </summary>
+        /// <param name="seconds">The number of seconds to advance the current time by</param>
+        public void AdvanceSeconds(double seconds)
+        {
+            Advance(TimeSpan.FromSeconds(seconds));
+        }
+    }
+}

Engine/DataFeeds/NullDataFeed.cs

@@ -0,0 +1,98 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Packets;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Null data feed implementation. <seealso cref="DataManager"/>
+    /// </summary>
+    public class NullDataFeed : IDataFeed
+    {
+        /// <summary>
+        /// Allows specifying if this implementation should throw always or not
+        /// </summary>
+        public bool ShouldThrow { get; set; } = true;
+
+        /// <inheritdoc />
+        public bool IsActive
+        {
+            get
+            {
+                if (!ShouldThrow)
+                {
+                    return true;
+                }
+                throw new NotImplementedException("Unexpected usage of null data feed implementation.");
+            }
+        }
+
+        /// <inheritdoc />
+        public void Initialize(
+            IAlgorithm algorithm,
+            AlgorithmNodePacket job,
+            IResultHandler resultHandler,
+            IMapFileProvider mapFileProvider,
+            IFactorFileProvider factorFileProvider,
+            IDataProvider dataProvider,
+            IDataFeedSubscriptionManager subscriptionManager,
+            IDataFeedTimeProvider dataFeedTimeProvider,
+            IDataChannelProvider dataChannelProvider
+            )
+        {
+            if (!ShouldThrow)
+            {
+                return;
+            }
+            throw new NotImplementedException("Unexpected usage of null data feed implementation.");
+        }
+
+        /// <inheritdoc />
+        public Subscription CreateSubscription(SubscriptionRequest request)
+        {
+            if (!ShouldThrow)
+            {
+                return null;
+            }
+            throw new NotImplementedException("Unexpected usage of null data feed implementation.");
+        }
+
+        /// <inheritdoc />
+        public void RemoveSubscription(Subscription subscription)
+        {
+            if (!ShouldThrow)
+            {
+                return;
+            }
+            throw new NotImplementedException("Unexpected usage of null data feed implementation.");
+        }
+
+        /// <inheritdoc />
+        public void Exit()
+        {
+            if (!ShouldThrow)
+            {
+                return;
+            }
+            throw new NotImplementedException("Unexpected usage of null data feed implementation.");
+        }
+    }
+}

Engine/DataFeeds/PendingRemovalsManager.cs

@@ -0,0 +1,201 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Helper class used to managed pending security removals <see cref="UniverseSelection"/>
+    /// </summary>
+    public class PendingRemovalsManager
+    {
+        private readonly Dictionary<Universe, List<Universe.Member>> _pendingRemovals;
+        private readonly IOrderProvider _orderProvider;
+
+        /// <summary>
+        /// Current pending removals
+        /// </summary>
+        public IReadOnlyDictionary<Universe, List<Universe.Member>> PendingRemovals => _pendingRemovals;
+
+        /// <summary>
+        /// Create a new instance
+        /// </summary>
+        /// <param name="orderProvider">The order provider used to determine if it is safe to remove a security</param>
+        public PendingRemovalsManager(IOrderProvider orderProvider)
+        {
+            _orderProvider = orderProvider;
+            _pendingRemovals = new Dictionary<Universe, List<Universe.Member>>();
+        }
+
+        /// <summary>
+        /// Determines if we can safely remove the security member from a universe.
+        /// We must ensure that we have zero holdings, no open orders, and no existing portfolio targets
+        /// </summary>
+        private bool IsSafeToRemove(Universe.Member member, Universe universe)
+        {
+            var security = member.Security;
+
+            // but don't physically remove it from the algorithm if we hold stock or have open orders against it or an open target
+            var openOrders = _orderProvider.GetOpenOrders(x => x.Symbol == security.Symbol);
+            if (!security.HoldStock && !openOrders.Any() && (security.Holdings.Target == null || security.Holdings.Target.Quantity == 0))
+            {
+                if (universe.Securities.Any(pair =>
+                    pair.Key.Underlying == security.Symbol && !IsSafeToRemove(pair.Value, universe)))
+                {
+                    // don't remove if any member in the universe which uses this 'member' as underlying can't be removed
+                    // covers the options use case
+                    return false;
+                }
+
+                // don't remove if there are unsettled positions
+                var unsettledCash = security.SettlementModel.GetUnsettledCash();
+                if (unsettledCash != default && unsettledCash.Amount > 0)
+                {
+                    return false;
+                }
+
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Will determine if the <see cref="Security"/> can be removed.
+        /// If it can be removed will add it to <see cref="PendingRemovals"/>
+        /// </summary>
+        /// <param name="member">The security to remove</param>
+        /// <param name="universe">The universe which the security is a member of</param>
+        /// <returns>The member to remove</returns>
+        public List<RemovedMember> TryRemoveMember(Universe.Member member, Universe universe)
+        {
+            if (IsSafeToRemove(member, universe))
+            {
+                return new List<RemovedMember> {new RemovedMember(universe, member.Security)};
+            }
+
+            if (_pendingRemovals.ContainsKey(universe))
+            {
+                if (!_pendingRemovals[universe].Contains(member))
+                {
+                    _pendingRemovals[universe].Add(member);
+                }
+            }
+            else
+            {
+                _pendingRemovals.Add(universe, new List<Universe.Member> { member });
+            }
+
+            return null;
+        }
+
+        /// <summary>
+        /// Will check if the security is pending for removal
+        /// </summary>
+        /// <param name="security">The security</param>
+        /// <param name="isInternal">Whether it's an internal subscription</param>
+        /// <returns>Whether the security is pending for removal</returns>
+        public bool IsPendingForRemoval(Security security, bool isInternal)
+        {
+            return _pendingRemovals.Values.Any(x => x.Any(y => y.IsInternal == isInternal && y.Security.Symbol == security.Symbol));
+        }
+
+        /// <summary>
+        /// Will check if the member is pending for removal
+        /// </summary>
+        /// <param name="member">The universe member</param>
+        /// <returns>Whether the security is pending for removal</returns>
+        public bool IsPendingForRemoval(Universe.Member member)
+        {
+            return IsPendingForRemoval(member.Security, member.IsInternal);
+        }
+
+        /// <summary>
+        /// Will check pending security removals
+        /// </summary>
+        /// <param name="selectedSymbols">Currently selected symbols</param>
+        /// <param name="currentUniverse">Current universe</param>
+        /// <returns>The members to be removed</returns>
+        public List<RemovedMember> CheckPendingRemovals(
+            HashSet<Symbol> selectedSymbols,
+            Universe currentUniverse)
+        {
+            var result = new List<RemovedMember>();
+            // remove previously deselected members which were kept in the universe because of holdings or open orders
+            foreach (var kvp in _pendingRemovals.ToList())
+            {
+                var universeRemoving = kvp.Key;
+                foreach (var member in kvp.Value.ToList())
+                {
+                    var isSafeToRemove = IsSafeToRemove(member, universeRemoving);
+                    if (isSafeToRemove
+                        ||
+                        // if we are re selecting it we remove it as a pending removal
+                        // else we might remove it when we do not want to do so
+                        universeRemoving == currentUniverse
+                        && selectedSymbols.Contains(member.Security.Symbol))
+                    {
+                        if (isSafeToRemove)
+                        {
+                            result.Add(new RemovedMember(universeRemoving, member.Security));
+                        }
+
+                        _pendingRemovals[universeRemoving].Remove(member);
+
+                        // if there are no more pending removals for this universe lets remove it
+                        if (!_pendingRemovals[universeRemoving].Any())
+                        {
+                            _pendingRemovals.Remove(universeRemoving);
+                        }
+                    }
+                }
+            }
+
+            return result;
+        }
+
+        /// <summary>
+        /// Helper class used to report removed universe members
+        /// </summary>
+        public class RemovedMember
+        {
+            /// <summary>
+            /// Universe the security was removed from
+            /// </summary>
+            public Universe Universe { get; }
+
+            /// <summary>
+            /// Security that is removed
+            /// </summary>
+            public Security Security { get; }
+
+            /// <summary>
+            /// Initialize a new instance of <see cref="RemovedMember"/>
+            /// </summary>
+            /// <param name="universe"><see cref="Universe"/> the security was removed from</param>
+            /// <param name="security"><see cref="Security"/> that is removed</param>
+            public RemovedMember(Universe universe, Security security)
+            {
+                Universe = universe;
+                Security = security;
+            }
+        }
+    }
+}

Engine/DataFeeds/PrecalculatedSubscriptionData.cs

@@ -0,0 +1,70 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Data;
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Store data both raw and adjusted and the time at which it should be synchronized
+    /// </summary>
+    public class PrecalculatedSubscriptionData : SubscriptionData
+    {
+        private BaseData _normalizedData;
+        private SubscriptionDataConfig _config;
+        private readonly DataNormalizationMode _mode;
+
+        /// <summary>
+        /// Gets the data
+        /// </summary>
+        public override BaseData Data
+        {
+            get
+            {
+                if (_config.DataNormalizationMode == DataNormalizationMode.Raw)
+                {
+                    return _data;
+                }
+                else if (_config.DataNormalizationMode == _mode)
+                {
+                    return _normalizedData;
+                }
+                else
+                {
+                    throw new ArgumentException($"DataNormalizationMode.{_config.DataNormalizationMode} was requested for " 
+                                                + $"symbol {_data.Symbol} but only {_mode} and Raw DataNormalizationMode are available. " 
+                                                + "Please configure the desired DataNormalizationMode initially when adding the Symbol");
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="PrecalculatedSubscriptionData"/> class
+        /// </summary>
+        /// <param name="configuration">The subscription's configuration</param>
+        /// <param name="rawData">The base data</param>
+        /// <param name="normalizedData">The normalized calculated based on raw data</param>
+        /// <param name="normalizationMode">Specifies how data is normalized</param>
+        /// <param name="emitTimeUtc">The emit time for the data</param>
+        public PrecalculatedSubscriptionData(SubscriptionDataConfig configuration, BaseData rawData, BaseData normalizedData, DataNormalizationMode normalizationMode, DateTime emitTimeUtc)
+            : base(rawData, emitTimeUtc)
+        {
+            _config = configuration;
+            _normalizedData = normalizedData;
+            _mode = normalizationMode;
+        }
+    }
+}

Engine/DataFeeds/PredicateTimeProvider.cs

@@ -0,0 +1,79 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Will generate time steps around the desired <see cref="ITimeProvider"/>
+    /// Provided step evaluator should return true when the next time step
+    /// is valid and time can advance
+    /// </summary>
+    public class PredicateTimeProvider : ITimeProvider
+    {
+        private readonly ITimeProvider _underlyingTimeProvider;
+        private readonly Func<DateTime, bool> _customStepEvaluator;
+        private DateTime _currentUtc = DateTime.MinValue;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="underlyingTimeProvider">The timer provider instance to wrap</param>
+        /// <param name="customStepEvaluator">Function to evaluate whether or not
+        /// to advance time. Should return true if provided <see cref="DateTime"/> is a
+        /// valid new next time. False will avoid time advancing</param>
+        public PredicateTimeProvider(ITimeProvider underlyingTimeProvider,
+            Func<DateTime, bool> customStepEvaluator)
+        {
+            _underlyingTimeProvider = underlyingTimeProvider;
+            _customStepEvaluator = customStepEvaluator;
+        }
+
+        /// <summary>
+        /// Gets the current utc time step
+        /// </summary>
+        public DateTime GetUtcNow()
+        {
+            if (_currentUtc == DateTime.MinValue)
+            {
+                Initialize();
+            }
+            var utcNow = _underlyingTimeProvider.GetUtcNow();
+
+            // we check if we should advance time based on the provided custom step evaluator
+            if (_customStepEvaluator(utcNow))
+            {
+                _currentUtc = utcNow;
+            }
+            return _currentUtc;
+        }
+
+        private void Initialize()
+        {
+            // to determine the current time we go backwards up to 2 days until we reach a valid time we don't want to start on an invalid time
+            var utcNow = _underlyingTimeProvider.GetUtcNow();
+            for (var i = 0; i < 48; i++)
+            {
+                var before = utcNow.AddHours(-1 * i);
+                if (_customStepEvaluator(before))
+                {
+                    _currentUtc = before;
+                }
+            }
+        }
+    }
+}

Engine/DataFeeds/ProcessedDataProvider.cs

@@ -0,0 +1,80 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*/
+
+using System.IO;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using QuantConnect.Configuration;
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// A data provider that will check the processed data folder first
+    /// </summary>
+    public class ProcessedDataProvider : IDataProvider, IDisposable
+    {
+        private readonly DefaultDataProvider _defaultDataProvider;
+        private readonly string _processedDataDirectory;
+
+        /// <summary>
+        /// Ignored
+        /// </summary>
+        public event EventHandler<DataProviderNewDataRequestEventArgs> NewDataRequest;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public ProcessedDataProvider()
+        {
+            _defaultDataProvider = new();
+            _processedDataDirectory = Config.Get("processed-data-directory") ?? string.Empty;
+            Log.Trace($"ProcessedDataProvider(): processed data directory to use {_processedDataDirectory}, exists: {Directory.Exists(_processedDataDirectory)}");
+        }
+
+        /// <summary>
+        /// Retrieves data from disc to be used in an algorithm
+        /// </summary>
+        /// <param name="key">A string representing where the data is stored</param>
+        /// <returns>A <see cref="Stream"/> of the data requested</returns>
+        public Stream Fetch(string key)
+        {
+            Stream result = null;
+
+            // we will try the processed data folder first
+            if (_processedDataDirectory.Length != 0 && key.StartsWith(Globals.DataFolder, StringComparison.OrdinalIgnoreCase))
+            {
+                result = _defaultDataProvider.Fetch(Path.Combine(_processedDataDirectory, key.Remove(0, Globals.DataFolder.Length).TrimStart('/', '\\')));
+                if (result != null)
+                {
+                    Log.Trace($"ProcessedDataProvider.Fetch({key}): fetched from processed data directory");
+                }
+            }
+
+            // fall back to existing data folder path
+            return result ?? _defaultDataProvider.Fetch(key);
+        }
+
+        /// <summary>
+        /// Disposes of resources
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Disposes of the internal data provider
+        /// </summary>
+        protected virtual void Dispose(bool disposing)
+        {
+            if (disposing)
+            {
+                _defaultDataProvider.Dispose();
+            }
+        }
+    }
+}

Engine/DataFeeds/Queues/DataQueue.cs

@@ -0,0 +1,66 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Packets;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Queues
+{
+    /// <summary>
+    /// Live Data Queue is the cut out implementation of how to bind a custom live data source
+    /// </summary>
+    public class LiveDataQueue : IDataQueueHandler
+    {
+        /// <summary>
+        /// Desktop/Local doesn't support live data from this handler
+        /// </summary>
+        public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
+        {
+            throw new NotImplementedException("QuantConnect.Queues.LiveDataQueue has not implemented live data.");
+        }
+
+        /// <summary>
+        /// Desktop/Local doesn't support live data from this handler
+        /// </summary>
+        public virtual void Unsubscribe(SubscriptionDataConfig dataConfig)
+        {
+            throw new NotImplementedException("QuantConnect.Queues.LiveDataQueue has not implemented live data.");
+        }
+
+        /// <summary>
+        /// Sets the job we're subscribing for
+        /// </summary>
+        /// <param name="job">Job we're subscribing for</param>
+        public void SetJob(LiveNodePacket job)
+        {
+        }
+
+        /// <summary>
+        /// Returns whether the data provider is connected
+        /// </summary>
+        /// <returns>true if the data provider is connected</returns>
+        public bool IsConnected => false;
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+        }
+    }
+}

Engine/DataFeeds/Queues/FakeDataQueue.cs

@@ -0,0 +1,255 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Packets;
+using QuantConnect.Securities;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using Timer = System.Timers.Timer;
+using QuantConnect.Lean.Engine.HistoricalData;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Queues
+{
+    /// <summary>
+    /// This is an implementation of <see cref="IDataQueueHandler"/> used for testing. <see cref="FakeHistoryProvider"/>
+    /// </summary>
+    public class FakeDataQueue : IDataQueueHandler, IDataQueueUniverseProvider
+    {
+        private int _count;
+        private readonly Random _random = new Random();
+        private int _dataPointsPerSecondPerSymbol;
+
+        private readonly Timer _timer;
+        private readonly IOptionChainProvider _optionChainProvider;
+        private readonly EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager;
+        private readonly IDataAggregator _aggregator;
+        private readonly MarketHoursDatabase _marketHoursDatabase;
+        private readonly Dictionary<Symbol, TimeZoneOffsetProvider> _symbolExchangeTimeZones;
+
+        /// <summary>
+        /// Continuous UTC time provider
+        /// </summary>
+        protected virtual ITimeProvider TimeProvider { get; } = RealTimeProvider.Instance;
+
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
+        /// </summary>
+        public FakeDataQueue()
+            : this(Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(nameof(AggregationManager)))
+        {
+
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FakeDataQueue"/> class to randomly emit data for each symbol
+        /// </summary>
+        public FakeDataQueue(IDataAggregator dataAggregator, int dataPointsPerSecondPerSymbol = 500000)
+        {
+            _aggregator = dataAggregator;
+            _dataPointsPerSecondPerSymbol = dataPointsPerSecondPerSymbol;
+
+            var mapFileProvider = Composer.Instance.GetPart<IMapFileProvider>();
+            var historyManager = (IHistoryProvider)Composer.Instance.GetPart<HistoryProviderManager>();
+            if (historyManager == null)
+            {
+                historyManager = Composer.Instance.GetPart<IHistoryProvider>();
+            }
+            var optionChainProvider = new LiveOptionChainProvider();
+            optionChainProvider.Initialize(new(mapFileProvider, historyManager));
+            _optionChainProvider = optionChainProvider;
+
+            _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
+            _symbolExchangeTimeZones = new Dictionary<Symbol, TimeZoneOffsetProvider>();
+            _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
+            _subscriptionManager.SubscribeImpl += (s, t) => true;
+            _subscriptionManager.UnsubscribeImpl += (s, t) => true;
+
+            _timer = new Timer
+            {
+                AutoReset = false,
+                Enabled = true,
+                Interval = 1000,
+            };
+
+            var lastCount = 0;
+            var lastTime = DateTime.UtcNow;
+            _timer.Elapsed += (sender, args) =>
+            {
+                var elapsed = (DateTime.UtcNow - lastTime);
+                var ticksPerSecond = (_count - lastCount)/elapsed.TotalSeconds;
+                Log.Trace("TICKS PER SECOND:: " + ticksPerSecond.ToStringInvariant("000000.0") + " ITEMS IN QUEUE:: " + 0);
+                lastCount = _count;
+                lastTime = DateTime.UtcNow;
+                PopulateQueue();
+                try
+                {
+                    _timer.Reset();
+                }
+                catch (ObjectDisposedException)
+                {
+                    // pass
+                }
+            };
+        }
+
+        /// <summary>
+        /// Subscribe to the specified configuration
+        /// </summary>
+        /// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
+        /// <param name="newDataAvailableHandler">handler to be fired on new data available</param>
+        /// <returns>The new enumerator for this subscription request</returns>
+        public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
+        {
+            var enumerator = _aggregator.Add(dataConfig, newDataAvailableHandler);
+            _subscriptionManager.Subscribe(dataConfig);
+
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Sets the job we're subscribing for
+        /// </summary>
+        /// <param name="job">Job we're subscribing for</param>
+        public void SetJob(LiveNodePacket job)
+        {
+        }
+
+        /// <summary>
+        /// Removes the specified configuration
+        /// </summary>
+        /// <param name="dataConfig">Subscription config to be removed</param>
+        public void Unsubscribe(SubscriptionDataConfig dataConfig)
+        {
+            _subscriptionManager.Unsubscribe(dataConfig);
+            _aggregator.Remove(dataConfig);
+        }
+
+        /// <summary>
+        /// Returns whether the data provider is connected
+        /// </summary>
+        /// <returns>true if the data provider is connected</returns>
+        public bool IsConnected => true;
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            _timer.Stop();
+            _timer.DisposeSafely();
+        }
+
+        /// <summary>
+        /// Pumps a bunch of ticks into the queue
+        /// </summary>
+        private void PopulateQueue()
+        {
+            var symbols = _subscriptionManager.GetSubscribedSymbols();
+
+
+            foreach (var symbol in symbols)
+            {
+                if (symbol.IsCanonical() || symbol.Contains("UNIVERSE"))
+                {
+                    continue;
+                }
+                var offsetProvider = GetTimeZoneOffsetProvider(symbol);
+                var trades = SubscriptionManager.DefaultDataTypes()[symbol.SecurityType].Contains(TickType.Trade);
+                var quotes = SubscriptionManager.DefaultDataTypes()[symbol.SecurityType].Contains(TickType.Quote);
+
+                // emits 500k per second
+                for (var i = 0; i < _dataPointsPerSecondPerSymbol; i++)
+                {
+                    var now = TimeProvider.GetUtcNow();
+                    var exchangeTime = offsetProvider.ConvertFromUtc(now);
+                    var lastTrade = 100 + (decimal)Math.Abs(Math.Sin(now.TimeOfDay.TotalMilliseconds));
+                    if (trades)
+                    {
+                        _count++;
+                        _aggregator.Update(new Tick
+                        {
+                            Time = exchangeTime,
+                            Symbol = symbol,
+                            Value = lastTrade,
+                            TickType = TickType.Trade,
+                            Quantity = _random.Next(10, (int)_timer.Interval)
+                        });
+                    }
+
+                    if (quotes)
+                    {
+                        _count++;
+                        var bidPrice = lastTrade * 0.95m;
+                        var askPrice = lastTrade * 1.05m;
+                        var bidSize = _random.Next(10, (int) _timer.Interval);
+                        var askSize = _random.Next(10, (int)_timer.Interval);
+                        _aggregator.Update(new Tick(exchangeTime, symbol, "", "", bidSize: bidSize, bidPrice: bidPrice, askPrice: askPrice, askSize: askSize));
+                    }
+                }
+            }
+        }
+
+        private TimeZoneOffsetProvider GetTimeZoneOffsetProvider(Symbol symbol)
+        {
+            TimeZoneOffsetProvider offsetProvider;
+            if (!_symbolExchangeTimeZones.TryGetValue(symbol, out offsetProvider))
+            {
+                // read the exchange time zone from market-hours-database
+                var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
+                _symbolExchangeTimeZones[symbol] = offsetProvider = new TimeZoneOffsetProvider(exchangeTimeZone, TimeProvider.GetUtcNow(), Time.EndOfTime);
+            }
+            return offsetProvider;
+        }
+
+        /// <summary>
+        /// Method returns a collection of Symbols that are available at the data source.
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup</param>
+        /// <param name="includeExpired">Include expired contracts</param>
+        /// <param name="securityCurrency">Expected security currency(if any)</param>
+        /// <returns>Enumerable of Symbols, that are associated with the provided Symbol</returns>
+        public IEnumerable<Symbol> LookupSymbols(Symbol symbol, bool includeExpired, string securityCurrency = null)
+        {
+            switch (symbol.SecurityType)
+            {
+                case SecurityType.Option:
+                case SecurityType.IndexOption:
+                case SecurityType.FutureOption:
+                    foreach (var result in _optionChainProvider.GetOptionContractList(symbol, DateTime.UtcNow.Date))
+                    {
+                        yield return result;
+                    }
+                    break;
+                default:
+                    break;
+            }
+        }
+
+        /// <summary>
+        /// Checks if the FakeDataQueue can perform selection
+        /// </summary>
+        public bool CanPerformSelection()
+        {
+            return true;
+        }
+    }
+}

Engine/DataFeeds/ReaderErrorEventArgs.cs

@@ -0,0 +1,52 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Event arguments for the <see cref="TextSubscriptionDataSourceReader.ReaderError"/> event.
+    /// </summary>
+    public sealed class ReaderErrorEventArgs : EventArgs
+    {
+        /// <summary>
+        /// Gets the line that caused the error
+        /// </summary>
+        public string Line
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the exception that was caught
+        /// </summary>
+        public Exception Exception
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ReaderErrorEventArgs"/> class
+        /// </summary>
+        /// <param name="line">The line that caused the error</param>
+        /// <param name="exception">The exception that was caught during the read</param>
+        public ReaderErrorEventArgs(string line, Exception exception)
+        {
+            Line = line;
+            Exception = exception;
+        }
+    }
+}

Engine/DataFeeds/RealTimeScheduleEventService.cs

@@ -0,0 +1,117 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Threading;
+using QuantConnect.Util;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Allows to setup a real time scheduled event, internally using a <see cref="Thread"/>,
+    /// that is guaranteed to trigger at or after the requested time, never before.
+    /// </summary>
+    /// <remarks>This class is of value because <see cref="Timer"/> could fire the
+    /// event before time.</remarks>
+    public class RealTimeScheduleEventService : IDisposable
+    {
+        private readonly Thread _pulseThread;
+        private readonly Queue<DateTime> _work;
+        private readonly ManualResetEvent _event;
+        private readonly CancellationTokenSource _tokenSource;
+
+        /// <summary>
+        /// Event fired when the scheduled time is past
+        /// </summary>
+        public event EventHandler NewEvent;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="timeProvider">The time provider to use</param>
+        public RealTimeScheduleEventService(ITimeProvider timeProvider)
+        {
+            _tokenSource = new CancellationTokenSource();
+            _event = new ManualResetEvent(false);
+            _work = new Queue<DateTime>();
+            _pulseThread = new Thread(() =>
+                {
+                    while (!_tokenSource.Token.IsCancellationRequested)
+                    {
+                        DateTime nextUtcScheduledEvent;
+                        lock (_work)
+                        {
+                            _work.TryDequeue(out nextUtcScheduledEvent);
+                        }
+
+                        if (nextUtcScheduledEvent == default)
+                        {
+                            _event.WaitOne(_tokenSource.Token);
+                            _event.Reset();
+                            if (_tokenSource.Token.IsCancellationRequested)
+                            {
+                                return;
+                            }
+                            continue;
+                        }
+
+                        // testing has shown that it sometimes requires more than one loop
+                        var diff = nextUtcScheduledEvent - timeProvider.GetUtcNow();
+                        while (diff.Ticks > 0)
+                        {
+                            _tokenSource.Token.WaitHandle.WaitOne(diff);
+
+                            diff = nextUtcScheduledEvent - timeProvider.GetUtcNow();
+
+                            if (_tokenSource.Token.IsCancellationRequested)
+                            {
+                                return;
+                            }
+                        }
+
+                        NewEvent?.Invoke(this, EventArgs.Empty);
+                    }
+                }) { IsBackground = true, Name = "RealTimeScheduleEventService" };
+            _pulseThread.Start();
+        }
+
+        /// <summary>
+        /// Schedules a new event
+        /// </summary>
+        /// <param name="dueTime">The desired due time</param>
+        /// <param name="utcNow">Current utc time</param>
+        /// <remarks>Scheduling a new event will try to disable previous scheduled event,
+        /// but it is not guaranteed.</remarks>
+        public void ScheduleEvent(TimeSpan dueTime, DateTime utcNow)
+        {
+            lock (_work)
+            {
+                _work.Enqueue(utcNow + dueTime);
+                _event.Set();
+            }
+        }
+
+        /// <summary>
+        /// Disposes of the underlying <see cref="Timer"/> instance
+        /// </summary>
+        public void Dispose()
+        {
+            _pulseThread.StopSafely(TimeSpan.FromSeconds(1), _tokenSource);
+            _tokenSource.DisposeSafely();
+            _event.DisposeSafely();
+        }
+    }
+}

Engine/DataFeeds/SingleEntryDataCacheProvider.cs

@@ -0,0 +1,120 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.IO;
+using Ionic.Zip;
+using System.Linq;
+using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Default implementation of the <see cref="IDataCacheProvider"/>
+    /// Does not cache data.  If the data is a zip, the first entry is returned
+    /// </summary>
+    public class SingleEntryDataCacheProvider : IDataCacheProvider
+    {
+        private readonly IDataProvider _dataProvider;
+        private ZipFile _zipFile;
+        private Stream _zipFileStream;
+
+        /// <summary>
+        /// Property indicating the data is temporary in nature and should not be cached.
+        /// </summary>
+        public bool IsDataEphemeral { get; }
+
+        /// <summary>
+        /// Constructor that takes the <see cref="IDataProvider"/> to be used to retrieve data
+        /// </summary>
+        public SingleEntryDataCacheProvider(IDataProvider dataProvider, bool isDataEphemeral = true)
+        {
+            _dataProvider = dataProvider;
+            IsDataEphemeral = isDataEphemeral;
+        }
+
+        /// <summary>
+        /// Fetch data from the cache
+        /// </summary>
+        /// <param name="key">A string representing the key of the cached data</param>
+        /// <returns>An <see cref="Stream"/> of the cached data</returns>
+        public Stream Fetch(string key)
+        {
+            LeanData.ParseKey(key, out var filePath, out var entryName);
+            var stream = _dataProvider.Fetch(filePath);
+
+            if (filePath.EndsWith(".zip") && stream != null)
+            {
+                // get the first entry from the zip file
+                try
+                {
+                    var entryStream = Compression.UnzipStream(stream, out _zipFile, entryName);
+
+                    // save the file stream so it can be disposed later
+                    _zipFileStream = stream;
+
+                    return entryStream;
+                }
+                catch (ZipException exception)
+                {
+                    Log.Error("SingleEntryDataCacheProvider.Fetch(): Corrupt file: " + key + " Error: " + exception);
+                    stream.DisposeSafely();
+                    return null;
+                }
+            }
+
+            return stream;
+        }
+
+        /// <summary>
+        /// Not implemented
+        /// </summary>
+        /// <param name="key">The source of the data, used as a key to retrieve data in the cache</param>
+        /// <param name="data">The data to cache as a byte array</param>
+        public void Store(string key, byte[] data)
+        {
+            //
+        }
+
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        public List<string> GetZipEntries(string zipFile)
+        {
+            var stream = _dataProvider.Fetch(zipFile);
+            if (stream == null)
+            {
+                throw new ArgumentException($"Failed to create source stream {zipFile}");
+            }
+            var entryNames = Compression.GetZipEntryFileNames(stream).ToList();
+            stream.DisposeSafely();
+
+            return entryNames;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            _zipFile?.DisposeSafely();
+            _zipFileStream?.DisposeSafely();
+        }
+    }
+}

Engine/DataFeeds/Subscription.cs

@@ -0,0 +1,309 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Represents the data required for a data feed to process a single subscription
+    /// </summary>
+    public class Subscription : IEnumerator<SubscriptionData>
+    {
+        private bool _removedFromUniverse;
+        private readonly IEnumerator<SubscriptionData> _enumerator;
+
+        /// <summary>
+        /// The subcription requests associated with this subscription
+        /// </summary>
+        internal List<SubscriptionRequest> SubscriptionRequests { get; private set; }
+
+        /// <summary>
+        /// Event fired when a new data point is available
+        /// </summary>
+        public event EventHandler NewDataAvailable;
+
+        /// <summary>
+        /// Gets the universe for this subscription
+        /// </summary>
+        public IEnumerable<Universe> Universes => SubscriptionRequests
+            .Where(x => x.Universe != null)
+            .Select(x => x.Universe);
+
+        /// <summary>
+        /// Gets the security this subscription points to
+        /// </summary>
+        public ISecurityPrice Security { get; init; }
+
+        /// <summary>
+        /// Gets the configuration for this subscritions
+        /// </summary>
+        public SubscriptionDataConfig Configuration { get; init; }
+
+        /// <summary>
+        /// Gets the exchange time zone associated with this subscription
+        /// </summary>
+        public DateTimeZone TimeZone { get; }
+
+        /// <summary>
+        /// Gets the offset provider for time zone conversions to and from the data's local time
+        /// </summary>
+        public TimeZoneOffsetProvider OffsetProvider { get; init; }
+
+        /// <summary>
+        /// Gets the most current value from the subscription source
+        /// </summary>
+        public decimal RealtimePrice { get; set; }
+
+        /// <summary>
+        /// Gets true if this subscription is finished, false otherwise
+        /// </summary>
+        public bool EndOfStream { get; private set; }
+
+        /// <summary>
+        /// Gets true if this subscription is used in universe selection
+        /// </summary>
+        public bool IsUniverseSelectionSubscription { get; }
+
+        /// <summary>
+        /// Gets the start time of this subscription in UTC
+        /// </summary>
+        public DateTime UtcStartTime { get; }
+
+        /// <summary>
+        /// Gets the end time of this subscription in UTC
+        /// </summary>
+        public DateTime UtcEndTime { get; }
+
+        /// <summary>
+        /// Gets whether or not this subscription has been removed from its parent universe
+        /// </summary>
+        public IReadOnlyRef<bool> RemovedFromUniverse { get; }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Subscription"/> class with a universe
+        /// </summary>
+        /// <param name="subscriptionRequest">Specified for universe subscriptions</param>
+        /// <param name="enumerator">The subscription's data source</param>
+        /// <param name="timeZoneOffsetProvider">The offset provider used to convert data local times to utc</param>
+        public Subscription(
+            SubscriptionRequest subscriptionRequest,
+            IEnumerator<SubscriptionData> enumerator,
+            TimeZoneOffsetProvider timeZoneOffsetProvider)
+        {
+            SubscriptionRequests = new List<SubscriptionRequest> { subscriptionRequest };
+            _enumerator = enumerator;
+            Security = subscriptionRequest.Security;
+            IsUniverseSelectionSubscription = subscriptionRequest.IsUniverseSubscription;
+            Configuration = subscriptionRequest.Configuration;
+            OffsetProvider = timeZoneOffsetProvider;
+            TimeZone = subscriptionRequest.Security.Exchange.TimeZone;
+            UtcStartTime = subscriptionRequest.StartTimeUtc;
+            UtcEndTime = subscriptionRequest.EndTimeUtc;
+
+            RemovedFromUniverse = Ref.CreateReadOnly(() => _removedFromUniverse);
+        }
+
+        /// <summary>
+        /// Adds a <see cref="SubscriptionRequest"/> for this subscription
+        /// </summary>
+        /// <param name="subscriptionRequest">The <see cref="SubscriptionRequest"/> to add</param>
+        public bool AddSubscriptionRequest(SubscriptionRequest subscriptionRequest)
+        {
+            if (IsUniverseSelectionSubscription
+                || subscriptionRequest.IsUniverseSubscription)
+            {
+                if (subscriptionRequest.Universe is UserDefinedUniverse)
+                {
+                    // for different reasons a user defined universe can trigger a subscription request, likes additions/removals
+                    return false;
+                }
+                throw new Exception("Subscription.AddSubscriptionRequest(): Universe selection" +
+                    " subscriptions should not have more than 1 SubscriptionRequest");
+            }
+
+            // this shouldn't happen but just in case..
+            if (subscriptionRequest.Configuration != Configuration)
+            {
+                throw new Exception("Subscription.AddSubscriptionRequest(): Requesting to add" +
+                    "a different SubscriptionDataConfig");
+            }
+
+            // Only allow one subscription request per universe
+            if (!Universes.Contains(subscriptionRequest.Universe))
+            {
+                SubscriptionRequests.Add(subscriptionRequest);
+                // TODO this might update the 'UtcStartTime' and 'UtcEndTime' of this subscription
+                return true;
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Removes one or all <see cref="SubscriptionRequest"/> from this subscription
+        /// </summary>
+        /// <param name="universe">Universe requesting to remove <see cref="SubscriptionRequest"/>.
+        /// Default value, null, will remove all universes</param>
+        /// <returns>True, if the subscription is empty and ready to be removed</returns>
+        public bool RemoveSubscriptionRequest(Universe universe = null)
+        {
+            // TODO this might update the 'UtcStartTime' and 'UtcEndTime' of this subscription
+            IEnumerable<Universe> removedUniverses;
+            if (universe == null)
+            {
+                var subscriptionRequests = SubscriptionRequests;
+                SubscriptionRequests = new List<SubscriptionRequest>();
+                removedUniverses = subscriptionRequests.Where(x => x.Universe != null)
+                    .Select(x => x.Universe);
+            }
+            else
+            {
+                SubscriptionRequests.RemoveAll(x => x.Universe == universe);
+                removedUniverses = new[] {universe};
+            }
+
+            var emptySubscription = !SubscriptionRequests.Any();
+            if (emptySubscription)
+            {
+                // if the security is no longer a member of the universe, then mark the subscription properly
+                // universe may be null for internal currency conversion feeds
+                // TODO : Put currency feeds in their own internal universe
+                if (!removedUniverses.Any(x => x.Securities.ContainsKey(Configuration.Symbol)))
+                {
+                    MarkAsRemovedFromUniverse();
+                }
+            }
+
+            return emptySubscription;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public virtual bool MoveNext()
+        {
+            if (EndOfStream)
+            {
+                return false;
+            }
+
+            var moveNext = _enumerator.MoveNext();
+            EndOfStream = !moveNext;
+            Current = _enumerator.Current;
+            return moveNext;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public SubscriptionData Current { get; private set; }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            EndOfStream = true;
+            _enumerator.DisposeSafely();
+        }
+
+        /// <summary>
+        /// Mark this subscription as having been removed from the universe.
+        /// Data for this time step will be discarded.
+        /// </summary>
+        public void MarkAsRemovedFromUniverse()
+        {
+            _removedFromUniverse = true;
+        }
+
+        /// <summary>
+        /// Serves as a hash function for a particular type.
+        /// </summary>
+        /// <returns>
+        /// A hash code for the current <see cref="T:System.Object"/>.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        public override int GetHashCode()
+        {
+            return Configuration.GetHashCode();
+        }
+
+        /// <summary>Determines whether the specified object is equal to the current object.</summary>
+        /// <param name="obj">The object to compare with the current object. </param>
+        /// <returns>
+        /// <see langword="true" /> if the specified object  is equal to the current object; otherwise, <see langword="false" />.</returns>
+        public override bool Equals(object obj)
+        {
+            var subscription = obj as Subscription;
+            if (subscription == null)
+            {
+                return false;
+            }
+
+            return subscription.Configuration.Equals(Configuration);
+        }
+
+        /// <summary>Returns a string that represents the current object.</summary>
+        /// <returns>A string that represents the current object.</returns>
+        /// <filterpriority>2</filterpriority>
+        public override string ToString()
+        {
+            return Configuration.ToString();
+        }
+
+        /// <summary>
+        /// Event invocator for the <see cref="NewDataAvailable"/> event
+        /// </summary>
+        public void OnNewDataAvailable()
+        {
+            NewDataAvailable?.Invoke(this, EventArgs.Empty);
+        }
+    }
+}

Engine/DataFeeds/SubscriptionCollection.cs

@@ -0,0 +1,237 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Provides a collection for holding subscriptions.
+    /// </summary>
+    public class SubscriptionCollection : IEnumerable<Subscription>
+    {
+        private readonly ConcurrentDictionary<SubscriptionDataConfig, Subscription> _subscriptions;
+        private bool _sortingSubscriptionRequired;
+        private bool _frozen;
+        private readonly Ref<TimeSpan> _fillForwardResolution;
+
+        // some asset types (options, futures, crypto) have multiple subscriptions for different tick types,
+        // we keep a sorted list of subscriptions so we can return them in a deterministic order
+        private List<Subscription> _subscriptionsByTickType;
+
+        /// <summary>
+        /// Event fired when the fill forward resolution changes
+        /// </summary>
+        public event EventHandler<FillForwardResolutionChangedEvent> FillForwardResolutionChanged;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SubscriptionCollection"/> class
+        /// </summary>
+        public SubscriptionCollection()
+        {
+            _subscriptions = new ConcurrentDictionary<SubscriptionDataConfig, Subscription>();
+            _subscriptionsByTickType = new List<Subscription>();
+            var ffres = Time.OneMinute;
+            _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
+        }
+
+        /// <summary>
+        /// Checks the collection for the specified subscription configuration
+        /// </summary>
+        /// <param name="configuration">The subscription configuration to check for</param>
+        /// <returns>True if a subscription with the specified configuration is found in this collection, false otherwise</returns>
+        public bool Contains(SubscriptionDataConfig configuration)
+        {
+            return _subscriptions.ContainsKey(configuration);
+        }
+
+        /// <summary>
+        /// Attempts to add the specified subscription to the collection. If another subscription
+        /// exists with the same configuration then it won't be added.
+        /// </summary>
+        /// <param name="subscription">The subscription to add</param>
+        /// <returns>True if the subscription is successfully added, false otherwise</returns>
+        public bool TryAdd(Subscription subscription)
+        {
+            if (_subscriptions.TryAdd(subscription.Configuration, subscription))
+            {
+                UpdateFillForwardResolution(FillForwardResolutionOperation.AfterAdd, subscription.Configuration);
+                _sortingSubscriptionRequired = true;
+                return true;
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Attempts to retrieve the subscription with the specified configuration
+        /// </summary>
+        /// <param name="configuration">The subscription's configuration</param>
+        /// <param name="subscription">The subscription matching the configuration, null if not found</param>
+        /// <returns>True if the subscription is successfully retrieved, false otherwise</returns>
+        public bool TryGetValue(SubscriptionDataConfig configuration, out Subscription subscription)
+        {
+            return _subscriptions.TryGetValue(configuration, out subscription);
+        }
+
+        /// <summary>
+        /// Attempts to remove the subscription with the specified configuraton from the collection.
+        /// </summary>
+        /// <param name="configuration">The configuration of the subscription to remove</param>
+        /// <param name="subscription">The removed subscription, null if not found.</param>
+        /// <returns>True if the subscription is successfully removed, false otherwise</returns>
+        public bool TryRemove(SubscriptionDataConfig configuration, out Subscription subscription)
+        {
+            if (_subscriptions.TryRemove(configuration, out subscription))
+            {
+                // for user friendlyness only look at removals triggerd by the user not those that are due to a data feed ending because of no more data,
+                // let's try to respect the users original FF enumerator request
+                if (!subscription.EndOfStream)
+                {
+                    UpdateFillForwardResolution(FillForwardResolutionOperation.AfterRemove, configuration);
+                }
+                _sortingSubscriptionRequired = true;
+                return true;
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through the collection.
+        /// </summary>
+        /// <returns>
+        /// An enumerator that can be used to iterate through the collection.
+        /// </returns>
+        public IEnumerator<Subscription> GetEnumerator()
+        {
+            SortSubscriptions();
+            return _subscriptionsByTickType.GetEnumerator();
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through a collection.
+        /// </summary>
+        /// <returns>
+        /// An <see cref="T:System.Collections.IEnumerator"/> object that can be used to iterate through the collection.
+        /// </returns>
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+
+        /// <summary>
+        /// Gets and updates the fill forward resolution by checking specified subscription configurations and
+        /// selecting the smallest resoluton not equal to tick
+        /// </summary>
+        public Ref<TimeSpan> UpdateAndGetFillForwardResolution(SubscriptionDataConfig configuration = null)
+        {
+            if (configuration != null)
+            {
+                UpdateFillForwardResolution(FillForwardResolutionOperation.BeforeAdd, configuration);
+            }
+            return _fillForwardResolution;
+        }
+
+        /// <summary>
+        /// Will disable or enable fill forward resolution updates
+        /// </summary>
+        public void FreezeFillForwardResolution(bool freeze)
+        {
+            _frozen = freeze;
+        }
+
+        /// <summary>
+        /// Helper method to validate a configuration to be included in the fill forward calculation
+        /// </summary>
+        private static bool ValidateFillForwardResolution(SubscriptionDataConfig configuration)
+        {
+            return !configuration.IsInternalFeed && configuration.Resolution != Resolution.Tick;
+        }
+        /// <summary>
+        /// Gets and updates the fill forward resolution by checking specified subscription configurations and
+        /// selecting the smallest resoluton not equal to tick
+        /// </summary>
+        private void UpdateFillForwardResolution(FillForwardResolutionOperation operation, SubscriptionDataConfig configuration)
+        {
+            if(_frozen)
+            {
+                return;
+            }
+
+            // Due to performance implications let's be jealous in updating the _fillForwardResolution
+            if (ValidateFillForwardResolution(configuration) &&
+                (
+                    ((FillForwardResolutionOperation.BeforeAdd == operation || FillForwardResolutionOperation.AfterAdd == operation)
+                     && configuration.Increment != _fillForwardResolution.Value) // check if the new Increment is different
+                ||
+                    (operation == FillForwardResolutionOperation.AfterRemove // We are removing
+                    && configuration.Increment == _fillForwardResolution.Value // True: We are removing the resolution we were using
+                    // False: there is at least another one equal, no need to update, but we only look at those valid configuration which are the ones which set the FF resolution
+                    && _subscriptions.Keys.All(x => !ValidateFillForwardResolution(x) || x.Resolution != configuration.Resolution)))
+                )
+            {
+                var configurations = (operation == FillForwardResolutionOperation.BeforeAdd)
+                    ? _subscriptions.Keys.Concat(new[] { configuration }) : _subscriptions.Keys;
+
+                var eventArgs = new FillForwardResolutionChangedEvent { Old = _fillForwardResolution.Value };
+                _fillForwardResolution.Value = configurations.Where(ValidateFillForwardResolution)
+                                                             .Select(x => x.Resolution)
+                                                             .Distinct()
+                                                             .DefaultIfEmpty(Resolution.Minute)
+                                                             .Min().ToTimeSpan();
+                if (_fillForwardResolution.Value != eventArgs.Old)
+                {
+                    eventArgs.New = _fillForwardResolution.Value;
+                    // notify consumers if any
+                    FillForwardResolutionChanged?.Invoke(this, eventArgs);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Sorts subscriptions so that equity subscriptions are enumerated before option
+        /// securities to ensure the underlying data is available when we process the options data
+        /// </summary>
+        private void SortSubscriptions()
+        {
+            if (_sortingSubscriptionRequired)
+            {
+                _sortingSubscriptionRequired = false;
+                // it's important that we enumerate underlying securities before derivatives to this end,
+                // we order by security type so that equity subscriptions are enumerated before option
+                // securities to ensure the underlying data is available when we process the options data
+                _subscriptionsByTickType = _subscriptions
+                    .Select(x => x.Value)
+                    .OrderBy(x => x.Configuration.SecurityType)
+                    .ThenBy(x => x.Configuration.TickType)
+                    .ThenBy(x => x.Configuration.Symbol)
+                    .ToList();
+            }
+        }
+
+        private enum FillForwardResolutionOperation
+        {
+            AfterRemove,
+            BeforeAdd,
+            AfterAdd
+        }
+    }
+}

Engine/DataFeeds/SubscriptionData.cs

@@ -0,0 +1,121 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Store data (either raw or adjusted) and the time at which it should be synchronized
+    /// </summary>
+    public class SubscriptionData
+    {
+        /// <summary>
+        /// Data
+        /// </summary>
+        protected BaseData _data { get; set; }
+
+        /// <summary>
+        /// Gets the data
+        /// </summary>
+        public virtual BaseData Data => _data;
+
+        /// <summary>
+        /// Gets the UTC emit time for this data
+        /// </summary>
+        public DateTime EmitTimeUtc { get; }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SubscriptionData"/> class
+        /// </summary>
+        /// <param name="data">The base data</param>
+        /// <param name="emitTimeUtc">The emit time for the data</param>
+        public SubscriptionData(BaseData data, DateTime emitTimeUtc)
+        {
+            _data = data;
+            EmitTimeUtc = emitTimeUtc;
+        }
+
+        /// <summary>
+        /// Clones the data, computes the utc emit time and performs exchange round down behavior, storing the result in a new <see cref="SubscriptionData"/> instance
+        /// </summary>
+        /// <param name="configuration">The subscription's configuration</param>
+        /// <param name="exchangeHours">The exchange hours of the security</param>
+        /// <param name="offsetProvider">The subscription's offset provider</param>
+        /// <param name="data">The data being emitted</param>
+        /// <param name="normalizationMode">Specifies how data is normalized</param>
+        /// <param name="factor">price scale factor</param>
+        /// <returns>A new <see cref="SubscriptionData"/> containing the specified data</returns>
+        public static SubscriptionData Create(bool dailyStrictEndTimeEnabled, SubscriptionDataConfig configuration, SecurityExchangeHours exchangeHours, TimeZoneOffsetProvider offsetProvider, BaseData data, DataNormalizationMode normalizationMode, decimal? factor = null)
+        {
+            if (data == null)
+            {
+                return null;
+            }
+
+            data = data.Clone(data.IsFillForward);
+            var emitTimeUtc = offsetProvider.ConvertToUtc(data.EndTime);
+            // during warmup, data might be emitted with a different span based on the warmup resolution, so let's get the actual bar span here
+            var barSpan = data.EndTime - data.Time;
+            // rounding down does not make sense for daily increments using strict end times
+            if (!LeanData.UseDailyStrictEndTimes(dailyStrictEndTimeEnabled, configuration.Type, configuration.Symbol, barSpan, exchangeHours))
+            {
+                // Let's round down for any data source that implements a time delta between
+                // the start of the data and end of the data (usually used with Bars).
+                // The time delta ensures that the time collected from `EndTime` has
+                // no look-ahead bias, and is point-in-time.
+                // When fill forwarding time and endtime might not respect the original ends times, here we will enforce it
+                // note we do this after fetching the 'emitTimeUtc' which should use the end time set by the fill forward enumerator
+                if (barSpan != TimeSpan.Zero)
+                {
+                    if (barSpan != configuration.Increment)
+                    {
+                        // when we detect a difference let's refetch the span in utc using noda time 'ConvertToUtc' that will not take into account day light savings difference
+                        // we don't do this always above because it's expensive, only do it if we need to.
+                        // Behavior asserted by tests 'FillsForwardBarsAroundDaylightMovementForDifferentResolutions_Algorithm' && 'ConvertToUtcAndDayLightSavings'.
+                        // Note: we don't use 'configuration.Increment' because during warmup, if the warmup resolution is set, we will emit data respecting it instead of the 'configuration'
+                        barSpan = data.EndTime.ConvertToUtc(configuration.ExchangeTimeZone) - data.Time.ConvertToUtc(configuration.ExchangeTimeZone);
+                    }
+                    data.Time = data.Time.ExchangeRoundDownInTimeZone(barSpan, exchangeHours, configuration.DataTimeZone, configuration.ExtendedMarketHours);
+                }
+            }
+            else if (data.IsFillForward)
+            {
+                // we need to adjust the time for a strict end time daily bar:
+                // If this is fill-forwarded with a lower resolution, the daily calendar for data.Time will be for the previous date
+                // (which is correct, since the last daily bar belongs to the previous date).
+                // If this is a fill-forwarded complete daily bar (ending at market close),
+                // the daily calendar will have the same time/end time so the bar times will not be adjusted.
+                // TODO: What about extended market hours? How to handle non-adjacent market hour segments in a day? Same in FillForwardEnumerator
+                var calendar = LeanData.GetDailyCalendar(data.Time, exchangeHours, false);
+                data.Time = calendar.Start;
+                data.EndTime = calendar.End;
+            }
+
+            if (factor.HasValue && (configuration.SecurityType != SecurityType.Equity || (factor.Value != 1 || configuration.SumOfDividends != 0)))
+            {
+                var normalizedData = data.Clone(data.IsFillForward).Normalize(factor.Value, normalizationMode, configuration.SumOfDividends);
+
+                return new PrecalculatedSubscriptionData(configuration, data, normalizedData, normalizationMode, emitTimeUtc);
+            }
+
+            return new SubscriptionData(data, emitTimeUtc);
+        }
+    }
+}