chore: import upstream snapshot with attribution

Engine/DataFeeds/Enumerators/AuxiliaryDataEnumerator.cs

@@ -0,0 +1,149 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using System.Collections;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Auxiliary data enumerator that will, initialize and call the <see cref="ITradableDateEventProvider.GetEvents"/>
+    /// implementation each time there is a new tradable day for every <see cref="ITradableDateEventProvider"/>
+    /// provided.
+    /// </summary>
+    public class AuxiliaryDataEnumerator : IEnumerator<BaseData>
+    {
+        private readonly Queue<BaseData> _auxiliaryData;
+        private bool _initialized;
+        private DateTime _startTime;
+        private IMapFileProvider _mapFileProvider;
+        private IFactorFileProvider _factorFileProvider;
+        private ITradableDateEventProvider[] _tradableDateEventProviders;
+
+        /// <summary>
+        /// The associated data configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; }
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="tradableDateEventProviders">The tradable dates event providers</param>
+        /// <param name="tradableDayNotifier">Tradable dates provider</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public AuxiliaryDataEnumerator(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            ITradableDateEventProvider []tradableDateEventProviders,
+            ITradableDatesNotifier tradableDayNotifier,
+            DateTime startTime)
+        {
+            Config = config;
+            _startTime = startTime;
+            _mapFileProvider = mapFileProvider;
+            _auxiliaryData = new Queue<BaseData>();
+            _factorFileProvider = factorFileProvider;
+            _tradableDateEventProviders = tradableDateEventProviders;
+
+            if (tradableDayNotifier != null)
+            {
+                tradableDayNotifier.NewTradableDate += NewTradableDate;
+            }
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element.
+        /// </summary>
+        /// <returns>Always true</returns>
+        public virtual bool MoveNext()
+        {
+            Current = _auxiliaryData.Count != 0 ? _auxiliaryData.Dequeue() : null;
+            return true;
+        }
+
+        /// <summary>
+        /// Handle a new tradable date, drives the <see cref="ITradableDateEventProvider"/> instances
+        /// </summary>
+        protected void NewTradableDate(object sender, NewTradableDateEventArgs eventArgs)
+        {
+            Initialize();
+            for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+            {
+                foreach (var newEvent in _tradableDateEventProviders[i].GetEvents(eventArgs))
+                {
+                    _auxiliaryData.Enqueue(newEvent);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the underlying tradable data event providers
+        /// </summary>
+        protected void Initialize()
+        {
+            if (!_initialized)
+            {
+                _initialized = true;
+                // Late initialization so it is performed in the data feed stack
+                for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+                {
+                    _tradableDateEventProviders[i].Initialize(Config, _factorFileProvider, _mapFileProvider, _startTime);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Dispose of the Stream Reader and close out the source stream and file connections.
+        /// </summary>
+        public void Dispose()
+        {
+            for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+            {
+                var disposable =_tradableDateEventProviders[i] as IDisposable;
+                disposable?.DisposeSafely();
+            }
+        }
+
+        /// <summary>
+        /// Reset the IEnumeration
+        /// </summary>
+        /// <remarks>Not used</remarks>
+        public void Reset()
+        {
+            throw new NotImplementedException("Reset method not implemented. Assumes loop will only be used once.");
+        }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Last read BaseData object from this type and source
+        /// </summary>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/BaseDataCollectionAggregatorEnumerator.cs

@@ -0,0 +1,234 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="IEnumerator{BaseDataCollection}"/>
+    /// that aggregates an underlying <see cref="IEnumerator{BaseData}"/> into a single
+    /// data packet
+    /// </summary>
+    public class BaseDataCollectionAggregatorEnumerator : IEnumerator<BaseDataCollection>
+    {
+        private bool _endOfStream;
+        private bool _needsMoveNext;
+        private bool _liveMode;
+        private readonly Symbol _symbol;
+        private readonly IEnumerator<BaseData> _enumerator;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="BaseDataCollectionAggregatorEnumerator"/> class
+        /// This will aggregate instances emitted from the underlying enumerator and tag them with the
+        /// specified symbol
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to aggregate</param>
+        /// <param name="symbol">The symbol to place on the aggregated collection</param>
+        /// <param name="liveMode">True if running in live mode</param>
+        public BaseDataCollectionAggregatorEnumerator(IEnumerator<BaseData> enumerator, Symbol symbol, bool liveMode = false)
+        {
+            _symbol = symbol;
+            _enumerator = enumerator;
+            _liveMode = liveMode;
+            _needsMoveNext = true;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            if (_endOfStream)
+            {
+                return false;
+            }
+
+            BaseDataCollection collection = null;
+            while (true)
+            {
+                if (_needsMoveNext)
+                {
+                    // move next if we dequeued the last item last time we were invoked
+                    if (!_enumerator.MoveNext())
+                    {
+                        _endOfStream = true;
+                        if (!IsValid(collection))
+                        {
+                            // we don't emit
+                            collection = null;
+                        }
+                        break;
+                    }
+                }
+
+                if (_enumerator.Current == null)
+                {
+                    // the underlying returned null, stop here and start again on the next call
+                    _needsMoveNext = true;
+                    break;
+                }
+
+                if (collection == null)
+                {
+                    // we have new data, set the collection's symbol/times
+                    var current = _enumerator.Current;
+                    collection = CreateCollection(_symbol, current.Time, current.EndTime);
+                }
+
+                if (collection.EndTime != _enumerator.Current.EndTime)
+                {
+                    // the data from the underlying is at a different time, stop here
+                    _needsMoveNext = false;
+                    if (IsValid(collection))
+                    {
+                        // we emit
+                        break;
+                    }
+                    // we try again
+                    collection = null;
+                    continue;
+                }
+
+                // this data belongs in this collection, keep going until null or bad time
+                Add(collection, _enumerator.Current);
+                _needsMoveNext = true;
+            }
+
+            Current = collection;
+            return _liveMode || collection != null;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseDataCollection Current
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Creates a new, empty <see cref="BaseDataCollection"/>.
+        /// </summary>
+        /// <param name="symbol">The base data collection symbol</param>
+        /// <param name="time">The start time of the collection</param>
+        /// <param name="endTime">The end time of the collection</param>
+        /// <returns>A new, empty <see cref="BaseDataCollection"/></returns>
+        private BaseDataCollection CreateCollection(Symbol symbol, DateTime time, DateTime endTime)
+        {
+            return new BaseDataCollection
+            {
+                Symbol = symbol,
+                Time = time,
+                EndTime = endTime
+            };
+        }
+
+        /// <summary>
+        /// Adds the specified instance of <see cref="BaseData"/> to the current collection
+        /// </summary>
+        /// <param name="collection">The collection to be added to</param>
+        /// <param name="current">The data to be added</param>
+        private void Add(BaseDataCollection collection, BaseData current)
+        {
+            var baseDataCollection = current as BaseDataCollection;
+            if (_symbol.HasUnderlying && _symbol.Underlying == current.Symbol)
+            {
+                // if the underlying has been aggregated, even if it shouldn't need to be, let's handle it nicely
+                if (baseDataCollection != null)
+                {
+                    collection.Underlying = baseDataCollection.Data[0];
+                }
+                else
+                {
+                    collection.Underlying = current;
+                }
+            }
+            else
+            {
+                if (baseDataCollection != null)
+                {
+                    // datapoint is already aggregated, let's see if it's a single point or a collection we can use already
+                    if(baseDataCollection.Data.Count > 1)
+                    {
+                        collection.Data = baseDataCollection.Data;
+                    }
+                    else
+                    {
+                        collection.Data.Add(baseDataCollection.Data[0]);
+                    }
+
+                    // Let's keep the underlying in case it's already there
+                    collection.Underlying ??= baseDataCollection.Underlying;
+                }
+                else
+                {
+                    collection.Data.Add(current);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Determines if a given data point is valid and can be emitted
+        /// </summary>
+        /// <param name="collection">The collection to be emitted</param>
+        /// <returns>True if its a valid data point</returns>
+        private static bool IsValid(BaseDataCollection collection)
+        {
+            return collection != null && collection.Data?.Count > 0;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/ConcatEnumerator.cs

@@ -0,0 +1,133 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using System.Collections;
+using QuantConnect.Logging;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that will concatenate enumerators together sequentially enumerating them in the provided order
+    /// </summary>
+    public class ConcatEnumerator : IEnumerator<BaseData>
+    {
+        private readonly List<IEnumerator<BaseData>> _enumerators;
+        private readonly bool _skipDuplicateEndTimes;
+        private DateTime? _lastEnumeratorEndTime;
+        private int _currentIndex;
+
+        /// <summary>
+        /// The current BaseData object
+        /// </summary>
+        public BaseData Current { get; set; }
+
+        /// <summary>
+        /// True if emitting a null data point is expected
+        /// </summary>
+        /// <remarks>Warmup enumerators are not allowed to return true and setting current to Null, this is because it's not a valid behavior for backtesting enumerators,
+        /// for example <see cref="FillForwardEnumerator"/></remarks>
+        public bool CanEmitNull { get; set; }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="skipDuplicateEndTimes">True will skip data points from enumerators if before or at the last end time</param>
+        /// <param name="enumerators">The sequence of enumerators to concatenate. Note that the order here matters, it will consume enumerators
+        /// and dispose of them, even if they return true and their current is null, except for the last which will be kept!</param>
+        public ConcatEnumerator(bool skipDuplicateEndTimes,
+            params IEnumerator<BaseData>[] enumerators
+            )
+        {
+            CanEmitNull = true;
+            _skipDuplicateEndTimes = skipDuplicateEndTimes;
+            _enumerators = enumerators.Where(enumerator => enumerator != null).ToList();
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>True if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.</returns>
+        public bool MoveNext()
+        {
+            for (; _currentIndex < _enumerators.Count; _currentIndex++)
+            {
+                var enumerator = _enumerators[_currentIndex];
+                while (enumerator.MoveNext())
+                {
+                    if (enumerator.Current == null && (_currentIndex < _enumerators.Count - 1 || !CanEmitNull))
+                    {
+                        // if there are more enumerators and the current stopped providing data drop it
+                        // in live trading, some enumerators will always return true (see TimeTriggeredUniverseSubscriptionEnumeratorFactory & InjectionEnumerator)
+                        // but unless it's the last enumerator we drop it, because these first are the warmup enumerators
+                        // or we are not allowed to return null
+                        break;
+                    }
+
+                    if (_skipDuplicateEndTimes
+                        && _lastEnumeratorEndTime.HasValue
+                        && enumerator.Current != null
+                        && enumerator.Current.EndTime <= _lastEnumeratorEndTime)
+                    {
+                        continue;
+                    }
+
+                    Current = enumerator.Current;
+                    return true;
+                }
+
+                _lastEnumeratorEndTime = Current?.EndTime;
+
+                if (Log.DebuggingEnabled)
+                {
+                    Log.Debug($"ConcatEnumerator.MoveNext(): disposing enumerator at position: {_currentIndex} Name: {enumerator.GetType().Name}");
+                }
+
+                // we wont be using this enumerator again, dispose of it and clear reference
+                enumerator.DisposeSafely();
+                _enumerators[_currentIndex] = null;
+            }
+
+            Current = null;
+            return false;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        public void Reset()
+        {
+            throw new InvalidOperationException($"Can not reset {nameof(ConcatEnumerator)}");
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            foreach (var enumerator in _enumerators)
+            {
+                enumerator.DisposeSafely();
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/DelistingEventProvider.cs

@@ -0,0 +1,118 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="Delisting"/> events
+    /// </summary>
+    public class DelistingEventProvider : ITradableDateEventProvider
+    {
+        // we'll use these flags to denote we've already fired off the DelistingType.Warning
+        // and a DelistedType.Delisted Delisting object, the _delistingType object is save here
+        // since we need to wait for the next trading day before emitting
+        private bool _delisted;
+        private bool _delistedWarning;
+        private IMapFileProvider _mapFileProvider;
+
+        /// <summary>
+        /// The delisting date
+        /// </summary>
+        protected ReferenceWrapper<DateTime> DelistingDate { get; set; }
+
+        /// <summary>
+        /// The current instance being used
+        /// </summary>
+        protected MapFile MapFile { get; private set; }
+
+        /// <summary>
+        /// The associated configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; private set; }
+
+        /// <summary>
+        /// Initializes this instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public virtual void Initialize(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime)
+        {
+            Config = config;
+            _mapFileProvider = mapFileProvider;
+
+            InitializeMapFile();
+        }
+
+        /// <summary>
+        /// Check for delistings
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New delisting event if any</returns>
+        public virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            if (Config.Symbol == eventArgs.Symbol)
+            {
+                // we send the delisting warning when we reach the delisting date, here we make sure we compare using the date component
+                // of the delisting date since for example some futures can trade a few hours in their delisting date, else we would skip on
+                // emitting the delisting warning, which triggers us to handle liquidation once delisted
+                if (!_delistedWarning && eventArgs.Date >= DelistingDate.Value.Date)
+                {
+                    _delistedWarning = true;
+                    var price = eventArgs.LastBaseData?.Price ?? 0;
+                    yield return new Delisting(
+                        eventArgs.Symbol,
+                        DelistingDate.Value.Date,
+                        price,
+                        DelistingType.Warning);
+                }
+                if (!_delisted && eventArgs.Date > DelistingDate.Value)
+                {
+                    _delisted = true;
+                    var price = eventArgs.LastBaseData?.Price ?? 0;
+                    // delisted at EOD
+                    yield return new Delisting(
+                        eventArgs.Symbol,
+                        DelistingDate.Value.AddDays(1),
+                        price,
+                        DelistingType.Delisted);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the factor file to use
+        /// </summary>
+        protected void InitializeMapFile()
+        {
+            MapFile = _mapFileProvider.ResolveMapFile(Config);
+            DelistingDate = new ReferenceWrapper<DateTime>(Config.Symbol.GetDelistingDate(MapFile));
+        }
+    }
+}

Engine/DataFeeds/Enumerators/DividendEventProvider.cs

@@ -0,0 +1,118 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="Dividend"/> events
+    /// </summary>
+    public class DividendEventProvider : ITradableDateEventProvider
+    {
+        // we set the price factor ratio when we encounter a dividend in the factor file
+        // and on the next trading day we use this data to produce the dividend instance
+        private decimal? _priceFactorRatio;
+        private decimal _referencePrice;
+        private IFactorFileProvider _factorFileProvider;
+        private MapFile _mapFile;
+
+        /// <summary>
+        /// The current instance being used
+        /// </summary>
+        protected CorporateFactorProvider FactorFile { get; private set; }
+
+        /// <summary>
+        /// The associated configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; private set; }
+
+        /// <summary>
+        /// Initializes this instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public void Initialize(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime)
+        {
+            Config = config;
+            _factorFileProvider = factorFileProvider;
+            _mapFile = mapFileProvider.ResolveMapFile(Config);
+            InitializeFactorFile();
+        }
+
+        /// <summary>
+        /// Check for dividends and returns them
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New Dividend event if any</returns>
+        public  virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            if (Config.Symbol == eventArgs.Symbol
+                && FactorFile != null
+                && _mapFile.HasData(eventArgs.Date))
+            {
+                if (_priceFactorRatio != null)
+                {
+                    if (_referencePrice == 0)
+                    {
+                        throw new InvalidOperationException($"Zero reference price for {Config.Symbol} dividend at {eventArgs.Date}");
+                    }
+
+                    var baseData = Dividend.Create(
+                        Config.Symbol,
+                        eventArgs.Date,
+                        _referencePrice,
+                        _priceFactorRatio.Value
+                    );
+                    // let the config know about it for normalization
+                    Config.SumOfDividends += baseData.Distribution;
+                    _priceFactorRatio = null;
+                    _referencePrice = 0;
+
+                    yield return baseData;
+                }
+
+                // check the factor file to see if we have a dividend event tomorrow
+                decimal priceFactorRatio;
+                decimal referencePrice;
+                if (FactorFile.HasDividendEventOnNextTradingDay(eventArgs.Date, out priceFactorRatio, out referencePrice))
+                {
+                    _priceFactorRatio = priceFactorRatio;
+                    _referencePrice = referencePrice;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the factor file to use
+        /// </summary>
+        protected void InitializeFactorFile()
+        {
+            FactorFile = _factorFileProvider.Get(Config.Symbol) as CorporateFactorProvider;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/EnqueueableEnumerator.cs

@@ -0,0 +1,214 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Threading;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// An implementation of <see cref="IEnumerator{T}"/> that relies on the
+    /// <see cref="Enqueue"/> method being called and only ends when <see cref="Stop"/>
+    /// is called
+    /// </summary>
+    /// <typeparam name="T">The item type yielded by the enumerator</typeparam>
+    public class EnqueueableEnumerator<T> : IEnumerator<T>
+    {
+        private T _current;
+        private bool _end;
+
+        private readonly bool _isBlocking;
+        private long _consumerCount;
+        private Queue<T> _consumer = new();
+        private Queue<T> _producer = new();
+        private readonly object _lock = new object();
+        private readonly ManualResetEventSlim _resetEvent = new(false);
+
+        /// <summary>
+        /// Gets the current number of items held in the internal queue
+        /// </summary>
+        public int Count
+        {
+            get
+            {
+                lock (_lock)
+                {
+                    if (_end) return 0;
+                    return _producer.Count + (int)Interlocked.Read(ref _consumerCount);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Returns true if the enumerator has finished and will not accept any more data
+        /// </summary>
+        public bool HasFinished
+        {
+            get { return _end; }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="EnqueueableEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="blocking">Specifies whether or not to use the blocking behavior</param>
+        public EnqueueableEnumerator(bool blocking = false)
+        {
+            _isBlocking = blocking;
+        }
+
+        /// <summary>
+        /// Enqueues the new data into this enumerator
+        /// </summary>
+        /// <param name="data">The data to be enqueued</param>
+        public void Enqueue(T data)
+        {
+            lock (_lock)
+            {
+                _producer.Enqueue(data);
+                // most of the time this will be set
+                if(!_resetEvent.IsSet)
+                {
+                    _resetEvent.Set();
+                }
+            }
+        }
+
+        /// <summary>
+        /// Signals the enumerator to stop enumerating when the items currently
+        /// held inside are gone. No more items will be added to this enumerator.
+        /// </summary>
+        public void Stop()
+        {
+            lock (_lock)
+            {
+                if (_end) return;
+                _end = true;
+
+                // no more items can be added, so no need to wait anymore
+                _resetEvent.Set();
+                _resetEvent.Dispose();
+            }
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            // we read with no lock most of the time
+            if (_consumer.TryDequeue(out _current))
+            {
+                Interlocked.Decrement(ref _consumerCount);
+                return true;
+            }
+
+            bool ended;
+            do
+            {
+                var producer = _producer;
+                lock (_lock)
+                {
+                    // swap queues
+                    ended = _end;
+                    _producer = _consumer;
+                }
+                _consumer = producer;
+                if(_consumer.Count > 0)
+                {
+                    _current = _consumer.Dequeue();
+                    Interlocked.Exchange(ref _consumerCount, _consumer.Count);
+                    break;
+                }
+
+                // if we are here no queue has data
+                if (ended)
+                {
+                    return false;
+                }
+
+                if (_isBlocking)
+                {
+                    try
+                    {
+                        _resetEvent.Wait(Timeout.Infinite);
+                        _resetEvent.Reset();
+                    }
+                    catch (ObjectDisposedException)
+                    {
+                        // can happen if disposed
+                    }
+                }
+                else
+                {
+                    break;
+                }
+            }
+            while (!ended);
+
+            // even if we don't have data to return, we haven't technically
+            // passed the end of the collection, so always return true until
+            // the enumerator is explicitly disposed or ended
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            throw new NotImplementedException("EnqueableEnumerator.Reset() has not been implemented yet.");
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            Stop();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/BaseDataCollectionSubscriptionEnumeratorFactory.cs

@@ -0,0 +1,74 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ISubscriptionEnumeratorFactory"/> that reads
+    /// an entire <see cref="SubscriptionDataSource"/> into a single <see cref="BaseDataCollection"/>
+    /// to be emitted on the tradable date at midnight
+    /// </summary>
+    /// <remarks>This enumerator factory is currently only used in backtesting with coarse data</remarks>
+    public class BaseDataCollectionSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
+    {
+        private IObjectStore _objectStore;
+
+        /// <summary>
+        /// Instanciates a new <see cref="BaseDataCollectionSubscriptionEnumeratorFactory"/>
+        /// </summary>
+        /// <param name="objectStore">The object store to use</param>
+        public BaseDataCollectionSubscriptionEnumeratorFactory(IObjectStore objectStore)
+        {
+            _objectStore = objectStore;
+        }
+
+        /// <summary>
+        /// Creates an enumerator to read the specified request
+        /// </summary>
+        /// <param name="request">The subscription request to be read</param>
+        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <returns>An enumerator reading the subscription request</returns>
+        public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
+        {
+            using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
+            {
+                var configuration = request.Configuration;
+                var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
+
+                // Behaves in the same way as in live trading
+                // (i.e. only emit coarse data on dates following a trading day)
+                // The shifting of dates is needed to ensure we never emit coarse data on the same date,
+                // because it would enable look-ahead bias.
+
+                foreach (var date in request.TradableDaysInDataTimeZone)
+                {
+                    var source = sourceFactory.GetSource(configuration, date, false);
+                    var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, configuration, date, false, sourceFactory,
+                        dataProvider, _objectStore);
+                    var coarseFundamentalForDate = factory.Read(source);
+                    //  shift all date of emitting the file forward one day to model emitting coarse midnight the next day.
+                    yield return new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate);
+                }
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/CorporateEventEnumeratorFactory.cs

@@ -0,0 +1,101 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Helper class used to create the corporate event providers
+    /// <see cref="MappingEventProvider"/>, <see cref="SplitEventProvider"/>,
+    /// <see cref="DividendEventProvider"/>, <see cref="DelistingEventProvider"/>
+    /// </summary>
+    public static class CorporateEventEnumeratorFactory
+    {
+        /// <summary>
+        /// Creates a new <see cref="AuxiliaryDataEnumerator"/> that will hold the
+        /// corporate event providers
+        /// </summary>
+        /// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">Used for getting factor files</param>
+        /// <param name="tradableDayNotifier">Tradable dates provider</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        /// <param name="endTime">
+        /// End date for the data request.
+        /// This will be used for <see cref="DataNormalizationMode.ScaledRaw"/> data normalization mode to adjust prices to the given end date
+        /// </param>
+        /// <param name="enablePriceScaling">Applies price factor</param>
+        /// <returns>The new auxiliary data enumerator</returns>
+        public static IEnumerator<BaseData> CreateEnumerators(
+            IEnumerator<BaseData> rawDataEnumerator,
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            ITradableDatesNotifier tradableDayNotifier,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime,
+            DateTime endTime,
+            bool enablePriceScaling = true)
+        {
+
+            var tradableEventProviders = new List<ITradableDateEventProvider>();
+
+            if (config.EmitSplitsAndDividends())
+            {
+                tradableEventProviders.Add(new SplitEventProvider());
+                tradableEventProviders.Add(new DividendEventProvider());
+            }
+
+            if (config.TickerShouldBeMapped())
+            {
+                tradableEventProviders.Add(new MappingEventProvider());
+            }
+
+            if (config.CanBeDelisted())
+            {
+                tradableEventProviders.Add(new DelistingEventProvider());
+            }
+
+            var enumerator = new AuxiliaryDataEnumerator(
+                config,
+                factorFileProvider,
+                mapFileProvider,
+                tradableEventProviders.ToArray(),
+                tradableDayNotifier,
+                startTime);
+
+            // avoid price scaling for backtesting; calculate it directly in worker
+            // and allow subscription to extract the the data depending on config data mode
+            var dataEnumerator = rawDataEnumerator;
+            if (enablePriceScaling && config.PricesShouldBeScaled())
+            {
+                dataEnumerator = new PriceScaleFactorEnumerator(
+                    rawDataEnumerator,
+                    config,
+                    factorFileProvider,
+                    endDate: endTime);
+            }
+
+            return new SynchronizingBaseDataEnumerator(dataEnumerator, enumerator);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs

@@ -0,0 +1,216 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Python.Runtime;
+using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ISubscriptionEnumeratorFactory"/> to handle live custom data.
+    /// </summary>
+    public class LiveCustomDataSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
+    {
+        private readonly TimeSpan _minimumIntervalCheck;
+        private readonly ITimeProvider _timeProvider;
+        private readonly Func<DateTime, DateTime> _dateAdjustment;
+        private readonly IObjectStore _objectStore;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LiveCustomDataSubscriptionEnumeratorFactory"/> class
+        /// </summary>
+        /// <param name="timeProvider">Time provider from data feed</param>
+        /// <param name="objectStore">The object store to use</param>
+        /// <param name="dateAdjustment">Func that allows adjusting the datetime to use</param>
+        /// <param name="minimumIntervalCheck">Allows specifying the minimum interval between each enumerator refresh and data check, default is 30 minutes</param>
+        public LiveCustomDataSubscriptionEnumeratorFactory(ITimeProvider timeProvider, IObjectStore objectStore,
+            Func<DateTime, DateTime> dateAdjustment = null, TimeSpan? minimumIntervalCheck = null)
+        {
+            _timeProvider = timeProvider;
+            _dateAdjustment = dateAdjustment;
+            _minimumIntervalCheck = minimumIntervalCheck ?? TimeSpan.FromMinutes(30);
+            _objectStore = objectStore;
+        }
+
+        /// <summary>
+        /// Creates an enumerator to read the specified request.
+        /// </summary>
+        /// <param name="request">The subscription request to be read</param>
+        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <returns>An enumerator reading the subscription request</returns>
+        public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
+        {
+            var config = request.Configuration;
+
+            // frontier value used to prevent emitting duplicate time stamps between refreshed enumerators
+            // also provides some immediate fast-forward to handle spooling through remote files quickly
+            var frontier = Ref.Create(_dateAdjustment?.Invoke(request.StartTimeLocal) ?? request.StartTimeLocal);
+            var lastSourceRefreshTime = DateTime.MinValue;
+            var sourceFactory = config.GetBaseDataInstance();
+
+            // this is refreshing the enumerator stack for each new source
+            var refresher = new RefreshEnumerator<BaseData>(() =>
+            {
+                // rate limit the refresh of this enumerator stack
+                var utcNow = _timeProvider.GetUtcNow();
+                var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment, _minimumIntervalCheck);
+                if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls)
+                {
+                    return Enumerable.Empty<BaseData>().GetEnumerator();
+                }
+
+                lastSourceRefreshTime = utcNow;
+                var localDate = _dateAdjustment?.Invoke(utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date) ?? utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date;
+                var source = sourceFactory.GetSource(config, localDate, true);
+
+                // fetch the new source and enumerate the data source reader
+                var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate, sourceFactory);
+
+                if (SourceRequiresFastForward(source))
+                {
+                    // The FastForwardEnumerator implements these two features:
+                    // (1) make sure we never emit past data
+                    // (2) data filtering based on a maximum data age
+                    // For custom data we don't want feature (2) because we would reject data points emitted later
+                    // (e.g. Quandl daily data after a weekend), so we disable it using a huge maximum data age.
+
+                    // apply fast forward logic for file transport mediums
+                    var maximumDataAge = GetMaximumDataAge(Time.MaxTimeSpan);
+                    enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge);
+                }
+                else
+                {
+                    // rate limit calls to this enumerator stack
+                    enumerator = new RateLimitEnumerator<BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls);
+                }
+
+                if (source.Format == FileFormat.UnfoldingCollection)
+                {
+                    // unroll collections into individual data points after fast forward/rate limiting applied
+                    enumerator = enumerator.SelectMany(data =>
+                    {
+                        var collection = data as BaseDataCollection;
+                        IEnumerator<BaseData> collectionEnumerator;
+                        if (collection != null)
+                        {
+                            if (source.TransportMedium == SubscriptionTransportMedium.Rest || source.TransportMedium == SubscriptionTransportMedium.RemoteFile)
+                            {
+                                // we want to make sure the data points we *unroll* are not past
+                                collectionEnumerator = collection.Data
+                                    .Where(baseData => baseData.EndTime > frontier.Value)
+                                    .GetEnumerator();
+                            }
+                            else
+                            {
+                                collectionEnumerator = collection.Data.GetEnumerator();
+                            }
+                        }
+                        else
+                        {
+                            collectionEnumerator = new List<BaseData> { data }.GetEnumerator();
+                        }
+                        return collectionEnumerator;
+                    });
+                }
+
+                return enumerator;
+            });
+
+            return refresher;
+        }
+
+        private IEnumerator<BaseData> EnumerateDataSourceReader(SubscriptionDataConfig config, IDataProvider dataProvider, Ref<DateTime> localFrontier, SubscriptionDataSource source, DateTime localDate, BaseData baseDataInstance)
+        {
+            using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
+            {
+                var newLocalFrontier = localFrontier.Value;
+                var dataSourceReader = GetSubscriptionDataSourceReader(source, dataCacheProvider, config, localDate, baseDataInstance, dataProvider);
+                using var subscriptionEnumerator = SortEnumerator<DateTime>.TryWrapSortEnumerator(source.Sort, dataSourceReader.Read(source));
+                foreach (var datum in subscriptionEnumerator)
+                {
+                    // always skip past all times emitted on the previous invocation of this enumerator
+                    // this allows data at the same time from the same refresh of the source while excluding
+                    // data from different refreshes of the source
+                    if (datum != null && datum.EndTime > localFrontier.Value)
+                    {
+                        yield return datum;
+                    }
+                    else if (!SourceRequiresFastForward(source))
+                    {
+                        // if the 'source' is Rest and there is no new value,
+                        // we *break*, else we will be caught in a tight loop
+                        // because Rest source never ends!
+                        // edit: we 'break' vs 'return null' so that the source is refreshed
+                        // allowing date changes to impact the source value
+                        // note it will respect 'minimumTimeBetweenCalls'
+                        break;
+                    }
+
+                    if (datum != null)
+                    {
+                        newLocalFrontier = Time.Max(datum.EndTime, newLocalFrontier);
+
+                        if (!SourceRequiresFastForward(source))
+                        {
+                            // if the 'source' is Rest we need to update the localFrontier here
+                            // because Rest source never ends!
+                            // Should be advance frontier for all source types here?
+                            localFrontier.Value = newLocalFrontier;
+                        }
+                    }
+                }
+
+                localFrontier.Value = newLocalFrontier;
+            }
+        }
+
+        /// <summary>
+        /// Gets the <see cref="ISubscriptionDataSourceReader"/> for the specified source
+        /// </summary>
+        protected virtual ISubscriptionDataSourceReader GetSubscriptionDataSourceReader(SubscriptionDataSource source,
+            IDataCacheProvider dataCacheProvider,
+            SubscriptionDataConfig config,
+            DateTime date,
+            BaseData baseDataInstance,
+            IDataProvider dataProvider
+            )
+        {
+            return SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, true, baseDataInstance, dataProvider, _objectStore);
+        }
+
+        private bool SourceRequiresFastForward(SubscriptionDataSource source)
+        {
+            return source.TransportMedium == SubscriptionTransportMedium.LocalFile
+                || source.TransportMedium == SubscriptionTransportMedium.RemoteFile;
+        }
+
+        private static TimeSpan GetMinimumTimeBetweenCalls(TimeSpan increment, TimeSpan minimumInterval)
+        {
+            return TimeSpan.FromTicks(Math.Min(increment.Ticks, minimumInterval.Ticks));
+        }
+
+        private static TimeSpan GetMaximumDataAge(TimeSpan increment)
+        {
+            return TimeSpan.FromTicks(Math.Max(increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs

@@ -0,0 +1,166 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ISubscriptionEnumeratorFactory"/> that used the <see cref="SubscriptionDataReader"/>
+    /// </summary>
+    /// <remarks>Only used on backtesting by the <see cref="FileSystemDataFeed"/></remarks>
+    public class SubscriptionDataReaderSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory, IDisposable
+    {
+        private readonly IResultHandler _resultHandler;
+        private readonly IFactorFileProvider _factorFileProvider;
+        private readonly IDataCacheProvider _dataCacheProvider;
+        private readonly ConcurrentDictionary<Symbol, string> _numericalPrecisionLimitedWarnings;
+        private readonly int _numericalPrecisionLimitedWarningsMaxCount = 10;
+        private readonly ConcurrentDictionary<Symbol, string> _startDateLimitedWarnings;
+        private readonly int _startDateLimitedWarningsMaxCount = 10;
+        private readonly IMapFileProvider _mapFileProvider;
+        private readonly bool _enablePriceScaling;
+        private readonly IAlgorithm _algorithm;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class
+        /// </summary>
+        /// <param name="resultHandler">The result handler for the algorithm</param>
+        /// <param name="mapFileProvider">The map file provider</param>
+        /// <param name="factorFileProvider">The factor file provider</param>
+        /// <param name="cacheProvider">Provider used to get data when it is not present on disk</param>
+        /// <param name="algorithm">The algorithm instance to use</param>
+        /// <param name="enablePriceScaling">Applies price factor</param>
+        public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler,
+            IMapFileProvider mapFileProvider,
+            IFactorFileProvider factorFileProvider,
+            IDataCacheProvider cacheProvider,
+            IAlgorithm algorithm,
+            bool enablePriceScaling = true
+            )
+        {
+            _algorithm = algorithm;
+            _resultHandler = resultHandler;
+            _mapFileProvider = mapFileProvider;
+            _factorFileProvider = factorFileProvider;
+            _dataCacheProvider = cacheProvider;
+            _numericalPrecisionLimitedWarnings = new ConcurrentDictionary<Symbol, string>();
+            _startDateLimitedWarnings = new ConcurrentDictionary<Symbol, string>();
+            _enablePriceScaling = enablePriceScaling;
+        }
+
+        /// <summary>
+        /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
+        /// </summary>
+        /// <param name="request">The subscription request to be read</param>
+        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <returns>An enumerator reading the subscription request</returns>
+        public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
+        {
+            var dataReader = new SubscriptionDataReader(request.Configuration,
+                request,
+                _mapFileProvider,
+                _factorFileProvider,
+                _dataCacheProvider,
+                dataProvider,
+                _algorithm.ObjectStore);
+
+            dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
+            dataReader.StartDateLimited += (sender, args) =>
+            {
+                // Queue this warning into our dictionary to report on dispose
+                if (_startDateLimitedWarnings.Count <= _startDateLimitedWarningsMaxCount)
+                {
+                    _startDateLimitedWarnings.TryAdd(args.Symbol, args.Message);
+                }
+            };
+            dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
+            dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
+            dataReader.NumericalPrecisionLimited += (sender, args) =>
+            {
+                // Set a hard limit to keep this warning list from getting unnecessarily large
+                if (_numericalPrecisionLimitedWarnings.Count <= _numericalPrecisionLimitedWarningsMaxCount)
+                {
+                    _numericalPrecisionLimitedWarnings.TryAdd(args.Symbol, args.Message);
+                }
+            };
+
+            IEnumerator<BaseData> enumerator = dataReader;
+            if (LeanData.UseDailyStrictEndTimes(_algorithm.Settings, request, request.Configuration.Symbol, request.Configuration.Increment))
+            {
+                // before corporate events which might yield data and we synchronize both feeds
+                enumerator = new StrictDailyEndTimesEnumerator(enumerator, request.ExchangeHours, request.StartTimeLocal);
+            }
+
+            enumerator = CorporateEventEnumeratorFactory.CreateEnumerators(
+                enumerator,
+                request.Configuration,
+                _factorFileProvider,
+                dataReader,
+                _mapFileProvider,
+                request.StartTimeLocal,
+                request.EndTimeLocal,
+                _enablePriceScaling);
+
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            // Log our numerical precision limited warnings if any
+            if (!_numericalPrecisionLimitedWarnings.IsNullOrEmpty())
+            {
+                var message = "Due to numerical precision issues in the factor file, data for the following" +
+                    $" symbols was adjust to a later starting date: {string.Join(", ", _numericalPrecisionLimitedWarnings.Values.Take(_numericalPrecisionLimitedWarningsMaxCount))}";
+
+                // If we reached our max warnings count suggest that more may have been left out
+                if (_numericalPrecisionLimitedWarnings.Count >= _numericalPrecisionLimitedWarningsMaxCount)
+                {
+                    message += "...";
+                }
+
+                _resultHandler.DebugMessage(message);
+            }
+
+            // Log our start date adjustments because of map files
+            if (!_startDateLimitedWarnings.IsNullOrEmpty())
+            {
+                var message = "The starting dates for the following symbols have been adjusted to match their" +
+                    $" map files first date: {string.Join(", ", _startDateLimitedWarnings.Values.Take(_startDateLimitedWarningsMaxCount))}";
+
+                // If we reached our max warnings count suggest that more may have been left out
+                if (_startDateLimitedWarnings.Count >= _startDateLimitedWarningsMaxCount)
+                {
+                    message += "...";
+                }
+
+                _resultHandler.DebugMessage(message);
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/Factories/TimeTriggeredUniverseSubscriptionEnumeratorFactory.cs

@@ -0,0 +1,61 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ISubscriptionEnumeratorFactory"/> to emit
+    /// ticks based on <see cref="UserDefinedUniverse.GetTriggerTimes"/>, allowing universe
+    /// selection to fire at planned times.
+    /// </summary>
+    public class TimeTriggeredUniverseSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
+    {
+        private readonly ITimeTriggeredUniverse _universe;
+        private readonly MarketHoursDatabase _marketHoursDatabase;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="TimeTriggeredUniverseSubscriptionEnumeratorFactory"/> class
+        /// </summary>
+        /// <param name="universe">The user defined universe</param>
+        /// <param name="marketHoursDatabase">The market hours database</param>
+        public TimeTriggeredUniverseSubscriptionEnumeratorFactory(ITimeTriggeredUniverse universe, MarketHoursDatabase marketHoursDatabase)
+        {
+            _universe = universe;
+            _marketHoursDatabase = marketHoursDatabase;
+        }
+
+        /// <summary>
+        /// Creates an enumerator to read the specified request
+        /// </summary>
+        /// <param name="request">The subscription request to be read</param>
+        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <returns>An enumerator reading the subscription request</returns>
+        public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
+        {
+            return _universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase)
+                .Select(x => new Tick { Time = x, Symbol = request.Configuration.Symbol })
+                .GetEnumerator();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/FastForwardEnumerator.cs

@@ -0,0 +1,132 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using NodaTime;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides the ability to fast forward an enumerator based on the age of the data
+    /// </summary>
+    public class FastForwardEnumerator : IEnumerator<BaseData>
+    {
+        private BaseData _current;
+
+        private readonly DateTimeZone _timeZone;
+        private readonly TimeSpan _maximumDataAge;
+        private readonly ITimeProvider _timeProvider;
+        private readonly IEnumerator<BaseData> _enumerator;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FastForwardEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerator">The source enumerator</param>
+        /// <param name="timeProvider">A time provider used to determine age of data</param>
+        /// <param name="timeZone">The data's time zone</param>
+        /// <param name="maximumDataAge">The maximum age of data allowed</param>
+        public FastForwardEnumerator(IEnumerator<BaseData> enumerator, ITimeProvider timeProvider, DateTimeZone timeZone, TimeSpan maximumDataAge)
+        {
+            _enumerator = enumerator;
+            _timeProvider = timeProvider;
+            _timeZone = timeZone;
+            _maximumDataAge = maximumDataAge;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        public bool MoveNext()
+        {
+            // keep churning until recent data or null
+            while (_enumerator.MoveNext())
+            {
+                // we can't fast forward nulls or bad times
+                if (_enumerator.Current == null || _enumerator.Current.Time == DateTime.MinValue)
+                {
+                    _current = null;
+                    return true;
+                }
+
+                // make sure we never emit past data
+                if (_current != null && _current.EndTime > _enumerator.Current.EndTime)
+                {
+                    continue;
+                }
+
+                // comute the age of the data, if within limits we're done
+                var age = _timeProvider.GetUtcNow().ConvertFromUtc(_timeZone) - _enumerator.Current.EndTime;
+                if (age <= _maximumDataAge)
+                {
+                    _current = _enumerator.Current;
+                    return true;
+                }
+            }
+
+            // we've exhausted the underlying enumerator, iterator completed
+            _current = null;
+            return false;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+    }
+}

Engine/DataFeeds/Enumerators/FillForwardEnumerator.cs

@@ -0,0 +1,638 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Data.Consolidators;
+using QuantConnect.Data.Market;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// The FillForwardEnumerator wraps an existing base data enumerator and inserts extra 'base data' instances
+    /// on a specified fill forward resolution
+    /// </summary>
+    public class FillForwardEnumerator : IEnumerator<BaseData>
+    {
+        private DateTime? _delistedTime;
+        private BaseData _previous;
+        private bool _ended;
+        private bool _isFillingForward;
+        private bool _initialized;
+
+        /// <summary>
+        /// Whether to use strict daily end times
+        /// </summary>
+        protected bool UseStrictEndTime { get; }
+
+        private readonly TimeSpan _dataResolution;
+        private readonly DateTimeZone _dataTimeZone;
+        private readonly bool _isExtendedMarketHours;
+        private readonly DateTime _subscriptionStartTime;
+        private readonly DateTime _subscriptionEndTime;
+        private readonly CalendarInfo _subscriptionEndDataCalendar;
+        private readonly IEnumerator<BaseData> _enumerator;
+        private readonly IReadOnlyRef<TimeSpan> _fillForwardResolution;
+        private readonly bool _strictEndTimeIntraDayFillForward;
+
+        /// <summary>
+        /// The exchange used to determine when to insert fill forward data
+        /// </summary>
+        protected SecurityExchange Exchange { get; init; }
+
+        /// <summary>
+        /// A reference to the last point emitted for the subscription.
+        /// This is used to feed the last point of a previous enumerator in cases like concatenated enumerators.
+        /// For instance, if this enumerator is concatenated to a warm up one, we can use this to feed
+        /// the last point of the warm up enumerator to this one, so that it can use it to fill forward if
+        /// the first actual point of this enumerator is ahead of the subscription start time or the first market open after it.
+        /// </summary>
+        private LastPointTracker _lastPointTracker;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FillForwardEnumerator"/> class that accepts
+        /// a reference to the fill forward resolution, useful if the fill forward resolution is dynamic
+        /// and changing as the enumeration progresses
+        /// </summary>
+        /// <param name="enumerator">The source enumerator to be filled forward</param>
+        /// <param name="exchange">The exchange used to determine when to insert fill forward data</param>
+        /// <param name="fillForwardResolution">The resolution we'd like to receive data on</param>
+        /// <param name="isExtendedMarketHours">True to use the exchange's extended market hours, false to use the regular market hours</param>
+        /// <param name="subscriptionStartTime">The start time of the subscription</param>
+        /// <param name="subscriptionEndTime">The end time of the subscription, once passing this date the enumerator will stop</param>
+        /// <param name="dataResolution">The source enumerator's data resolution</param>
+        /// <param name="dataTimeZone">The time zone of the underlying source data. This is used for rounding calculations and
+        /// is NOT the time zone on the BaseData instances (unless of course data time zone equals the exchange time zone)</param>
+        /// <param name="dailyStrictEndTimeEnabled">True if daily strict end times are enabled</param>
+        /// <param name="dataType">The configuration data type this enumerator is for</param>
+        /// <param name="lastPointTracker">A reference to the last point emitted before this enumerator is first enumerated</param>
+        public FillForwardEnumerator(IEnumerator<BaseData> enumerator,
+            SecurityExchange exchange,
+            IReadOnlyRef<TimeSpan> fillForwardResolution,
+            bool isExtendedMarketHours,
+            DateTime subscriptionStartTime,
+            DateTime subscriptionEndTime,
+            TimeSpan dataResolution,
+            DateTimeZone dataTimeZone,
+            bool dailyStrictEndTimeEnabled,
+            Type dataType = null,
+            LastPointTracker lastPointTracker = null
+            )
+        {
+            _subscriptionStartTime = subscriptionStartTime;
+            _subscriptionEndTime = subscriptionEndTime;
+            Exchange = exchange;
+            _enumerator = enumerator;
+            _dataResolution = dataResolution;
+            _dataTimeZone = dataTimeZone;
+            _fillForwardResolution = fillForwardResolution;
+            _isExtendedMarketHours = isExtendedMarketHours;
+            _lastPointTracker = lastPointTracker;
+            UseStrictEndTime = dailyStrictEndTimeEnabled;
+            // OI data is fill-forwarded to the market close time when strict end times is enabled.
+            // Open interest data can arrive at any time and this would allow to synchronize it with trades and quotes when daily
+            // strict end times is enabled
+            _strictEndTimeIntraDayFillForward = dailyStrictEndTimeEnabled && dataType != null && dataType == typeof(OpenInterest);
+
+            // '_dataResolution' and '_subscriptionEndTime' are readonly they won't change, so lets calculate this once here since it's expensive.
+            // if UseStrictEndTime and also _strictEndTimeIntraDayFillForward, this is a subscription with data that is not adjusted
+            // for the strict end time (like open interest) but require fill forward to synchronize with other data.
+            // Use the non strict end time calendar for the last day of data so that all data for that date is emitted.
+            if (UseStrictEndTime && !_strictEndTimeIntraDayFillForward)
+            {
+                var lastDayCalendar = GetDailyCalendar(_subscriptionEndTime);
+                while (lastDayCalendar.End > _subscriptionEndTime)
+                {
+                    lastDayCalendar = GetDailyCalendar(lastDayCalendar.Start.AddDays(-1));
+                }
+                _subscriptionEndDataCalendar = lastDayCalendar;
+            }
+            else
+            {
+                _subscriptionEndDataCalendar = new (RoundDown(_subscriptionEndTime, _dataResolution), _dataResolution);
+            }
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current => Current;
+
+        private void Initialize()
+        {
+            if (_initialized)
+            {
+                return;
+            }
+
+            if (_lastPointTracker?.LastDataPoint != null)
+            {
+                // adjust the previous data point to the subscription start time to
+                // avoid emitting fill forward data before that
+                _previous = _lastPointTracker.LastDataPoint.Clone();
+                _previous.Time = _subscriptionStartTime - _dataResolution;
+            }
+
+            _initialized = true;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            Initialize();
+
+            if (_delistedTime.HasValue)
+            {
+                // don't fill forward after data after the delisted date
+                if (_previous == null || _previous.EndTime >= _delistedTime.Value)
+                {
+                    return false;
+                }
+            }
+
+            if (Current != null && Current.DataType != MarketDataType.Auxiliary)
+            {
+                // only set the _previous if the last item we emitted was NOT auxilliary data,
+                // since _previous is used for fill forward behavior
+                _previous = Current;
+            }
+
+            BaseData fillForward;
+
+            if (!_isFillingForward)
+            {
+                // if we're filling forward we don't need to move next since we haven't emitted _enumerator.Current yet
+                if (!_enumerator.MoveNext())
+                {
+                    _ended = true;
+                    if (_delistedTime.HasValue)
+                    {
+                        // don't fill forward delisted data
+                        return false;
+                    }
+
+                    // check to see if we ran out of data before the end of the subscription
+                    if (_previous == null || _previous.EndTime >= _subscriptionEndTime)
+                    {
+                        // we passed the end of subscription, we're finished
+                        return false;
+                    }
+
+                    // we can fill forward the rest of this subscription if required
+                    var endOfSubscription = (Current ?? _previous).Clone(true);
+                    endOfSubscription.Time = _subscriptionEndDataCalendar.Start;
+                    endOfSubscription.EndTime = endOfSubscription.Time + _subscriptionEndDataCalendar.Period;
+                    if (RequiresFillForwardData(_fillForwardResolution.Value, _previous, endOfSubscription, out fillForward))
+                    {
+                        // don't mark as filling forward so we come back into this block, subscription is done
+                        //_isFillingForward = true;
+                        Current = fillForward;
+                        return true;
+                    }
+
+                    // don't emit the last bar if the market isn't considered open!
+                    if (!Exchange.IsOpenDuringBar(endOfSubscription.Time, endOfSubscription.EndTime, _isExtendedMarketHours))
+                    {
+                        return false;
+                    }
+
+                    if (Current != null && Current.EndTime == endOfSubscription.EndTime
+                        // TODO this changes stats, why would the FF enumerator emit a data point beyoned the end time he was requested
+                        //|| endOfSubscription.EndTime > _subscriptionEndTime
+                        )
+                    {
+                        return false;
+                    }
+                    Current = endOfSubscription;
+                    return true;
+                }
+            }
+            // If we are filling forward and the underlying is null, let's MoveNext() as long as it didn't end.
+            // This only applies for live trading, so that the LiveFillForwardEnumerator does not stall whenever
+            // we generate a fill-forward bar. The underlying enumerator is advanced so that we don't get stuck
+            // in a cycle of generating infinite fill-forward bars.
+            else if (_enumerator.Current == null && !_ended)
+            {
+                _ended = _enumerator.MoveNext();
+            }
+
+            var underlyingCurrent = _enumerator.Current;
+            if (underlyingCurrent != null && underlyingCurrent.DataType == MarketDataType.Auxiliary)
+            {
+                var delisting = underlyingCurrent as Delisting;
+                if (delisting?.Type == DelistingType.Delisted)
+                {
+                    _delistedTime = delisting.EndTime;
+                }
+            }
+
+            if (_previous == null)
+            {
+                // first data point we dutifully emit without modification
+                Current = underlyingCurrent;
+                return true;
+            }
+
+            if (RequiresFillForwardData(_fillForwardResolution.Value, _previous, underlyingCurrent, out fillForward))
+            {
+                if (_previous.EndTime >= _subscriptionEndTime)
+                {
+                    // we passed the end of subscription, we're finished
+                    return false;
+                }
+                // we require fill forward data because the _enumerator.Current is too far in future
+                _isFillingForward = true;
+                Current = fillForward;
+                return true;
+            }
+
+            _isFillingForward = false;
+            Current = underlyingCurrent;
+            return true;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Determines whether or not fill forward is required, and if true, will produce the new fill forward data
+        /// </summary>
+        /// <param name="fillForwardResolution"></param>
+        /// <param name="previous">The last piece of data emitted by this enumerator</param>
+        /// <param name="next">The next piece of data on the source enumerator</param>
+        /// <param name="fillForward">When this function returns true, this will have a non-null value, null when the function returns false</param>
+        /// <returns>True when a new fill forward piece of data was produced and should be emitted by this enumerator</returns>
+        protected virtual bool RequiresFillForwardData(TimeSpan fillForwardResolution, BaseData previous, BaseData next, out BaseData fillForward)
+        {
+            // in live trading next can be null, in which case we create a potential FF bar and the live FF enumerator will decide what to do
+            var nextCalculatedEndTimeUtc = DateTime.MaxValue;
+            if (next != null)
+            {
+                // convert times to UTC for accurate comparisons and differences across DST changes
+                var previousTimeUtc = previous.Time.ConvertToUtc(Exchange.TimeZone);
+                var nextTimeUtc = next.Time.ConvertToUtc(Exchange.TimeZone);
+                var nextEndTimeUtc = next.EndTime.ConvertToUtc(Exchange.TimeZone);
+
+                if (nextEndTimeUtc < previousTimeUtc)
+                {
+                    if (_lastPointTracker == null || next.EndTime > _subscriptionStartTime)
+                    {
+                        // in some cases we might emit auxiliary data even before our actual start time, which can happen in some cases during warmup
+                        // where previous was initialized through the last point tracker, this point will be filtered out
+                        // but in any other case though let's log it, shouldn't happen
+                        Log.Error("FillForwardEnumerator received data out of order. Symbol: " + previous.Symbol.ID);
+                    }
+                    fillForward = null;
+                    return false;
+                }
+
+                // check to see if the gap between previous and next warrants fill forward behavior
+                if (!ShouldFillForward(previousTimeUtc, nextTimeUtc, fillForwardResolution))
+                {
+                    fillForward = null;
+                    return false;
+                }
+
+                // Double check!
+                // This might be the last FF bar before the next data point, and it might not be to be
+                // emitted because it will overlap with the next point.
+                // If the previous point was fill forwarded, its time might have been rounded down,
+                // we need to compare apples to apples.
+                // (e.g. daily bars with times != midnight and without strict end times)
+                var nextPeriod = nextEndTimeUtc - nextTimeUtc;
+                if (previous.IsFillForward && (!UseStrictEndTime || nextPeriod <= Time.OneHour))
+                {
+                    var roundedNextTimeUtc = RoundDown(next.Time, nextPeriod).ConvertToUtc(Exchange.TimeZone);
+                    if (!ShouldFillForward(previousTimeUtc, roundedNextTimeUtc, fillForwardResolution))
+                    {
+                        fillForward = null;
+                        return false;
+                    }
+                }
+
+                var period = _dataResolution;
+                if (UseStrictEndTime)
+                {
+                    // the period is not the data resolution (1 day) and can actually change dynamically, for example early close/late open
+                    period = next.EndTime - next.Time;
+                }
+                else if (next.Time == next.EndTime)
+                {
+                    // we merge corporate event data points (mapping, delisting, splits, dividend) which do not have
+                    // a period or resolution
+                    period = TimeSpan.Zero;
+                }
+                nextCalculatedEndTimeUtc = nextTimeUtc + period;
+            }
+
+            // every bar emitted MUST be of the data resolution.
+
+            // compute end times of the four potential fill forward scenarios
+            // 1. the next fill forward bar. 09:00-10:00 followed by 10:00-11:00 where 01:00 is the fill forward resolution
+            // 2. the next data resolution bar, same as above but with the data resolution instead
+            // 3. the next fill forward bar following the next market open, 15:00-16:00 followed by 09:00-10:00 the following open market day
+            // 4. the next data resolution bar following the next market open, same as above but with the data resolution instead
+
+            // the precedence for validation is based on the order of the end times, obviously if a potential match
+            // is before a later match, the earliest match should win.
+
+            foreach (var item in GetSortedReferenceDateIntervals(previous, fillForwardResolution, _dataResolution))
+            {
+                // issue GH 4925 , more description https://github.com/QuantConnect/Lean/pull/4941
+                // To build Time/EndTime we always use '+'/'-' dataResolution
+                // DataTime TZ = UTC -5; Exchange TZ = America/New York (-5/-4)
+                // Standard TimeZone    00:00:00 + 1 day = 1.00:00:00
+                // Daylight Time        01:00:00 + 1 day = 1.01:00:00
+
+                // daylight saving time starts/end at 2 a.m. on Sunday
+                // Having this information we find that the specific bar of Sunday
+                // Starts in one TZ (Standard TZ), but finishes in another (Daylight TZ) (consider winter => summer)
+                // During simple arithmetic operations like +/- we shift the time, but not the time zone
+                // which is sensitive for specific dates (daylight movement) if we are  in Exchange TimeZone, for example
+                // We have 00:00:00 + 1 day = 1.00:00:00, so both are in Standard TZ, but we expect endTime in Daylight, i.e. 1.01:00:00
+
+                // futher down double Convert (Exchange TZ => data TZ => Exchange TZ)
+                // allows us to calculate Time using it's own TZ (aka reapply)
+                // and don't rely on TZ of bar start/end time
+                // i.e. 00:00:00 + 1 day = 1.01:00:00, both start and end are in their own TZ
+                // it's interesting that NodaTime  consider next
+                // if time great or equal than 01:00 AM it's considered as "moved" (Standard, not Daylight)
+                // when time less than 01:00 AM it's considered as previous TZ (Standard, not Daylight)
+                // it's easy to fix this behavior by substract 1 tick  before first convert, and then return it back.
+                // so we work with 0:59:59.. AM instead.
+                // but now follow native behavior
+
+                // all above means, that all Time values, calculated using simple +/- operations
+                // sticks to original Time Zone, swallowing its own TZ and movement i.e.
+                // EndTime = Time + resolution, both Time and EndTime in the TZ of Time (Standard/Daylight)
+                // Time = EndTime - resolution, both Time and EndTime in the TZ of EndTime (Standard/Daylight)
+
+                // next.EndTime sticks to Time TZ,
+                // potentialBarEndTime should be calculated in the same way as bar.EndTime, i.e. Time + resolution
+                // round down doesn't make sense for daily data using strict times
+                var startTime = (UseStrictEndTime && item.Period > Time.OneHour) ? item.Start : RoundDown(item.Start, item.Period);
+                var potentialBarEndTime = startTime.ConvertToUtc(Exchange.TimeZone) + item.Period;
+
+                // to avoid duality it's necessary to compare potentialBarEndTime with
+                // next.EndTime calculated as Time + resolution,
+                // and both should be based on the same TZ (for example UTC)
+                if (potentialBarEndTime < nextCalculatedEndTimeUtc
+                    // let's fill forward based on previous (which isn't auxiliary) if next is auxiliary and they share the end time
+                    // we do allow emitting both an auxiliary data point and a Filled Forwared data for the same end time
+                    || next != null && next.DataType == MarketDataType.Auxiliary && potentialBarEndTime == nextCalculatedEndTimeUtc)
+                {
+                    // to check open hours we need to convert potential
+                    // bar EndTime into exchange time zone
+                    var potentialBarEndTimeInExchangeTZ =
+                        potentialBarEndTime.ConvertFromUtc(Exchange.TimeZone);
+                    var nextFillForwardBarStartTime = potentialBarEndTimeInExchangeTZ - item.Period;
+
+                    if (Exchange.IsOpenDuringBar(nextFillForwardBarStartTime, potentialBarEndTimeInExchangeTZ, _isExtendedMarketHours))
+                    {
+                        fillForward = previous.Clone(true);
+
+                        // bar are ALWAYS of the data resolution
+                        var expectedPeriod = _dataResolution;
+                        if (UseStrictEndTime)
+                        {
+                            // TODO: what about extended market hours
+                            // NOTE: Not using Exchange.Hours.RegularMarketDuration so we can handle things like early closes.
+
+                            // The earliest start time would be endTime - regularMarketDuration,
+                            // we use that as the potential time to get the exchange hours.
+                            // We don't use directly nextFillForwardBarStartTime because there might be cases where there are
+                            // adjacent extended and regular market hours segments that might cause the calendar start to be
+                            // in the previous date, and if it's an extended hours-only date like a Sunday for futures,
+                            // the market duration would be zero.
+                            var marketHoursDateTime = potentialBarEndTimeInExchangeTZ - Exchange.Hours.RegularMarketDuration;
+                            // That potential start is even before the calendar start, so we use the calendar start
+                            if (marketHoursDateTime < item.Start)
+                            {
+                                marketHoursDateTime = item.Start;
+                            }
+                            var marketHours = Exchange.Hours.GetMarketHours(marketHoursDateTime);
+                            expectedPeriod = marketHours.MarketDuration;
+                        }
+                        fillForward.Time = (potentialBarEndTime - expectedPeriod).ConvertFromUtc(Exchange.TimeZone);
+                        fillForward.EndTime = potentialBarEndTimeInExchangeTZ;
+                        return true;
+                    }
+                }
+                else
+                {
+                    break;
+                }
+            }
+
+            // the next is before the next fill forward time, so do nothing
+            fillForward = null;
+            return false;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private bool ShouldFillForward(DateTime previousTimeUtc, DateTime nextTimeUtc, TimeSpan fillForwardResolution)
+        {
+            var nextPreviousTimeUtcDelta = nextTimeUtc - previousTimeUtc;
+            return nextPreviousTimeUtcDelta > fillForwardResolution ||
+                nextPreviousTimeUtcDelta > _dataResolution ||
+                // even if there is no gap between the two data points, we still fill forward to ensure a FF bar is emitted at strict end time
+                _strictEndTimeIntraDayFillForward;
+        }
+
+        private IEnumerable<CalendarInfo> GetSortedReferenceDateIntervals(BaseData previous, TimeSpan fillForwardResolution, TimeSpan dataResolution)
+        {
+            if (fillForwardResolution < dataResolution)
+            {
+                return GetReferenceDateIntervals(previous.EndTime, fillForwardResolution, dataResolution);
+            }
+
+            if (fillForwardResolution > dataResolution)
+            {
+                return GetReferenceDateIntervals(previous.EndTime, dataResolution, fillForwardResolution);
+            }
+
+            return GetReferenceDateIntervals(previous.EndTime, fillForwardResolution);
+        }
+
+        /// <summary>
+        /// Get potential next fill forward bars.
+        /// </summary>
+        /// <remarks>Special case where fill forward resolution and data resolution are equal</remarks>
+        private IEnumerable<CalendarInfo> GetReferenceDateIntervals(DateTime previousEndTime, TimeSpan resolution)
+        {
+            // say daily bar goes from 9:30 to 16:00, if resolution is 1 day, IsOpenDuringBar can return true but it's not what we want
+            if (!UseStrictEndTime && Exchange.IsOpenDuringBar(previousEndTime, previousEndTime + resolution, _isExtendedMarketHours))
+            {
+                // if next in market us it
+                yield return new (previousEndTime, resolution);
+            }
+
+            if (UseStrictEndTime)
+            {
+                // If we're using strict end times for open interest data, for instance, the actual data comes at any time
+                // but we want to emit a ff point at market close. If extended market hours are enabled, and previousEndTime
+                // is Thursday after last segment open time, the daily calendar will be for Monday, because a next market open
+                // won't be found for Friday. So we use the Date of the previousEndTime to get calendar starting that day (Thursday)
+                // and ending the next one (Friday).
+                if (_strictEndTimeIntraDayFillForward)
+                {
+                    var firtMarketOpen = Exchange.Hours.GetNextMarketOpen(previousEndTime.Date, _isExtendedMarketHours);
+                    var firstCalendar = LeanData.GetDailyCalendar(firtMarketOpen, Exchange.Hours, false);
+
+                    if (firstCalendar.End > previousEndTime)
+                    {
+                        yield return firstCalendar;
+                    }
+                }
+
+                // now we can try the bar after next market open
+                var marketOpen = Exchange.Hours.GetNextMarketOpen(previousEndTime, false);
+                yield return GetDailyCalendar(marketOpen);
+            }
+            else
+            {
+                // now we can try the bar after next market open
+                var marketOpen = Exchange.Hours.GetNextMarketOpen(previousEndTime, _isExtendedMarketHours);
+                yield return new(marketOpen, resolution);
+            }
+        }
+
+        /// <summary>
+        /// Get potential next fill forward bars.
+        /// </summary>
+        private IEnumerable<CalendarInfo> GetReferenceDateIntervals(DateTime previousEndTime, TimeSpan smallerResolution, TimeSpan largerResolution)
+        {
+            List<CalendarInfo> result = null;
+            if (Exchange.IsOpenDuringBar(previousEndTime, previousEndTime + smallerResolution, _isExtendedMarketHours))
+            {
+                if (UseStrictEndTime)
+                {
+                    // case A
+                    result = new()
+                    {
+                        new(previousEndTime, smallerResolution)
+                    };
+                }
+                else
+                {
+                    // at the end of this method we perform an OrderBy which does not apply for this case because the consumer of this method
+                    // will perform a round down that will end up using an unexpected FF bar. This behavior is covered by tests
+                    yield return new (previousEndTime, smallerResolution);
+                }
+            }
+            result ??= new List<CalendarInfo>(4);
+
+            // we need to round down because previous end time could be of the smaller resolution, in data TZ!
+            if (UseStrictEndTime)
+            {
+                // case B: say smaller resolution (FF res) is 1 hour, larget resolution (daily data resolution) is 1 day
+                // For example for SPX we need to emit the daily FF bar from 8:30->15:15, even before the 'A' case above which would be 15->16 bar
+                var dailyCalendar = GetDailyCalendar(previousEndTime);
+                if (previousEndTime < (dailyCalendar.Start + dailyCalendar.Period))
+                {
+                    result.Add(new(dailyCalendar.Start, dailyCalendar.Period));
+                }
+            }
+            else
+            {
+                var start = RoundDown(previousEndTime, largerResolution);
+                if (Exchange.IsOpenDuringBar(start, start + largerResolution, _isExtendedMarketHours))
+                {
+                    result.Add(new(start, largerResolution));
+                }
+            }
+
+            // this is typically daily data being filled forward on a higher resolution
+            // since the previous bar was not in market hours then we can just fast forward
+            // to the next market open
+            var marketOpen = Exchange.Hours.GetNextMarketOpen(previousEndTime, _isExtendedMarketHours);
+            result.Add(new (marketOpen, smallerResolution));
+            if (UseStrictEndTime)
+            {
+                result.Add(GetDailyCalendar(Exchange.Hours.GetNextMarketOpen(previousEndTime, false)));
+            }
+
+            // we need to order them because they might not be in an incremental order and consumer expects them to be
+            foreach (var referenceDateInterval in result.OrderBy(interval => interval.Start + interval.Period))
+            {
+                yield return referenceDateInterval;
+            }
+        }
+
+        /// <summary>
+        /// We need to round down in data timezone.
+        /// For example GH issue 4392: Forex daily data, exchange tz time is 8PM, but time in data tz is 12AM
+        /// so rounding down on exchange tz will crop it, while rounding on data tz will return the same data point time.
+        /// Why are we even doing this? being able to determine the next valid data point for a resolution from a data point that might be in another resolution
+        /// </summary>
+        private DateTime RoundDown(DateTime value, TimeSpan interval)
+        {
+            return value.RoundDownInTimeZone(interval, Exchange.TimeZone, _dataTimeZone);
+        }
+
+        private CalendarInfo GetDailyCalendar(DateTime localReferenceTime)
+        {
+            // daily data does not have extended market hours, even if requested
+            // and it's times are always market hours if using strict end times see 'SetStrictEndTimes'
+            return LeanData.GetDailyCalendar(localReferenceTime, Exchange.Hours, extendedMarketHours: false);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/FilterEnumerator.cs

@@ -0,0 +1,96 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that allow applying a filtering function
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public class FilterEnumerator<T> : IEnumerator<T>
+    {
+        private readonly IEnumerator<T> _enumerator;
+        private readonly Func<T, bool> _filter;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to filter on</param>
+        /// <param name="filter">The filter to apply</param>
+        public FilterEnumerator(IEnumerator<T> enumerator, Func<T, bool> filter)
+        {
+            _enumerator = enumerator;
+            _filter = filter;
+        }
+
+        #region Implementation of IDisposable
+
+        /// <summary>
+        /// Disposes the FilterEnumerator
+        /// </summary>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        #endregion
+
+        #region Implementation of IEnumerator
+
+        /// <summary>
+        /// Moves the FilterEnumerator to the next item
+        /// </summary>
+        public bool MoveNext()
+        {
+            // run the enumerator until it passes the specified filter
+            while (_enumerator.MoveNext())
+            {
+                if (_filter(_enumerator.Current))
+                {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Resets the FilterEnumerator
+        /// </summary>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the current item in the FilterEnumerator
+        /// </summary>
+        public T Current
+        {
+            get { return _enumerator.Current; }
+        }
+
+        object IEnumerator.Current
+        {
+            get { return _enumerator.Current; }
+        }
+
+        #endregion
+    }
+}

Engine/DataFeeds/Enumerators/FrontierAwareEnumerator.cs

@@ -0,0 +1,149 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="IEnumerator{BaseData}"/> that will not emit
+    /// data ahead of the frontier as specified by an instance of <see cref="ITimeProvider"/>.
+    /// An instance of <see cref="TimeZoneOffsetProvider"/> is used to convert between UTC
+    /// and the data's native time zone
+    /// </summary>
+    public class FrontierAwareEnumerator : IEnumerator<BaseData>
+    {
+        private BaseData _current;
+        private bool _needsMoveNext = true;
+
+        private readonly ITimeProvider _timeProvider;
+        private readonly IEnumerator<BaseData> _enumerator;
+        private readonly TimeZoneOffsetProvider _offsetProvider;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FrontierAwareEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to make frontier aware</param>
+        /// <param name="timeProvider">The time provider used for resolving the current frontier time</param>
+        /// <param name="offsetProvider">An offset provider used for converting the frontier UTC time into the data's native time zone</param>
+        public FrontierAwareEnumerator(IEnumerator<BaseData> enumerator, ITimeProvider timeProvider, TimeZoneOffsetProvider offsetProvider)
+        {
+            _enumerator = enumerator;
+            _timeProvider = timeProvider;
+            _offsetProvider = offsetProvider;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            var underlyingCurrent = _enumerator.Current;
+            var frontier = _timeProvider.GetUtcNow();
+            var localFrontier = new DateTime(frontier.Ticks + _offsetProvider.GetOffsetTicks(frontier));
+
+            // if we moved next, but didn't emit, check to see if it's time to emit yet
+            if (!_needsMoveNext && underlyingCurrent != null)
+            {
+                if (underlyingCurrent.EndTime <= localFrontier)
+                {
+                    // we can now emit the underlyingCurrent as part of this time slice
+                    _current = underlyingCurrent;
+                    _needsMoveNext = true;
+                }
+                else
+                {
+                    // it's still not time to emit the underlyingCurrent, keep waiting for time to advance
+                    _current = null;
+                    _needsMoveNext = false;
+                }
+                return true;
+            }
+
+            // we've exhausted the underlying enumerator, iteration completed
+            if (_needsMoveNext && !_enumerator.MoveNext())
+            {
+                _needsMoveNext = true;
+                _current = null;
+                return false;
+            }
+
+            underlyingCurrent = _enumerator.Current;
+
+            if (underlyingCurrent != null && underlyingCurrent.EndTime <= localFrontier)
+            {
+                _needsMoveNext = true;
+                _current = underlyingCurrent;
+            }
+            else
+            {
+                _current = null;
+                _needsMoveNext = underlyingCurrent == null;
+            }
+
+            // technically we still need to return true since the iteration is not completed,
+            // however, Current may be null follow a true result here
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/ITradableDateEventProvider.cs

@@ -0,0 +1,49 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Interface for event providers for new tradable dates
+    /// </summary>
+    public interface ITradableDateEventProvider
+    {
+        /// <summary>
+        /// Called each time there is a new tradable day
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New corporate event if any</returns>
+        IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs);
+
+        /// <summary>
+        /// Initializes the event provider instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        void Initialize(SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime);
+    }
+}

Engine/DataFeeds/Enumerators/ITradableDatesNotifier.cs

@@ -0,0 +1,32 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Interface which will provide an event handler
+    /// who will be fired with each new tradable day
+    /// </summary>
+    public interface ITradableDatesNotifier
+    {
+        /// <summary>
+        /// Event fired when there is a new tradable date
+        /// </summary>
+        event EventHandler<NewTradableDateEventArgs> NewTradableDate;
+    }
+}

Engine/DataFeeds/Enumerators/LastPointTracker.cs

@@ -0,0 +1,43 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Tracks the last data point received by an enumerator.
+    /// </summary>
+    public class LastPointTracker
+    {
+        private BaseData _lastPoint;
+
+        /// <summary>
+        /// Tracks the last data point received by the enumerator.
+        /// </summary>
+        public BaseData LastDataPoint
+        {
+            get => _lastPoint;
+            set
+            {
+                if (value != null && !value.IsFillForward && value.DataType != MarketDataType.Auxiliary)
+                {
+                    _lastPoint = value;
+                }
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveAuxiliaryDataEnumerator.cs

@@ -0,0 +1,114 @@
+/*
+* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Auxiliary data enumerator that will trigger new tradable dates event accordingly
+    /// </summary>
+    public class LiveAuxiliaryDataEnumerator : AuxiliaryDataEnumerator
+    {
+        private DateTime _lastTime;
+        private ITimeProvider _timeProvider;
+        private SecurityCache _securityCache;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="tradableDateEventProviders">The tradable dates event providers</param>
+        /// <param name="startTime">Start date for the data request</param>
+        /// <param name="timeProvider">The time provider to use</param>
+        /// <param name="securityCache">The security cache</param>
+        public LiveAuxiliaryDataEnumerator(SubscriptionDataConfig config, IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider, ITradableDateEventProvider[] tradableDateEventProviders,
+            DateTime startTime,
+            ITimeProvider timeProvider,
+            SecurityCache securityCache)
+            // tradableDayNotifier: null -> we are going to trigger the new tradables events for the base implementation
+            : base(config, factorFileProvider, mapFileProvider, tradableDateEventProviders, tradableDayNotifier:null, startTime)
+        {
+            _securityCache = securityCache;
+            _timeProvider = timeProvider;
+
+            // initialize providers right away so mapping happens before we subscribe
+            Initialize();
+        }
+
+        /// <summary>
+        /// Moves the LiveAuxiliaryDataEnumerator to the next item
+        /// </summary>
+        public override bool MoveNext()
+        {
+            var currentDate = _timeProvider.GetUtcNow().ConvertFromUtc(Config.ExchangeTimeZone).Add(-Time.LiveAuxiliaryDataOffset).Date;
+            if (currentDate != _lastTime)
+            {
+                // when the date changes for the security we trigger a new tradable date event
+                var newDayEvent = new NewTradableDateEventArgs(currentDate, _securityCache.GetData(), Config.Symbol, null);
+
+                NewTradableDate(this, newDayEvent);
+                // update last time
+                _lastTime = currentDate;
+            }
+
+            return base.MoveNext();
+        }
+
+        /// <summary>
+        /// Helper method to create a new instance.
+        /// Knows which security types should create one and determines the appropriate delisting event provider to use
+        /// </summary>
+        public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider,
+            SecurityCache securityCache, IMapFileProvider mapFileProvider, IFactorFileProvider fileProvider, DateTime startTime,
+            out IEnumerator<BaseData> enumerator)
+        {
+            enumerator = null;
+            var securityType = dataConfig.SecurityType;
+            if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity)
+            {
+                var providers = new List<ITradableDateEventProvider>
+                {
+                    securityType == SecurityType.Equity
+                        ? new LiveDelistingEventProvider()
+                        : new DelistingEventProvider()
+                };
+
+                if (dataConfig.TickerShouldBeMapped())
+                {
+                    providers.Add(new LiveMappingEventProvider());
+                }
+
+                if (dataConfig.EmitSplitsAndDividends())
+                {
+                    providers.Add(new LiveDividendEventProvider());
+                    providers.Add(new LiveSplitEventProvider());
+                }
+
+                enumerator = new LiveAuxiliaryDataEnumerator(dataConfig, fileProvider, mapFileProvider,
+                    providers.ToArray(), startTime, timeProvider, securityCache);
+            }
+            return enumerator != null;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveAuxiliaryDataSynchronizingEnumerator.cs

@@ -0,0 +1,191 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Represents an enumerator capable of synchronizing live equity data enumerators in time.
+    /// This assumes that all enumerators have data time stamped in the same time zone.
+    /// </summary>
+    public class LiveAuxiliaryDataSynchronizingEnumerator : IEnumerator<BaseData>
+    {
+        private readonly ITimeProvider _timeProvider;
+        private readonly DateTimeZone _exchangeTimeZone;
+        private readonly List<IEnumerator<BaseData>> _auxDataEnumerators;
+        private readonly IEnumerator<BaseData> _tradeBarAggregator;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LiveAuxiliaryDataSynchronizingEnumerator"/> class
+        /// </summary>
+        /// <param name="timeProvider">The source of time used to gauge when this enumerator should emit extra bars when null data is returned from the source enumerator</param>
+        /// <param name="exchangeTimeZone">The time zone the raw data is time stamped in</param>
+        /// <param name="tradeBarAggregator">The trade bar aggregator enumerator</param>
+        /// <param name="auxDataEnumerators">The auxiliary data enumerators</param>
+        public LiveAuxiliaryDataSynchronizingEnumerator(ITimeProvider timeProvider, DateTimeZone exchangeTimeZone, IEnumerator<BaseData> tradeBarAggregator, List<IEnumerator<BaseData>> auxDataEnumerators)
+        {
+            _timeProvider = timeProvider;
+            _exchangeTimeZone = exchangeTimeZone;
+            _auxDataEnumerators = auxDataEnumerators;
+            _tradeBarAggregator = tradeBarAggregator;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns> true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.</returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
+        public bool MoveNext()
+        {
+            // use manual time provider from LiveTradingDataFeed
+            var frontierUtc = _timeProvider.GetUtcNow();
+
+            // check if any enumerator is ready to emit
+            if (DataPointEmitted(frontierUtc))
+                return true;
+
+            // advance enumerators with no current data
+            for (var i = 0; i < _auxDataEnumerators.Count; i++)
+            {
+                if (_auxDataEnumerators[i].Current == null)
+                {
+                    _auxDataEnumerators[i].MoveNext();
+                }
+            }
+            if (_tradeBarAggregator.Current == null) _tradeBarAggregator.MoveNext();
+
+            // check if any enumerator is ready to emit
+            if (DataPointEmitted(frontierUtc))
+                return true;
+
+            Current = null;
+
+            // IEnumerator contract dictates that we return true unless we're actually
+            // finished with the 'collection' and since this is live, we're never finished
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
+        public void Reset()
+        {
+            foreach (var auxDataEnumerator in _auxDataEnumerators)
+            {
+                auxDataEnumerator.Reset();
+            }
+            _tradeBarAggregator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>The element in the collection at the current position of the enumerator.</returns>
+        public BaseData Current { get; private set; }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>The current element in the collection.</returns>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            foreach (var auxDataEnumerator in _auxDataEnumerators)
+            {
+                auxDataEnumerator.DisposeSafely();
+            }
+            _tradeBarAggregator.DisposeSafely();
+        }
+
+        private bool DataPointEmitted(DateTime frontierUtc)
+        {
+            // we get the aux enumerator that has the smallest endTime if any
+            IEnumerator<BaseData> auxDataEnumerator = null;
+            for (var i = 0; i < _auxDataEnumerators.Count; i++)
+            {
+                var currentEnum = _auxDataEnumerators[i];
+                if (currentEnum.Current != null)
+                {
+                    if (auxDataEnumerator == null)
+                    {
+                        auxDataEnumerator = currentEnum;
+                    }
+                    else
+                    {
+                        auxDataEnumerator = auxDataEnumerator.Current.EndTime > currentEnum.Current.EndTime ? currentEnum : auxDataEnumerator;
+                    }
+                }
+            }
+
+            // check if any enumerator is ready to emit
+            if (auxDataEnumerator?.Current != null && _tradeBarAggregator.Current != null)
+            {
+                var auxDataEndTime = auxDataEnumerator.Current.EndTime.ConvertToUtc(_exchangeTimeZone);
+                var tradeBarEndTime = _tradeBarAggregator.Current.EndTime.ConvertToUtc(_exchangeTimeZone);
+                if (auxDataEndTime < tradeBarEndTime)
+                {
+                    if (auxDataEndTime <= frontierUtc)
+                    {
+                        Current = auxDataEnumerator.Current;
+                        auxDataEnumerator.MoveNext();
+                        return true;
+                    }
+                }
+                else
+                {
+                    if (tradeBarEndTime <= frontierUtc)
+                    {
+                        Current = _tradeBarAggregator.Current;
+                        _tradeBarAggregator.MoveNext();
+                        return true;
+                    }
+                }
+            }
+            else if (auxDataEnumerator?.Current != null)
+            {
+                var auxDataEndTime = auxDataEnumerator.Current.EndTime.ConvertToUtc(_exchangeTimeZone);
+                if (auxDataEndTime <= frontierUtc)
+                {
+                    Current = auxDataEnumerator.Current;
+                    auxDataEnumerator.MoveNext();
+                    return true;
+                }
+            }
+            else if (_tradeBarAggregator.Current != null)
+            {
+                var tradeBarEndTime = _tradeBarAggregator.Current.EndTime.ConvertToUtc(_exchangeTimeZone);
+                if (tradeBarEndTime <= frontierUtc)
+                {
+                    Current = _tradeBarAggregator.Current;
+                    _tradeBarAggregator.MoveNext();
+                    return true;
+                }
+            }
+
+            return false;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveDelistingEventProvider.cs

@@ -0,0 +1,54 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Delisting event provider implementation which will source the delisting date based on new map files
+    /// </summary>
+    public class LiveDelistingEventProvider : DelistingEventProvider
+    {
+        /// <summary>
+        /// Check for delistings
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New delisting event if any</returns>
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            var currentInstance = MapFile;
+            // refresh map file instance
+            InitializeMapFile();
+            var newInstance = MapFile;
+
+            if (currentInstance?.LastOrDefault()?.Date != newInstance?.LastOrDefault()?.Date)
+            {
+                // All future and option contracts sharing the same canonical symbol, share the same configuration too. Thus, in
+                // order to reduce logs, we log the configuration using the canonical symbol. See the optional parameter
+                // "overrideMessageFloodProtection" in Log.Trace() method for more information
+                var symbol = Config.Symbol.HasCanonical() ? Config.Symbol.Canonical.Value : Config.Symbol.Value;
+                Log.Trace($"LiveDelistingEventProvider({Config.ToString(symbol)}): new tradable date {eventArgs.Date:yyyyMMdd}. " +
+                    $"MapFile.LastDate Old: {currentInstance?.LastOrDefault()?.Date:yyyyMMdd} New: {newInstance?.LastOrDefault()?.Date:yyyyMMdd}");
+            }
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveDividendEventProvider.cs

@@ -0,0 +1,57 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    /// <remarks>Only special behavior is that it will refresh factor file on each new tradable date event</remarks>
+    public class LiveDividendEventProvider : DividendEventProvider
+    {
+        /// <summary>
+        /// Check for dividends and returns them
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New Dividend event if any</returns>
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            var currentInstance = FactorFile;
+            // refresh factor file instance
+            InitializeFactorFile();
+            var newInstance = FactorFile;
+
+            if (currentInstance?.Count() != newInstance?.Count())
+            {
+                // All future and option contracts sharing the same canonical symbol, share the same configuration too. Thus, in
+                // order to reduce logs, we log the configuration using the canonical symbol. See the optional parameter
+                // "overrideMessageFloodProtection" in Log.Trace() method for more information
+                var symbol = Config.Symbol.HasCanonical() ? Config.Symbol.Canonical.Value : Config.Symbol.Value;
+                Log.Trace($"LiveDividendEventProvider({Config.ToString(symbol)}): new tradable date {eventArgs.Date:yyyyMMdd}. " +
+                    $"New FactorFile: {!ReferenceEquals(currentInstance, newInstance)}. " +
+                    $"FactorFile.Count Old: {currentInstance?.Count()} New: {newInstance?.Count()}");
+            }
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveFillForwardEnumerator.cs

@@ -0,0 +1,157 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// An implementation of the <see cref="FillForwardEnumerator"/> that uses an <see cref="ITimeProvider"/>
+    /// to determine if a fill forward bar needs to be emitted
+    /// </summary>
+    public class LiveFillForwardEnumerator : FillForwardEnumerator
+    {
+        private readonly TimeSpan _dataResolution;
+        private readonly TimeSpan _underlyingTimeout;
+        private readonly ITimeProvider _timeProvider;
+
+        private TimeSpan _marketCloseTimeSpan;
+        private TimeSpan _marketOpenTimeSpan;
+        private DateTime _lastDate;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LiveFillForwardEnumerator"/> class that accepts
+        /// a reference to the fill forward resolution, useful if the fill forward resolution is dynamic
+        /// and changing as the enumeration progresses
+        /// </summary>
+        /// <param name="timeProvider">The source of time used to gauage when this enumerator should emit extra bars when
+        /// null data is returned from the source enumerator</param>
+        /// <param name="enumerator">The source enumerator to be filled forward</param>
+        /// <param name="exchange">The exchange used to determine when to insert fill forward data</param>
+        /// <param name="fillForwardResolution">The resolution we'd like to receive data on</param>
+        /// <param name="isExtendedMarketHours">True to use the exchange's extended market hours, false to use the regular market hours</param>
+        /// <param name="subscriptionStartTime">The start time of the subscription</param>
+        /// <param name="subscriptionEndTime">The end time of the subscription, once passing this date the enumerator will stop</param>
+        /// <param name="dataResolution">The source enumerator's data resolution</param>
+        /// <param name="dataTimeZone">Time zone of the underlying source data</param>
+        /// <param name="dailyStrictEndTimeEnabled">True if daily strict end times are enabled</param>
+        /// <param name="dataType">The configuration data type this enumerator is for</param>
+        /// <param name="lastPointTracker">A reference to the last point emitted before this enumerator is first enumerated</param>
+        public LiveFillForwardEnumerator(ITimeProvider timeProvider, IEnumerator<BaseData> enumerator, SecurityExchange exchange, IReadOnlyRef<TimeSpan> fillForwardResolution,
+            bool isExtendedMarketHours, DateTime subscriptionStartTime, DateTime subscriptionEndTime, Resolution dataResolution, DateTimeZone dataTimeZone, bool dailyStrictEndTimeEnabled,
+            Type dataType = null, LastPointTracker lastPointTracker = null)
+            : base(enumerator, exchange, fillForwardResolution, isExtendedMarketHours, subscriptionStartTime, subscriptionEndTime, dataResolution.ToTimeSpan(), dataTimeZone,
+                  dailyStrictEndTimeEnabled, dataType, lastPointTracker)
+        {
+            _timeProvider = timeProvider;
+            _dataResolution = dataResolution.ToTimeSpan();
+            _underlyingTimeout = GetMaximumDataTimeout(dataResolution);
+        }
+
+        /// <summary>
+        /// Determines whether or not fill forward is required, and if true, will produce the new fill forward data
+        /// </summary>
+        /// <param name="fillForwardResolution"></param>
+        /// <param name="previous">The last piece of data emitted by this enumerator</param>
+        /// <param name="next">The next piece of data on the source enumerator, this may be null</param>
+        /// <param name="fillForward">When this function returns true, this will have a non-null value, null when the function returns false</param>
+        /// <returns>True when a new fill forward piece of data was produced and should be emitted by this enumerator</returns>
+        protected override bool RequiresFillForwardData(TimeSpan fillForwardResolution, BaseData previous, BaseData next, out BaseData fillForward)
+        {
+            if (base.RequiresFillForwardData(fillForwardResolution, previous, next, out fillForward))
+            {
+                var underlyingTimeout = TimeSpan.Zero;
+                if (fillForwardResolution >= _dataResolution && ShouldWaitForData(fillForward))
+                {
+                    // we enforce the underlying FF timeout when the FF resolution matches it or is bigger, not the other way round, for example:
+                    // this is a daily enumerator and FF resolution is second, we are expected to emit a bar every second, we can't wait until the timeout each time
+                    underlyingTimeout = _underlyingTimeout;
+                }
+
+                var nextEndTimeUtc = (fillForward.EndTime + underlyingTimeout).ConvertToUtc(Exchange.TimeZone);
+                if (next != null || nextEndTimeUtc <= _timeProvider.GetUtcNow())
+                {
+                    // we FF if next is here but in the future or next has not come yet and we've wait enough time
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Helper method to determine if we should wait for data before emitting a fill forward bar.
+        /// We only wait for data if the fill forward bar is either in the market open or close time.
+        /// </summary>
+        private bool ShouldWaitForData(BaseData fillForward)
+        {
+            if (fillForward.Symbol.SecurityType != SecurityType.Equity || Exchange.Hours.IsMarketAlwaysOpen)
+            {
+                return false;
+            }
+
+            // Update market open and close daily
+            if (_lastDate != fillForward.EndTime.Date ||
+                // Update market open and close for days with multiple sessions, e.g. early close and then late open
+                fillForward.Time.TimeOfDay > _marketCloseTimeSpan)
+            {
+                _lastDate = fillForward.EndTime.Date;
+                var marketOpen = Exchange.Hours.GetNextMarketOpen(_lastDate, false);
+                var marketClose = Exchange.Hours.GetNextMarketClose(_lastDate, false);
+
+                if (_dataResolution == Time.OneHour || (_dataResolution == Time.OneDay && !UseStrictEndTime))
+                {
+                    marketOpen = marketOpen.RoundDown(_dataResolution);
+                    marketClose = marketClose.RoundUp(_dataResolution);
+                }
+
+                _marketOpenTimeSpan = marketOpen.TimeOfDay;
+                _marketCloseTimeSpan = marketClose.TimeOfDay;
+            }
+
+            // we only wait for data if the fill forward bar is not in the market open or close time
+            return fillForward.Time.TimeOfDay == _marketOpenTimeSpan || fillForward.EndTime.TimeOfDay == _marketCloseTimeSpan;
+        }
+
+        /// <summary>
+        /// Helper method to know how much we should wait before fill forwarding a bar in live trading
+        /// </summary>
+        /// <remarks>This allows us to create bars taking into account the market auction close and open official prices. Also it will
+        /// allow data providers which might have some delay on creating the bars on their end, to be consumed correctly, when available, by Lean</remarks>
+        public static TimeSpan GetMaximumDataTimeout(Resolution resolution)
+        {
+            switch (resolution)
+            {
+                case Resolution.Tick:
+                    return TimeSpan.Zero;
+                case Resolution.Second:
+                    return TimeSpan.FromSeconds(0.9);
+                case Resolution.Minute:
+                    return TimeSpan.FromMinutes(0.9);
+                case Resolution.Hour:
+                    return TimeSpan.FromMinutes(10);
+                case Resolution.Daily:
+                    return TimeSpan.FromMinutes(10);
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveMappingEventProvider.cs

@@ -0,0 +1,52 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    /// <remarks>Only special behavior is that it will refresh map file on each new tradable date event</remarks>
+    public class LiveMappingEventProvider : MappingEventProvider
+    {
+        /// <summary>
+        /// Check for new mappings
+        /// </summary>
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            var currentInstance = MapFile;
+            // refresh map file instance
+            InitializeMapFile();
+            var newInstance = MapFile;
+
+            // All future and option contracts sharing the same canonical symbol, share the same configuration too. Thus, in
+            // order to reduce logs, we log the configuration using the canonical symbol. See the optional parameter
+            // "overrideMessageFloodProtection" in Log.Trace() method for more information
+            var symbol = Config.Symbol.HasCanonical() ? Config.Symbol.Canonical.Value : Config.Symbol.Value;
+            Log.Trace($"LiveMappingEventProvider({Config.ToString(symbol)}): new tradable date {eventArgs.Date:yyyyMMdd}. " +
+                $"New MapFile: {!ReferenceEquals(currentInstance, newInstance)}. " +
+                $"MapFile.Count Old: {currentInstance?.Count()} New: {newInstance?.Count()}");
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveSplitEventProvider.cs

@@ -0,0 +1,57 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    /// <remarks>Only special behavior is that it will refresh factor file on each new tradable date event</remarks>
+    public class LiveSplitEventProvider : SplitEventProvider
+    {
+        /// <summary>
+        /// Check for dividends and returns them
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New Dividend event if any</returns>
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            var currentInstance = FactorFile;
+            // refresh factor file instance
+            InitializeFactorFile();
+            var newInstance = FactorFile;
+
+            if(currentInstance?.Count() != newInstance?.Count())
+            {
+                // All future and option contracts sharing the same canonical symbol, share the same configuration too. Thus, in
+                // order to reduce logs, we log the configuration using the canonical symbol. See the optional parameter
+                // "overrideMessageFloodProtection" in Log.Trace() method for more information
+                var symbol = Config.Symbol.HasCanonical() ? Config.Symbol.Canonical.Value : Config.Symbol.Value;
+                Log.Trace($"LiveSplitEventProvider({Config.ToString(symbol)}): new tradable date {eventArgs.Date:yyyyMMdd}. " +
+                    $"New FactorFile: {!ReferenceEquals(currentInstance, newInstance)}. " +
+                    $"FactorFile.Count Old: {currentInstance?.Count()} New: {newInstance?.Count()}");
+            }
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveSubscriptionEnumerator.cs

@@ -0,0 +1,120 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using System.Collections;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that will subscribe through the provided data queue handler and refresh the subscription if any mapping occurs
+    /// </summary>
+    public class LiveSubscriptionEnumerator : IEnumerator<BaseData>
+    {
+        private BaseData _current;
+        private readonly Symbol _requestedSymbol;
+        private SubscriptionDataConfig _currentConfig;
+        private IEnumerator<BaseData> _previousEnumerator;
+        private IEnumerator<BaseData> _underlyingEnumerator;
+
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
+        public BaseData Current => _current;
+
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public LiveSubscriptionEnumerator(SubscriptionDataConfig dataConfig, IDataQueueHandler dataQueueHandler, EventHandler handler, Func<SubscriptionDataConfig, bool> isExpired)
+        {
+            _requestedSymbol = dataConfig.Symbol;
+            _underlyingEnumerator = dataQueueHandler.SubscribeWithMapping(dataConfig, handler, isExpired, out _currentConfig);
+
+            // for any mapping event we will re subscribe
+            dataConfig.NewSymbol += (_, _) =>
+            {
+                dataQueueHandler.Unsubscribe(_currentConfig);
+                _previousEnumerator = _underlyingEnumerator;
+
+                var oldSymbol = _currentConfig.Symbol;
+                _underlyingEnumerator = dataQueueHandler.SubscribeWithMapping(dataConfig, handler, isExpired, out _currentConfig);
+
+                Log.Trace($"LiveSubscriptionEnumerator({_requestedSymbol}): " +
+                    $"resubscribing old: '{oldSymbol.Value}' new '{_currentConfig.Symbol.Value}'");
+            };
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element.
+        /// </summary>
+        public bool MoveNext()
+        {
+            if (_previousEnumerator != null)
+            {
+                // if previous is set we dispose of it here since we are the consumers of it
+                _previousEnumerator.DisposeSafely();
+                _previousEnumerator = null;
+            }
+
+            var result = _underlyingEnumerator.MoveNext();
+            if (result)
+            {
+                _current = _underlyingEnumerator.Current;
+            }
+            else
+            {
+                _current = null;
+            }
+
+            if (_current != null && _current.Symbol != _requestedSymbol)
+            {
+                // if we've done some mapping at this layer let's clone the underlying and set the requested symbol,
+                // don't trust the IDQH implementations for data uniqueness, since the configuration could be shared
+                _current = _current.Clone();
+                _current.Symbol = _requestedSymbol;
+            }
+
+            return result;
+        }
+
+        /// <summary>
+        /// Reset the IEnumeration
+        /// </summary>
+        public void Reset()
+        {
+            _underlyingEnumerator.Reset();
+        }
+
+        /// <summary>
+        /// Disposes of the used enumerators
+        /// </summary>
+        public void Dispose()
+        {
+            _previousEnumerator.DisposeSafely();
+            _underlyingEnumerator.DisposeSafely();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/MappingEventProvider.cs

@@ -0,0 +1,102 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    public class MappingEventProvider : ITradableDateEventProvider
+    {
+        private IMapFileProvider _mapFileProvider;
+
+        /// <summary>
+        /// The associated configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; private set; }
+
+        /// <summary>
+        /// The current instance being used
+        /// </summary>
+        protected MapFile MapFile { get; private set; }
+
+        /// <summary>
+        /// Initializes this instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public virtual void Initialize(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime)
+        {
+            _mapFileProvider = mapFileProvider;
+            Config = config;
+            InitializeMapFile();
+
+            if (MapFile.HasData(startTime.Date))
+            {
+                // initialize mapped symbol using request start date
+                Config.MappedSymbol = MapFile.GetMappedSymbol(startTime.Date, Config.MappedSymbol, Config.DataMappingMode);
+            }
+        }
+
+        /// <summary>
+        /// Check for new mappings
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New mapping event if any</returns>
+        public virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            if (Config.Symbol == eventArgs.Symbol
+                && MapFile.HasData(eventArgs.Date))
+            {
+                var old = Config.MappedSymbol;
+                var newSymbol = MapFile.GetMappedSymbol(eventArgs.Date, Config.MappedSymbol, Config.DataMappingMode);
+                Config.MappedSymbol = newSymbol;
+
+                // check to see if the symbol was remapped
+                if (old != Config.MappedSymbol)
+                {
+                    var changed = new SymbolChangedEvent(
+                        Config.Symbol,
+                        eventArgs.Date,
+                        old,
+                        Config.MappedSymbol);
+                    yield return changed;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the map file to use
+        /// </summary>
+        protected void InitializeMapFile()
+        {
+            MapFile = _mapFileProvider.ResolveMapFile(Config);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/NewDataAvailableEventArgs.cs

@@ -0,0 +1,32 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event args for when a new data point is ready to be emitted
+    /// </summary>
+    public class NewDataAvailableEventArgs : EventArgs
+    {
+        /// <summary>
+        /// The new data point
+        /// </summary>
+        public IBaseData DataPoint { get; set; }
+    }
+}

Engine/DataFeeds/Enumerators/PriceScaleFactorEnumerator.cs

@@ -0,0 +1,138 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// This enumerator will update the <see cref="SubscriptionDataConfig.PriceScaleFactor"/> when required
+    /// and adjust the raw <see cref="BaseData"/> prices based on the provided <see cref="SubscriptionDataConfig"/>.
+    /// Assumes the prices of the provided <see cref="IEnumerator"/> are in raw mode.
+    /// </summary>
+    public class PriceScaleFactorEnumerator : IEnumerator<BaseData>
+    {
+        private readonly IEnumerator<BaseData> _rawDataEnumerator;
+        private readonly SubscriptionDataConfig _config;
+        private readonly IFactorFileProvider _factorFileProvider;
+        private DateTime _nextTradableDate;
+        private IFactorProvider _factorFile;
+        private bool _liveMode;
+        private DateTime? _endDate;
+
+        /// <summary>
+        /// Explicit interface implementation for <see cref="Current"/>
+        /// </summary>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Last read <see cref="BaseData"/> object from this type and source
+        /// </summary>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Creates a new instance of the <see cref="PriceScaleFactorEnumerator"/>.
+        /// </summary>
+        /// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/> to enumerate for.
+        /// Will determine the <see cref="DataNormalizationMode"/> to use.</param>
+        /// <param name="factorFileProvider">The <see cref="IFactorFileProvider"/> instance to use</param>
+        /// <param name="liveMode">True, is this is a live mode data stream</param>
+        /// <param name="endDate">The enumerator end date</param>
+        /// <remarks>
+        /// For <see cref="DataNormalizationMode.ScaledRaw"/> normalization mode,
+        /// the prices are scaled to the prices on the <paramref name="endDate"/>
+        /// </remarks>
+        public PriceScaleFactorEnumerator(
+            IEnumerator<BaseData> rawDataEnumerator,
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            bool liveMode = false,
+            DateTime? endDate = null)
+        {
+            _config = config;
+            _liveMode = liveMode;
+            _nextTradableDate = DateTime.MinValue;
+            _rawDataEnumerator = rawDataEnumerator;
+            _factorFileProvider = factorFileProvider;
+            _endDate = endDate;
+        }
+
+        /// <summary>
+        /// Dispose of the underlying enumerator.
+        /// </summary>
+        public void Dispose()
+        {
+            _rawDataEnumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// True if the enumerator was successfully advanced to the next element;
+        /// False if the enumerator has passed the end of the collection.
+        /// </returns>
+        public bool MoveNext()
+        {
+            var underlyingReturnValue = _rawDataEnumerator.MoveNext();
+            Current = _rawDataEnumerator.Current;
+
+            if (underlyingReturnValue
+                && Current != null
+                && _factorFileProvider != null
+                && _config.DataNormalizationMode != DataNormalizationMode.Raw)
+            {
+                var priceScaleFrontier = Current.GetUpdatePriceScaleFrontier();
+                if (priceScaleFrontier >= _nextTradableDate)
+                {
+                    _factorFile = _factorFileProvider.Get(_config.Symbol);
+                    _config.PriceScaleFactor = _factorFile.GetPriceScale(priceScaleFrontier.Date, _config.DataNormalizationMode, _config.ContractDepthOffset, _config.DataMappingMode, _endDate);
+
+                    // update factor files every day
+                    _nextTradableDate = priceScaleFrontier.Date.AddDays(1);
+                    if (_liveMode)
+                    {
+                        // in live trading we add a offset to make sure new factor files are available
+                        _nextTradableDate = _nextTradableDate.Add(Time.LiveAuxiliaryDataOffset);
+                    }
+                }
+
+                Current = Current.Normalize(_config.PriceScaleFactor, _config.DataNormalizationMode, _config.SumOfDividends);
+            }
+
+            return underlyingReturnValue;
+        }
+
+        /// <summary>
+        /// Reset the IEnumeration
+        /// </summary>
+        /// <remarks>Not used</remarks>
+        public void Reset()
+        {
+            throw new NotImplementedException("Reset method not implemented. Assumes loop will only be used once.");
+        }
+    }
+}

Engine/DataFeeds/Enumerators/QuoteBarFillForwardEnumerator.cs

@@ -0,0 +1,113 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// The QuoteBarFillForwardEnumerator wraps an existing base data enumerator
+    /// If the current QuoteBar has null Bid and/or Ask bars, it copies them from the previous QuoteBar
+    /// </summary>
+    public class QuoteBarFillForwardEnumerator : IEnumerator<BaseData>
+    {
+        private QuoteBar _previous;
+        private readonly IEnumerator<BaseData> _enumerator;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FillForwardEnumerator"/> class
+        /// </summary>
+        public QuoteBarFillForwardEnumerator(IEnumerator<BaseData> enumerator)
+        {
+            _enumerator = enumerator;
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception>
+        public bool MoveNext()
+        {
+            if (!_enumerator.MoveNext()) return false;
+
+            var bar = _enumerator.Current as QuoteBar;
+            if (bar != null)
+            {
+                if (_previous != null)
+                {
+                    if (bar.Bid == null)
+                    {
+                        bar.Bid = _previous.Bid;
+                    }
+
+                    if (bar.Ask == null)
+                    {
+                        bar.Ask = _previous.Ask;
+                    }
+                }
+
+                _previous = bar;
+            }
+
+            Current = _enumerator.Current;
+
+            return true;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/RateLimitEnumerator.cs

@@ -0,0 +1,128 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides augmentation of how often an enumerator can be called. Time is measured using
+    /// an <see cref="ITimeProvider"/> instance and calls to the underlying enumerator are limited
+    /// to a minimum time between each call.
+    /// </summary>
+    public class RateLimitEnumerator<T> : IEnumerator<T>
+    {
+        private T _current;
+        private DateTime _lastCallTime;
+
+        private readonly ITimeProvider _timeProvider;
+        private readonly IEnumerator<T> _enumerator;
+        private readonly TimeSpan _minimumTimeBetweenCalls;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RateLimitEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to place rate limits on</param>
+        /// <param name="timeProvider">Time provider used for determing the time between calls</param>
+        /// <param name="minimumTimeBetweenCalls">The minimum time allowed between calls to the underlying enumerator</param>
+        public RateLimitEnumerator(IEnumerator<T> enumerator, ITimeProvider timeProvider, TimeSpan minimumTimeBetweenCalls)
+        {
+            _enumerator = enumerator;
+            _timeProvider = timeProvider;
+            _minimumTimeBetweenCalls = minimumTimeBetweenCalls;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            // determine time since last successful call, do this on units of the minimum time
+            // this will give us nice round emit times
+            var currentTime = _timeProvider.GetUtcNow().RoundDown(_minimumTimeBetweenCalls);
+            var timeBetweenCalls = currentTime - _lastCallTime;
+
+            // if within limits, patch it through to move next
+            if (timeBetweenCalls >= _minimumTimeBetweenCalls)
+            {
+                if (!_enumerator.MoveNext())
+                {
+                    // our underlying is finished
+                    _current = default(T);
+                    return false;
+                }
+
+                // only update last call time on non rate limited requests
+                _lastCallTime = currentTime;
+                _current = _enumerator.Current;
+            }
+            else
+            {
+                // we've been rate limitted
+                _current = default(T);
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/RefreshEnumerator.cs

@@ -0,0 +1,135 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using QuantConnect.Util;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="IEnumerator{T}"/> that will
+    /// always return true via MoveNext.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public class RefreshEnumerator<T> : IEnumerator<T>
+    {
+        private T _current;
+        private IEnumerator<T> _enumerator;
+        private readonly Func<IEnumerator<T>> _enumeratorFactory;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RefreshEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="enumeratorFactory">Enumerator factory used to regenerate the underlying
+        /// enumerator when it ends</param>
+        public RefreshEnumerator(Func<IEnumerator<T>> enumeratorFactory)
+        {
+            _enumeratorFactory = enumeratorFactory;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            if (_enumerator == null)
+            {
+                _enumerator = _enumeratorFactory.Invoke();
+            }
+
+            var moveNext = false;
+            try
+            {
+                moveNext = _enumerator.MoveNext();
+                if (moveNext)
+                {
+                    _current = _enumerator.Current;
+                }
+            }
+            catch (IOException exception)
+            {
+                // we will ignore stale file handle exceptions and retry instead, enumerator will be refreshed
+                if (exception.Message == null || !exception.Message.Contains("Stale file handle", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    throw;
+                }
+            }
+
+            if (!moveNext)
+            {
+                _enumerator.DisposeSafely();
+
+                _enumerator = null;
+                _current = default(T);
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            if (_enumerator != null)
+            {
+                _enumerator.Reset();
+            }
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current
+        {
+            get { return _current; }
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            if (_enumerator != null)
+            {
+                _enumerator.Dispose();
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/ScannableEnumerator.cs

@@ -0,0 +1,174 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Data.Consolidators;
+using System;
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// An implementation of <see cref="IEnumerator{T}"/> that relies on "consolidated" data
+    /// </summary>
+    /// <typeparam name="T">The item type yielded by the enumerator</typeparam>
+    public class ScannableEnumerator<T> : IEnumerator<T> where T : class, IBaseData
+    {
+        private T _current;
+        private bool _consolidated;
+        private bool _isPeriodBase;
+        private bool _validateInputType;
+        private Type _consolidatorInputType;
+        private readonly DateTimeZone _timeZone;
+        private readonly ConcurrentQueue<T> _queue;
+        private readonly ITimeProvider _timeProvider;
+        private readonly EventHandler _newDataAvailableHandler;
+        private readonly IDataConsolidator _consolidator;
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current => _current;
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ScannableEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="consolidator">Consolidator taking BaseData updates and firing events containing new 'consolidated' data</param>
+        /// <param name="timeZone">The time zone the raw data is time stamped in</param>
+        /// <param name="timeProvider">The time provider instance used to determine when bars are completed and can be emitted</param>
+        /// <param name="newDataAvailableHandler">The event handler for a new available data point</param>
+        /// <param name="isPeriodBased">The consolidator is period based, this will enable scanning on <see cref="MoveNext"/></param>
+        public ScannableEnumerator(IDataConsolidator consolidator, DateTimeZone timeZone, ITimeProvider timeProvider, EventHandler newDataAvailableHandler, bool isPeriodBased = true)
+        {
+            _timeZone = timeZone;
+            _timeProvider = timeProvider;
+            _consolidator = consolidator;
+            _isPeriodBase = isPeriodBased;
+            _queue = new ConcurrentQueue<T>();
+            _consolidatorInputType = consolidator.InputType;
+            _validateInputType = _consolidatorInputType != typeof(BaseData);
+            _newDataAvailableHandler = newDataAvailableHandler ?? ((s, e) => { });
+
+            _consolidator.DataConsolidated += DataConsolidatedHandler;
+        }
+
+        /// <summary>
+        /// Updates the consolidator
+        /// </summary>
+        /// <param name="data">The data to consolidate</param>
+        public void Update(T data)
+        {
+            // if the input type of the consolidator isn't generic we validate it's correct before sending it in
+            if (_validateInputType && data.GetType() != _consolidatorInputType)
+            {
+                return;
+            }
+
+            if (_isPeriodBase)
+            {
+                // we only need to lock if it's period base since the move next call could trigger a scan
+                lock (_consolidator)
+                {
+                    _consolidator.Update(data);
+                }
+            }
+            else
+            {
+                _consolidator.Update(data);
+            }
+        }
+
+        /// <summary>
+        /// Enqueues the new data into this enumerator
+        /// </summary>
+        /// <param name="data">The data to be enqueued</param>
+        private void Enqueue(T data)
+        {
+            _queue.Enqueue(data);
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            if (!_queue.TryDequeue(out _current) && _isPeriodBase)
+            {
+                _consolidated = false;
+                lock (_consolidator)
+                {
+                    // if there is a working bar we will try to pull it out if the time is right, each consolidator knows when it's right
+                    var localTime = _timeProvider.GetUtcNow().ConvertFromUtc(_timeZone);
+                    _consolidator.Scan(localTime);
+                }
+
+                if (_consolidated)
+                {
+                    _queue.TryDequeue(out _current);
+                }
+            }
+
+            // even if we don't have data to return, we haven't technically
+            // passed the end of the collection, so always return true until
+            // the enumerator is explicitly disposed or ended
+            return true;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _consolidator.DataConsolidated -= DataConsolidatedHandler;
+        }
+
+        private void DataConsolidatedHandler(object sender, IBaseData data)
+        {
+            var dataPoint = data as T;
+            _consolidated = true;
+            Enqueue(dataPoint);
+            _newDataAvailableHandler(sender, new NewDataAvailableEventArgs { DataPoint = dataPoint });
+        }
+    }
+}

Engine/DataFeeds/Enumerators/ScheduledEnumerator.cs

@@ -0,0 +1,197 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NodaTime;
+using QuantConnect.Data;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// This enumerator will filter out data of the underlying enumerator based on a provided schedule.
+    /// Will respect the schedule above the data, meaning will let older data through if the underlying provides none for the schedule date
+    /// </summary>
+    public class ScheduledEnumerator : IEnumerator<BaseData>
+    {
+        private readonly IEnumerator<BaseData> _underlyingEnumerator;
+        private readonly IEnumerator<DateTime> _scheduledTimes;
+        private readonly ITimeProvider _frontierTimeProvider;
+        private readonly DateTimeZone _scheduleTimeZone;
+        private BaseData _underlyingCandidateDataPoint;
+        private bool _scheduledTimesEnded;
+
+        /// <summary>
+        /// The current data point
+        /// </summary>
+        public BaseData Current { get; private set; }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="underlyingEnumerator">The underlying enumerator to filter</param>
+        /// <param name="scheduledTimes">The scheduled times to emit new data points</param>
+        /// <param name="frontierTimeProvider"></param>
+        /// <param name="scheduleTimeZone"></param>
+        /// <param name="startTime">the underlying request start time</param>
+        public ScheduledEnumerator(IEnumerator<BaseData> underlyingEnumerator,
+            IEnumerable<DateTime> scheduledTimes,
+            ITimeProvider frontierTimeProvider,
+            DateTimeZone scheduleTimeZone,
+            DateTime startTime)
+        {
+            _scheduleTimeZone = scheduleTimeZone;
+            _frontierTimeProvider = frontierTimeProvider;
+            _underlyingEnumerator = underlyingEnumerator;
+            _scheduledTimes = scheduledTimes.GetEnumerator();
+            // move our schedule enumerator to current start time
+            MoveScheduleForward(startTime);
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns> True if the enumerator was successfully advanced to the next element;
+        /// false if the enumerator has passed the end of the collection.
+        /// </returns>
+        public bool MoveNext()
+        {
+            if (_scheduledTimesEnded)
+            {
+                Current = null;
+                return false;
+            }
+
+            // lets get our candidate data point to emit
+            if (_underlyingCandidateDataPoint == null)
+            {
+                if (_underlyingEnumerator.Current != null && _underlyingEnumerator.Current.EndTime <= _scheduledTimes.Current)
+                {
+                    _underlyingCandidateDataPoint = _underlyingEnumerator.Current;
+                }
+                else if (Current != null)
+                {
+                    // we will keep the last data point, even if we already emitted it, there could be a case where the user has a schedule in a
+                    // period where there's not new data (or it's far in the future) so let's just FF the previous point
+                    _underlyingCandidateDataPoint = Current.Clone(fillForward: true);
+                }
+            }
+
+            // lets try to get a better candidate
+            if (_underlyingEnumerator.Current == null
+                || _underlyingEnumerator.Current.EndTime < _scheduledTimes.Current)
+            {
+                bool pullAgain;
+                do
+                {
+                    pullAgain = false;
+                    if (!_underlyingEnumerator.MoveNext())
+                    {
+                        if (_underlyingCandidateDataPoint != null)
+                        {
+                            // if we still have a candidate wait till we emit him before stopping
+                            break;
+                        }
+                        Current = null;
+                        return false;
+                    }
+
+                    if (_underlyingEnumerator.Current != null)
+                    {
+                        if (_underlyingEnumerator.Current.EndTime <= _scheduledTimes.Current)
+                        {
+                            // lets try again
+                            pullAgain = true;
+                            // we got another data point which is a newer candidate to emit so let use it instead
+                            // and drop the previous
+                            _underlyingCandidateDataPoint = _underlyingEnumerator.Current;
+                        }
+                        else if (_underlyingCandidateDataPoint == null)
+                        {
+                            // this is the first data point we got and it's After our schedule, let's move our schedule forward
+                            _underlyingCandidateDataPoint = _underlyingEnumerator.Current;
+                            MoveScheduleForward();
+                        }
+                    }
+                } while (pullAgain);
+            }
+
+            if (_underlyingCandidateDataPoint != null
+            // if we are at or past the schedule time we try to emit, in backtest this emits right away, since time is data driven, in live though
+            // we don't emit right away because the underlying might provide us with a newer data point
+                && _scheduledTimes.Current.ConvertToUtc(_scheduleTimeZone) <= GetUtcNow())
+            {
+                Current = _underlyingCandidateDataPoint;
+                // we align the data endtime with the schedule, we respect the schedule above the data time. In backtesting,
+                // time is driven by the data, so let's make sure we emit at the scheduled time even if the data is older
+                Current.EndTime = _scheduledTimes.Current;
+                if (Current.Time > Current.EndTime)
+                {
+                    Current.Time = _scheduledTimes.Current;
+                }
+
+                MoveScheduleForward();
+                _underlyingCandidateDataPoint = null;
+                return true;
+            }
+
+            Current = null;
+            return true;
+        }
+
+        /// <summary>
+        /// Resets the underlying enumerator
+        /// </summary>
+        public void Reset()
+        {
+            _underlyingEnumerator.Reset();
+        }
+
+        /// <summary>
+        /// Disposes of the underlying enumerator
+        /// </summary>
+        public void Dispose()
+        {
+            _scheduledTimes.Dispose();
+            _underlyingEnumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Available in live trading only, in backtesting frontier is driven and sycned already by the data itself
+        /// so we can't hold data here based on it
+        /// </summary>
+        private DateTime GetUtcNow()
+        {
+            if (_frontierTimeProvider != null)
+            {
+                return _frontierTimeProvider.GetUtcNow();
+            }
+            return DateTime.MaxValue;
+        }
+
+        private void MoveScheduleForward(DateTime? frontier = null)
+        {
+            do
+            {
+                _scheduledTimesEnded = !_scheduledTimes.MoveNext();
+            }
+            while (!_scheduledTimesEnded && frontier.HasValue && _scheduledTimes.Current < frontier.Value);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SortEnumerator.cs

@@ -0,0 +1,113 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Provides an enumerator for sorting collections of <see cref="BaseData"/> objects based on a specified property.
+    /// The sorting occurs lazily, only when enumeration begins.
+    /// </summary>
+    /// <typeparam name="TKey">The type of the key used for sorting.</typeparam>
+    public sealed class SortEnumerator<TKey> : IEnumerator<BaseData>, IDisposable
+    {
+        private readonly IEnumerable<BaseData> _data;
+#pragma warning disable CA2213 // call csutom DisposeSafely() in Dispose()
+        private IEnumerator<BaseData> _sortedEnumerator;
+#pragma warning restore CA2213 // call csutom DisposeSafely() in Dispose()
+        private readonly Func<BaseData, TKey> _keySelector;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SortEnumerator{TKey}"/> class.
+        /// </summary>
+        /// <param name="data">The collection of <see cref="BaseData"/> to enumerate over.</param>
+        /// <param name="keySelector">A function that defines the key to sort by. Defaults to sorting by <see cref="BaseData.EndTime"/>.</param>
+        public SortEnumerator(IEnumerable<BaseData> data, Func<BaseData, TKey> keySelector = null)
+        {
+            _data = data;
+            _sortedEnumerator = GetSortedData().GetEnumerator();
+            _keySelector = keySelector ??= baseData => (TKey)(object)baseData.EndTime;
+        }
+
+        /// <summary>
+        /// Static method to wrap an enumerable with the sort enumerator.
+        /// </summary>
+        /// <param name="preSorted">Indicates if the data is pre-sorted.</param>
+        /// <param name="data">The data to be wrapped into the enumerator.</param>
+        /// <returns>An enumerator over the <see cref="BaseData"/>.</returns>
+        public static IEnumerator<BaseData> TryWrapSortEnumerator(bool preSorted, IEnumerable<BaseData> data)
+        {
+            return preSorted ? new SortEnumerator<TKey>(data) : data.GetEnumerator();
+        }
+
+        /// <summary>
+        /// Lazily retrieves the sorted data.
+        /// </summary>
+        /// <returns>An enumerable collection of <see cref="BaseData"/>.</returns>
+        private IEnumerable<BaseData> GetSortedData()
+        {
+            foreach (var item in _data.OrderBy(_keySelector))
+            {
+                yield return item;
+            }
+        }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Gets the current <see cref="BaseData"/> element in the collection.
+        /// </summary>
+        public BaseData Current
+        {
+            get => _sortedEnumerator.Current;
+        }
+
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// <c>true</c> if the enumerator was successfully advanced to the next element; 
+        /// <c>false</c> if the enumerator has passed the end of the collection.
+        /// </returns>
+        public bool MoveNext()
+        {
+            return _sortedEnumerator.MoveNext();
+        }
+
+        /// <summary>
+        /// Resets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        public void Reset()
+        {
+            _sortedEnumerator = null;
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="SortEnumerator{TKey}"/> and suppresses finalization.
+        /// </summary>
+        public void Dispose()
+        {
+            _sortedEnumerator?.DisposeSafely();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SplitEventProvider.cs

@@ -0,0 +1,121 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="Split"/> events
+    /// </summary>
+    public class SplitEventProvider : ITradableDateEventProvider
+    {
+        // we set the split factor when we encounter a split in the factor file
+        // and on the next trading day we use this data to produce the split instance
+        private decimal? _splitFactor;
+        private decimal _referencePrice;
+        private IFactorFileProvider _factorFileProvider;
+        private MapFile _mapFile;
+
+        /// <summary>
+        /// The current instance being used
+        /// </summary>
+        protected CorporateFactorProvider FactorFile { get; private set; }
+
+        /// <summary>
+        /// The associated configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; private set; }
+
+        /// <summary>
+        /// Initializes this instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
+        /// <param name="startTime">Start date for the data request</param>
+        public void Initialize(
+            SubscriptionDataConfig config,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
+            DateTime startTime)
+        {
+            Config = config;
+            _factorFileProvider = factorFileProvider;
+            _mapFile = mapFileProvider.ResolveMapFile(Config);
+            InitializeFactorFile();
+        }
+
+        /// <summary>
+        /// Check for new splits
+        /// </summary>
+        /// <param name="eventArgs">The new tradable day event arguments</param>
+        /// <returns>New split event if any</returns>
+        public virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            if (Config.Symbol == eventArgs.Symbol
+                && FactorFile != null
+                && _mapFile.HasData(eventArgs.Date))
+            {
+                var factor = _splitFactor;
+                if (factor != null)
+                {
+                    var close = _referencePrice;
+                    if (close == 0)
+                    {
+                        throw new InvalidOperationException($"Zero reference price for {Config.Symbol} split at {eventArgs.Date}");
+                    }
+
+                    _splitFactor = null;
+                    _referencePrice = 0;
+                    yield return new Split(
+                        eventArgs.Symbol,
+                        eventArgs.Date,
+                        close,
+                        factor.Value,
+                        SplitType.SplitOccurred);
+                }
+
+                decimal splitFactor;
+                decimal referencePrice;
+                if (FactorFile.HasSplitEventOnNextTradingDay(eventArgs.Date, out splitFactor, out referencePrice))
+                {
+                    _splitFactor = splitFactor;
+                    _referencePrice = referencePrice;
+                    yield return new Split(
+                        eventArgs.Symbol,
+                        eventArgs.Date,
+                        eventArgs.LastRawPrice ?? 0,
+                        splitFactor,
+                        SplitType.Warning);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the factor file to use
+        /// </summary>
+        protected void InitializeFactorFile()
+        {
+            FactorFile = _factorFileProvider.Get(Config.Symbol) as CorporateFactorProvider;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/StrictDailyEndTimesEnumerator.cs

@@ -0,0 +1,95 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using System.Collections;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that will handle adjusting daily strict end times if appropriate
+    /// </summary>
+    public class StrictDailyEndTimesEnumerator : IEnumerator<BaseData>
+    {
+        private readonly DateTime _localStartTime;
+        private readonly SecurityExchangeHours _securityExchange;
+        private readonly IEnumerator<BaseData> _underlying;
+
+        /// <summary>
+        /// Current value of the enumerator
+        /// </summary>
+        public BaseData Current { get; private set; }
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public StrictDailyEndTimesEnumerator(IEnumerator<BaseData> underlying, SecurityExchangeHours securityExchangeHours, DateTime localStartTime)
+        {
+            _underlying = underlying;
+            _localStartTime = localStartTime;
+            _securityExchange = securityExchangeHours;
+        }
+
+        /// <summary>
+        /// Move to the next date
+        /// </summary>
+        public bool MoveNext()
+        {
+            Current = null;
+            bool result;
+            do
+            {
+                result = _underlying.MoveNext();
+                if (!result || !LeanData.UseDailyStrictEndTimes(_underlying.Current?.GetType()))
+                {
+                    break;
+                }
+
+                // before setting the strict daily end times, let's clone it because underlying enumerator (SubscriptionDataReader) might be using it
+                var pontentialNewBar = _underlying.Current.Clone();
+                if (LeanData.SetStrictEndTimes(pontentialNewBar, _securityExchange) && pontentialNewBar.EndTime >= _localStartTime)
+                {
+                    Current = pontentialNewBar;
+                    break;
+                }
+            }
+            while (true);
+
+            return result;
+        }
+
+        /// <summary>
+        /// Reset the enumerator
+        /// </summary>
+        public void Reset()
+        {
+            _underlying.Reset();
+        }
+
+        /// <summary>
+        /// Dispose the enumerator
+        /// </summary>
+        public void Dispose()
+        {
+            _underlying.Dispose();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SubscriptionDataEnumerator.cs

@@ -0,0 +1,106 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// An <see cref="IEnumerator{SubscriptionData}"/> which wraps an existing <see cref="IEnumerator{BaseData}"/>.
+    /// </summary>
+    /// <remarks>Using this class is important, versus directly yielding, because we setup the <see cref="Dispose"/> chain</remarks>
+    public class SubscriptionDataEnumerator : IEnumerator<SubscriptionData>
+    {
+        private readonly IEnumerator<BaseData> _enumerator;
+        private readonly SubscriptionDataConfig _configuration;
+        private readonly SecurityExchangeHours _exchangeHours;
+        private readonly TimeZoneOffsetProvider _offsetProvider;
+        private readonly bool _isUniverse;
+        private readonly bool _dailyStrictEndTimeEnabled;
+
+        object IEnumerator.Current => Current;
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        public SubscriptionData Current { get; private set; }
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="configuration">The subscription's configuration</param>
+        /// <param name="exchangeHours">The security's exchange hours</param>
+        /// <param name="offsetProvider">The subscription's time zone offset provider</param>
+        /// <param name="enumerator">The underlying data enumerator</param>
+        /// <param name="isUniverse">The subscription is a universe subscription</param>
+        /// <returns>A subscription data enumerator</returns>
+        public SubscriptionDataEnumerator(SubscriptionDataConfig configuration,
+            SecurityExchangeHours exchangeHours,
+            TimeZoneOffsetProvider offsetProvider,
+            IEnumerator<BaseData> enumerator,
+            bool isUniverse,
+            bool dailyStrictEndTimeEnabled)
+        {
+            _enumerator = enumerator;
+            _offsetProvider = offsetProvider;
+            _exchangeHours = exchangeHours;
+            _configuration = configuration;
+            _isUniverse = isUniverse;
+            _dailyStrictEndTimeEnabled = dailyStrictEndTimeEnabled;
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>True if the enumerator was successfully advanced to the next element;
+        /// False if the enumerator has passed the end of the collection.</returns>
+        public bool MoveNext()
+        {
+            var result = _enumerator.MoveNext();
+            if (result)
+            {
+                // Use our config filter to see if we should emit this
+                // This currently catches Auxiliary data that we don't want to emit
+                if (_enumerator.Current != null && !_configuration.ShouldEmitData(_enumerator.Current, _isUniverse))
+                {
+                    // We shouldn't emit this data, so we will MoveNext() again.
+                    return MoveNext();
+                }
+
+                Current = SubscriptionData.Create(_dailyStrictEndTimeEnabled, _configuration, _exchangeHours, _offsetProvider, _enumerator.Current, _configuration.DataNormalizationMode);
+            }
+            return result;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SubscriptionFilterEnumerator.cs

@@ -0,0 +1,195 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+using QuantConnect.Securities.Interfaces;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Implements a wrapper around a base data enumerator to provide a final filtering step
+    /// </summary>
+    public class SubscriptionFilterEnumerator : IEnumerator<BaseData>
+    {
+        /// <summary>
+        /// Fired when there's an error executing a user's data filter
+        /// </summary>
+        public event EventHandler<Exception> DataFilterError;
+
+        private readonly bool _liveMode;
+        private readonly Security _security;
+        private readonly DateTime _endTime;
+        private readonly bool _extendedMarketHours;
+        private readonly SecurityExchangeHours _exchangeHours;
+        private readonly ISecurityDataFilter _dataFilter;
+        private readonly IEnumerator<BaseData> _enumerator;
+
+        /// <summary>
+        /// Convenience method to wrap the enumerator and attach the data filter event to log and alery users of errors
+        /// </summary>
+        /// <param name="resultHandler">Result handler reference used to send errors</param>
+        /// <param name="enumerator">The source enumerator to be wrapped</param>
+        /// <param name="security">The security who's data is being enumerated</param>
+        /// <param name="endTime">The end time of the subscription</param>
+        /// <param name="extendedMarketHours">True if extended market hours are enabled</param>
+        /// <param name="liveMode">True if live mode</param>
+        /// <param name="securityExchangeHours">The security exchange hours instance to use</param>
+        /// <returns>A new instance of the <see cref="SubscriptionFilterEnumerator"/> class that has had it's <see cref="DataFilterError"/>
+        /// event subscribed to to send errors to the result handler</returns>
+        public static SubscriptionFilterEnumerator WrapForDataFeed(IResultHandler resultHandler, IEnumerator<BaseData> enumerator, Security security, DateTime endTime, bool extendedMarketHours, bool liveMode,
+            SecurityExchangeHours securityExchangeHours)
+        {
+            var filter = new SubscriptionFilterEnumerator(enumerator, security, endTime, extendedMarketHours, liveMode, securityExchangeHours);
+            filter.DataFilterError += (sender, exception) =>
+            {
+                Log.Error(exception, "WrapForDataFeed");
+                resultHandler.RuntimeError("Runtime error applying data filter. Assuming filter pass: " + exception.Message, exception.StackTrace);
+            };
+            return filter;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SubscriptionFilterEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerator">The source enumerator to be wrapped</param>
+        /// <param name="security">The security containing an exchange and data filter</param>
+        /// <param name="endTime">The end time of the subscription</param>
+        /// <param name="extendedMarketHours">True if extended market hours are enabled</param>
+        /// <param name="liveMode">True if live mode</param>
+        /// <param name="securityExchangeHours">The security exchange hours instance to use</param>
+        public SubscriptionFilterEnumerator(IEnumerator<BaseData> enumerator, Security security, DateTime endTime, bool extendedMarketHours, bool liveMode, SecurityExchangeHours securityExchangeHours)
+        {
+            _liveMode = liveMode;
+            _enumerator = enumerator;
+            _security = security;
+            _endTime = endTime;
+            _exchangeHours = securityExchangeHours;
+            _dataFilter = _security.DataFilter;
+            _extendedMarketHours = extendedMarketHours;
+        }
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public BaseData Current
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public bool MoveNext()
+        {
+            while (_enumerator.MoveNext())
+            {
+                var current = _enumerator.Current;
+                if (current != null)
+                {
+                    try
+                    {
+                        // execute user data filters
+                        if (current.DataType != MarketDataType.Auxiliary && !_dataFilter.Filter(_security, current))
+                        {
+                            continue;
+                        }
+                    }
+                    catch (Exception err)
+                    {
+                        OnDataFilterError(err);
+                        continue;
+                    }
+
+                    // verify that the bar is within the exchange's market hours
+                    if (current.DataType != MarketDataType.Auxiliary && !_exchangeHours.IsOpen(current.Time, current.EndTime, _extendedMarketHours))
+                    {
+                        if (_liveMode && !current.IsFillForward)
+                        {
+                            // TODO: replace for setting security.RealTimePrice not to modify security cache data directly
+                            _security.SetMarketPrice(current);
+                        }
+                        continue;
+                    }
+
+                    // make sure we haven't passed the end
+                    if (current.Time > _endTime)
+                    {
+                        return false;
+                    }
+                }
+
+                Current = current;
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception><filterpriority>2</filterpriority>
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        /// <summary>
+        /// Event invocated for the <see cref="DataFilterError"/> event
+        /// </summary>
+        /// <param name="exception">The exception that was thrown when trying to perform data filtering</param>
+        private void OnDataFilterError(Exception exception)
+        {
+            var handler = DataFilterError;
+            if (handler != null) handler(this, exception);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SynchronizingBaseDataEnumerator.cs

@@ -0,0 +1,55 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using QuantConnect.Data;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Represents an enumerator capable of synchronizing other base data enumerators in time.
+    /// This assumes that all enumerators have data time stamped in the same time zone
+    /// </summary>
+    public class SynchronizingBaseDataEnumerator : SynchronizingEnumerator<BaseData>
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingBaseDataEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        public SynchronizingBaseDataEnumerator(params IEnumerator<BaseData>[] enumerators)
+            : this((IEnumerable<IEnumerator>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingBaseDataEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        public SynchronizingBaseDataEnumerator(IEnumerable<IEnumerator> enumerators) : base((IEnumerable<IEnumerator<BaseData>>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Gets the Timestamp for the data
+        /// </summary>
+        protected override DateTime GetInstanceTime(BaseData instance)
+        {
+            return instance.EndTime;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SynchronizingEnumerator.cs

@@ -0,0 +1,204 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Represents an enumerator capable of synchronizing other enumerators of type T in time.
+    /// This assumes that all enumerators have data time stamped in the same time zone
+    /// </summary>
+    public abstract class SynchronizingEnumerator<T> : IEnumerator<T>
+    {
+        private IEnumerator<T> _syncer;
+        private readonly IEnumerator<T>[] _enumerators;
+
+        /// <summary>
+        /// Gets the Timestamp for the data
+        /// </summary>
+        protected abstract DateTime GetInstanceTime(T instance);
+
+        /// <summary>
+        /// Gets the element in the collection at the current position of the enumerator.
+        /// </summary>
+        /// <returns>
+        /// The element in the collection at the current position of the enumerator.
+        /// </returns>
+        public T Current
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the current element in the collection.
+        /// </summary>
+        /// <returns>
+        /// The current element in the collection.
+        /// </returns>
+        object IEnumerator.Current
+        {
+            get { return Current; }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        /// <remark>The type of data we want, for example, <see cref="BaseData"/> or <see cref="Slice"/>, ect...</remark>
+        protected SynchronizingEnumerator(params IEnumerator<T>[] enumerators)
+            : this ((IEnumerable<IEnumerator<T>>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingEnumerator{T}"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        /// <remark>The type of data we want, for example, <see cref="BaseData"/> or <see cref="Slice"/>, ect...</remark>
+        protected SynchronizingEnumerator(IEnumerable<IEnumerator<T>> enumerators)
+        {
+            _enumerators = enumerators.ToArray();
+            _syncer = GetSynchronizedEnumerator(_enumerators);
+        }
+
+        /// <summary>
+        /// Advances the enumerator to the next element of the collection.
+        /// </summary>
+        /// <returns>
+        /// true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.
+        /// </returns>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception>
+        public bool MoveNext()
+        {
+            var moveNext =  _syncer.MoveNext();
+            Current = moveNext ? _syncer.Current : default(T);
+            return moveNext;
+        }
+
+        /// <summary>
+        /// Sets the enumerator to its initial position, which is before the first element in the collection.
+        /// </summary>
+        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created. </exception>
+        public void Reset()
+        {
+            foreach (var enumerator in _enumerators)
+            {
+                enumerator.Reset();
+            }
+            // don't call syncer.reset since the impl will just throw
+            _syncer = GetSynchronizedEnumerator(_enumerators);
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            foreach (var enumerator in _enumerators)
+            {
+                enumerator.Dispose();
+            }
+            _syncer.Dispose();
+        }
+
+        /// <summary>
+        /// Synchronization system for the enumerator:
+        /// </summary>
+        /// <param name="enumerators"></param>
+        /// <returns></returns>
+        private IEnumerator<T> GetSynchronizedEnumerator(IEnumerator<T>[] enumerators)
+        {
+            return GetBruteForceMethod(enumerators);
+        }
+
+        /// <summary>
+        /// Brute force implementation for synchronizing the enumerator.
+        /// Will remove enumerators returning false to the call to MoveNext.
+        /// Will not remove enumerators with Current Null returning true to the call to MoveNext
+        /// </summary>
+        private IEnumerator<T> GetBruteForceMethod(IEnumerator<T>[] enumerators)
+        {
+            var ticks = DateTime.MaxValue.Ticks;
+            var collection = new HashSet<IEnumerator<T>>();
+            foreach (var enumerator in enumerators)
+            {
+                if (enumerator.MoveNext())
+                {
+                    if (enumerator.Current != null)
+                    {
+                        ticks = Math.Min(ticks, GetInstanceTime(enumerator.Current).Ticks);
+                    }
+                    collection.Add(enumerator);
+                }
+                else
+                {
+                    enumerator.Dispose();
+                }
+            }
+
+            var frontier = new DateTime(ticks);
+            var toRemove = new List<IEnumerator<T>>();
+            while (collection.Count > 0)
+            {
+                var nextFrontierTicks = DateTime.MaxValue.Ticks;
+                foreach (var enumerator in collection)
+                {
+                    while (enumerator.Current == null || GetInstanceTime(enumerator.Current) <= frontier)
+                    {
+                        if (enumerator.Current != null)
+                        {
+                            yield return enumerator.Current;
+                        }
+                        if (!enumerator.MoveNext())
+                        {
+                            toRemove.Add(enumerator);
+                            break;
+                        }
+                        if (enumerator.Current == null)
+                        {
+                            break;
+                        }
+                    }
+
+                    if (enumerator.Current != null)
+                    {
+                        nextFrontierTicks = Math.Min(nextFrontierTicks, GetInstanceTime(enumerator.Current).Ticks);
+                    }
+                }
+
+                if (toRemove.Count > 0)
+                {
+                    foreach (var enumerator in toRemove)
+                    {
+                        collection.Remove(enumerator);
+                    }
+                    toRemove.Clear();
+                }
+
+                frontier = new DateTime(nextFrontierTicks);
+                if (frontier == DateTime.MaxValue)
+                {
+                    break;
+                }
+            }
+        }
+    }
+}

Engine/DataFeeds/Enumerators/SynchronizingSliceEnumerator.cs

@@ -0,0 +1,55 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using QuantConnect.Data;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Represents an enumerator capable of synchronizing other slice enumerators in time.
+    /// This assumes that all enumerators have data time stamped in the same time zone
+    /// </summary>
+    public class SynchronizingSliceEnumerator : SynchronizingEnumerator<Slice>
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingSliceEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        public SynchronizingSliceEnumerator(params IEnumerator<Slice>[] enumerators)
+            : this((IEnumerable<IEnumerator>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SynchronizingSliceEnumerator"/> class
+        /// </summary>
+        /// <param name="enumerators">The enumerators to be synchronized. NOTE: Assumes the same time zone for all data</param>
+        public SynchronizingSliceEnumerator(IEnumerable<IEnumerator> enumerators) : base((IEnumerable<IEnumerator<Slice>>)enumerators)
+        {
+        }
+
+        /// <summary>
+        /// Gets the Timestamp for the data
+        /// </summary>
+        protected override DateTime GetInstanceTime(Slice instance)
+        {
+            return instance.UtcTime;
+        }
+    }
+}