chore: import upstream snapshot with attribution

Tests/Engine/DataFeeds/Enumerators/AuxiliaryDataEnumeratorTests.cs

@@ -0,0 +1,103 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using QuantConnect.Tests.Common.Securities;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class AuxiliaryDataEnumeratorTests
+    {
+        private SubscriptionDataConfig _config;
+        private TestTradableDayNotifier _tradableDayNotifier;
+        private Delisting _delistingEvent;
+
+        [SetUp]
+        public void Setup()
+        {
+            _config = SecurityTests.CreateTradeBarConfig();
+            _tradableDayNotifier = new TestTradableDayNotifier();
+            _tradableDayNotifier.Symbol = _config.Symbol;
+            _delistingEvent = new Delisting(_config.Symbol, new DateTime(2009, 1, 1), 1, DelistingType.Delisted);
+        }
+
+        [Test]
+        public void IsSetToNullIfNoDataAlwaysReturnsTrue()
+        {
+            var eventProvider = new TestableEventProvider();
+            var enumerator = new AuxiliaryDataEnumerator(
+                _config,
+                null,
+                null,
+                new ITradableDateEventProvider[] { eventProvider },
+                _tradableDayNotifier,
+                DateTime.UtcNow
+            );
+
+            eventProvider.Data.Enqueue(_delistingEvent);
+            _tradableDayNotifier.TriggerEvent();
+
+            Assert.Null(enumerator.Current);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.NotNull(enumerator.Current);
+            Assert.AreEqual(_delistingEvent, enumerator.Current);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.Null(enumerator.Current);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.Null(enumerator.Current);
+
+            enumerator.Dispose();
+        }
+    }
+
+    class TestableEventProvider : ITradableDateEventProvider
+    {
+        public readonly Queue<BaseData> Data = new Queue<BaseData>();
+
+        public IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            yield return Data.Dequeue();
+        }
+
+        public void Initialize(SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, IMapFileProvider mapFileProvider, DateTime startTime)
+        {
+        }
+    }
+
+    class TestTradableDayNotifier : ITradableDatesNotifier
+    {
+        public event EventHandler<NewTradableDateEventArgs> NewTradableDate;
+        public DateTime TradableDate { get; set; }
+        public BaseData LastBaseData { get; set; }
+        public Symbol Symbol { get; set; }
+
+        public void TriggerEvent()
+        {
+            NewTradableDate?.Invoke(this, new NewTradableDateEventArgs(TradableDate, LastBaseData, Symbol, null));
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/BaseDataCollectionAggregatorEnumeratorTests.cs

@@ -0,0 +1,58 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class BaseDataCollectionAggregatorEnumeratorTests
+    {
+        [Test]
+        public void AggregatesUntilNull()
+        {
+            var time = new DateTime(2015, 10, 20);
+            var underlying = Enumerable.Range(0, 5).Select(x => new Tick { Time = time }).ToList();
+            underlying.AddRange(new Tick[] { null, null, null });
+
+            var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), Symbols.SPY);
+
+            Assert.IsTrue(aggregator.MoveNext());
+            Assert.IsNotNull(aggregator.Current);
+            Assert.AreEqual(5, aggregator.Current.Data.Count);
+
+            aggregator.Dispose();
+        }
+        [Test]
+        public void AggregatesUntilTimeChange()
+        {
+            var time = new DateTime(2015, 10, 20);
+            var underlying = Enumerable.Range(0, 5).Select(x => new Tick { Time = time }).ToList();
+            underlying.AddRange(Enumerable.Range(0, 5).Select(x => new Tick {Time = time.AddSeconds(1)}));
+
+            var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), Symbols.SPY);
+
+            Assert.IsTrue(aggregator.MoveNext());
+            Assert.IsNotNull(aggregator.Current);
+            Assert.AreEqual(5, aggregator.Current.Data.Count);
+
+            aggregator.Dispose();
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/ConcatEnumeratorTests.cs

@@ -0,0 +1,146 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NUnit.Framework;
+using QuantConnect.Data;
+using System.Collections;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class ConcatEnumeratorTests
+    {
+        [TestCase(true)]
+        [TestCase(false)]
+        public void SkipsBasedOnEndTime(bool skipsBasedOnEndTime)
+        {
+            var time = new DateTime(2020, 1, 1);
+            var enumerator1 = new List<BaseData> { new Tick(time, Symbols.SPY, 10, 10) }.GetEnumerator();
+            var enumerator2 = new List<BaseData>
+            {
+                new Tick(time.AddSeconds(-1), Symbols.SPY, 20, 20), //should be skipped because end time is before previous tick
+                new Tick(time.AddSeconds(1), Symbols.SPY, 30 , 30)
+            }.GetEnumerator();
+
+            var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, enumerator2);
+
+            Assert.IsTrue(concat.MoveNext());
+            Assert.AreEqual(10, (concat.Current as Tick).AskPrice);
+
+            if (!skipsBasedOnEndTime)
+            {
+                Assert.IsTrue(concat.MoveNext());
+                Assert.AreEqual(20, (concat.Current as Tick).AskPrice);
+            }
+
+            Assert.IsTrue(concat.MoveNext());
+            Assert.AreEqual(30, (concat.Current as Tick).AskPrice);
+
+            Assert.IsFalse(concat.MoveNext());
+            Assert.IsNull(concat.Current);
+
+            concat.Dispose();
+        }
+
+        [TestCase(true)]
+        [TestCase(false)]
+        public void EmptyNullEnumerators(bool skipsBasedOnEndTime)
+        {
+            var time = new DateTime(2020, 1, 1);
+            // empty enumerators
+            var enumerator1 = new List<BaseData>().GetEnumerator();
+            var enumerator2 = new List<BaseData>().GetEnumerator();
+
+            var enumerator3 = new List<BaseData>
+            {
+                new Tick(time, Symbols.SPY, 10, 10),
+                new Tick(time.AddSeconds(-1), Symbols.SPY, 20, 20),
+                new Tick(time.AddSeconds(1), Symbols.SPY, 30 , 30)
+            }.GetEnumerator();
+
+            var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, null, enumerator2, enumerator3);
+
+            Assert.IsTrue(concat.MoveNext());
+            Assert.AreEqual(10, (concat.Current as Tick).AskPrice);
+
+            Assert.IsTrue(concat.MoveNext());
+            Assert.AreEqual(20, (concat.Current as Tick).AskPrice);
+
+            Assert.IsTrue(concat.MoveNext());
+            Assert.AreEqual(30, (concat.Current as Tick).AskPrice);
+
+            Assert.IsFalse(concat.MoveNext());
+            Assert.IsNull(concat.Current);
+
+            concat.Dispose();
+        }
+
+        [TestCase(true)]
+        [TestCase(false)]
+        public void DropsEnumeratorsReturningNullAndTrue(bool skipsBasedOnEndTime)
+        {
+            var enumerator1 = new TestEnumerator();
+            var enumerator2 = new TestEnumerator();
+
+            var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, null, enumerator2);
+
+            Assert.IsTrue(concat.MoveNext());
+
+            Assert.IsNull(concat.Current);
+            Assert.IsTrue(enumerator1.Disposed);
+            Assert.IsFalse(enumerator2.Disposed);
+            Assert.AreEqual(1, enumerator2.MoveNextCount);
+
+            Assert.IsTrue(concat.MoveNext());
+
+            // we assert it just keeps the last enumerator and drops the rest
+            Assert.IsTrue(enumerator1.Disposed);
+            Assert.IsFalse(enumerator2.Disposed);
+            Assert.IsNull(concat.Current);
+            Assert.AreEqual(2, enumerator2.MoveNextCount);
+
+            concat.Dispose();
+        }
+
+        private class TestEnumerator : IEnumerator<BaseData>
+        {
+            public bool Disposed { get; private set; }
+            public int MoveNextCount { get; private set; }
+            public BaseData Current => null;
+
+            object IEnumerator.Current => null;
+
+            public void Dispose()
+            {
+                Disposed = true;
+            }
+
+            public bool MoveNext()
+            {
+                MoveNextCount++;
+                return true;
+            }
+
+            public void Reset()
+            {
+            }
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/DelistingEnumeratorTests.cs

@@ -0,0 +1,245 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class DelistingEnumeratorTests
+    {
+        private SubscriptionDataConfig _config;
+
+        [SetUp]
+        public void SetUp()
+        {
+            var symbol = Symbol.CreateFuture("ASD", Market.USA, new DateTime(2018, 01, 01));
+
+            _config = new SubscriptionDataConfig(typeof(TradeBar),
+                symbol,
+                Resolution.Daily,
+                TimeZones.NewYork,
+                TimeZones.NewYork,
+                true,
+                true,
+                false);
+        }
+
+        [Test]
+        public void EmitsBothEventsIfDateIsPastDelisted()
+        {
+            var eventProvider = new DelistingEventProvider();
+            eventProvider.Initialize(_config,
+                null,
+                null,
+                DateTime.UtcNow);
+
+            var enumerator = eventProvider.GetEvents(
+                new NewTradableDateEventArgs(
+                    DateTime.UtcNow,
+                    new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
+                    _config.Symbol,
+                    null
+                )
+            ).GetEnumerator();
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current as Delisting);
+            Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
+            Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(_config.Symbol.ID.Date, (enumerator.Current as Delisting).Time.Date);
+            Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current as Delisting);
+            Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
+            Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(_config.Symbol.ID.Date.AddDays(1), (enumerator.Current as Delisting).Time.Date);
+            Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
+
+            Assert.IsFalse(enumerator.MoveNext());
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void EmitsWarningAsOffDelistingDate()
+        {
+            var eventProvider = new DelistingEventProvider();
+            eventProvider.Initialize(_config,
+                null,
+                null,
+                DateTime.UtcNow);
+
+            // should NOT emit
+            var enumerator = eventProvider.GetEvents(
+                new NewTradableDateEventArgs(
+                    _config.Symbol.ID.Date.Subtract(TimeSpan.FromMinutes(1)),
+                    new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
+                    _config.Symbol,
+                    null
+                )
+            ).GetEnumerator();
+
+            Assert.IsFalse(enumerator.MoveNext());
+
+            // should emit
+            enumerator = eventProvider.GetEvents(
+                new NewTradableDateEventArgs(
+                    _config.Symbol.ID.Date,
+                    new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
+                    _config.Symbol,
+                    null
+                )
+            ).GetEnumerator();
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current as Delisting);
+            Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
+            Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(_config.Symbol.ID.Date, (enumerator.Current as Delisting).Time.Date);
+            Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
+
+            Assert.IsFalse(enumerator.MoveNext());
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void EmitsDelistedAfterDelistingDate()
+        {
+            var eventProvider = new DelistingEventProvider();
+            eventProvider.Initialize(_config,
+                null,
+                null,
+                DateTime.UtcNow);
+
+            // should emit warning
+            var enumerator = eventProvider.GetEvents(
+                new NewTradableDateEventArgs(
+                    _config.Symbol.ID.Date,
+                    new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
+                    _config.Symbol,
+                    null
+                )
+            ).GetEnumerator();
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current as Delisting);
+            Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
+
+            // should NOT emit if not AFTER delisting date
+            enumerator = eventProvider.GetEvents(
+                new NewTradableDateEventArgs(
+                    _config.Symbol.ID.Date,
+                    new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
+                    _config.Symbol,
+                    null
+                )
+            ).GetEnumerator();
+            Assert.IsFalse(enumerator.MoveNext());
+
+            // should emit AFTER delisting date
+            enumerator = eventProvider.GetEvents(
+                new NewTradableDateEventArgs(
+                    _config.Symbol.ID.Date.AddMinutes(1),
+                    new Tick(DateTime.UtcNow, _config.Symbol, 10, 5),
+                    _config.Symbol,
+                    null
+                )
+            ).GetEnumerator();
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current as Delisting);
+            Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
+            Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(_config.Symbol.ID.Date.AddDays(1), (enumerator.Current as Delisting).Time.Date);
+            Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
+
+            Assert.IsFalse(enumerator.MoveNext());
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void EmitsDelistedWarningOnNonTradableDay()
+        {
+            // Unit test to simulate and reproduce #5545
+
+            // Give us two tradable days before and after expiration
+            var tradableDays = new List<DateTime> { new DateTime(2021, 01, 01), new DateTime(2021, 01, 04) };
+
+            // Set expiration as 1/2/21 a Saturday, not included in our tradble days
+            var expiration = new DateTime(2021, 01, 02);
+            var symbol = Symbol.CreateFuture("ASD", Market.USA, expiration);
+            var config = new SubscriptionDataConfig(typeof(TradeBar),
+                symbol,
+                Resolution.Daily,
+                TimeZones.NewYork,
+                TimeZones.NewYork,
+                true,
+                true,
+                false);
+            
+            var eventProvider = new DelistingEventProvider();
+            eventProvider.Initialize(config,
+                null,
+                null,
+                DateTime.UtcNow);
+
+            var tradableDateEvents = tradableDays.Select(day => new NewTradableDateEventArgs(
+                day,
+                new Tick(day, config.Symbol, 10, 5),
+                config.Symbol,
+                null
+            )).GetEnumerator();
+
+            // Pass in the day before expiration should be nothing
+            tradableDateEvents.MoveNext();
+            var enumerator = eventProvider.GetEvents(tradableDateEvents.Current).GetEnumerator();
+            Assert.IsFalse(enumerator.MoveNext());
+
+            // Pass in the following monday, should be both but warning first and still scheduled for saturday
+            tradableDateEvents.MoveNext();
+            enumerator = eventProvider.GetEvents(tradableDateEvents.Current).GetEnumerator();
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current as Delisting);
+            Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
+            Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(config.Symbol.ID.Date, (enumerator.Current as Delisting).Time.Date);
+            Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current as Delisting);
+            Assert.AreEqual(MarketDataType.Auxiliary, enumerator.Current.DataType);
+            Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(config.Symbol.ID.Date.AddDays(1), (enumerator.Current as Delisting).Time.Date);
+            Assert.AreEqual(7.5, (enumerator.Current as Delisting).Price);
+
+            Assert.IsFalse(enumerator.MoveNext());
+
+            tradableDateEvents.Dispose();
+            enumerator.Dispose();
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/DividendEventProviderTests.cs

@@ -0,0 +1,137 @@
+/*
+* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using System.Globalization;
+using QuantConnect.Data.Market;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class DividendEventProviderTests
+    {
+        // From https://www.nasdaq.com/market-activity/stocks/aapl/dividend-history
+        [TestCase("20121106", 2.65)]
+        [TestCase("20130206", 2.65)]
+        [TestCase("20130508", 3.05)]
+        [TestCase("20130807", 3.05)]
+        [TestCase("20131105", 3.05)]
+        [TestCase("20140205", 3.05)]
+        [TestCase("20140507", 3.29)]
+        [TestCase("20140806", 0.47)]
+        [TestCase("20141105", 0.47)]
+        [TestCase("20150204", 0.47)]
+        [TestCase("20150506", 0.52)]
+        [TestCase("20150805", 0.52)]
+        [TestCase("20151104", 0.52)]
+        [TestCase("20160203", 0.52)]
+        [TestCase("20160504", 0.57)]
+        [TestCase("20160803", 0.57)]
+        [TestCase("20161102", 0.57)]
+        [TestCase("20170208", 0.57)]
+        [TestCase("20170510", 0.63)]
+        [TestCase("20170809", 0.63)]
+        [TestCase("20171109", 0.63)]
+        [TestCase("20180208", 0.63)]
+        [TestCase("20180510", 0.73)]
+        public void DividendsDistribution(string exDividendDateStr, decimal expectedDistribution)
+        {
+            var dividendProvider = new DividendEventProvider();
+            var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork,
+                false, false, false);
+
+            var start = new DateTime(1998, 01, 02);
+            dividendProvider.Initialize(config, TestGlobals.FactorFileProvider, TestGlobals.MapFileProvider, start);
+
+            var exDividendDate = DateTime.ParseExact(exDividendDateStr, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
+
+            var events = dividendProvider
+                .GetEvents(new NewTradableDateEventArgs(exDividendDate, null, Symbols.AAPL, null))
+                .ToList();
+            // ex dividend date does not emit anything
+            Assert.AreEqual(0, events.Count);
+
+            events = dividendProvider
+                .GetEvents(new NewTradableDateEventArgs(exDividendDate.AddDays(1), null, Symbols.AAPL, null))
+                .ToList();
+
+            Assert.AreEqual(1, events.Count);
+            var dividend = events[0] as Dividend;
+            Assert.IsNotNull(dividend);
+
+            Assert.AreEqual(expectedDistribution, dividend.Distribution);
+        }
+
+        [Test]
+        public void ThrowsWhenEmptyReferencePrice()
+        {
+            var dividendProvider = new DividendEventProvider();
+            var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork,
+                false, false, false);
+            var start = new DateTime(1998, 01, 02);
+            var row1 = new DateTime(2000, 01, 02);
+            var row2 = new DateTime(2001, 01, 02);
+            var row3 = new DateTime(2002, 01, 02);
+
+            var factorFileProvider = new TestFactorFileProvider
+            {
+                FactorFile = new CorporateFactorProvider("AAPL", new []
+                {
+                    new CorporateFactorRow(row1, 0.693m, 1),
+                    new CorporateFactorRow(row2, 0.77m, 1),
+                    new CorporateFactorRow(row3, 0.85555m, 1)
+                }, start)
+            };
+
+            dividendProvider.Initialize(config, factorFileProvider, TestGlobals.MapFileProvider, start);
+
+            foreach (var row in factorFileProvider.FactorFile.Take(1))
+            {
+                var lastRawPrice = 100;
+                var events = dividendProvider
+                    .GetEvents(new NewTradableDateEventArgs(row.Date, null, Symbols.AAPL, lastRawPrice))
+                    .ToList();
+                // ex dividend date does not emit anything
+                Assert.AreEqual(0, events.Count);
+
+                Assert.Throws<InvalidOperationException>(() =>
+                {
+                    dividendProvider
+                        .GetEvents(new NewTradableDateEventArgs(row.Date.AddDays(1), null, Symbols.AAPL, lastRawPrice))
+                        .ToList();
+                });
+            }
+        }
+
+        private class TestFactorFileProvider : IFactorFileProvider
+        {
+            public CorporateFactorProvider FactorFile { get; set; }
+            public void Initialize(IMapFileProvider mapFileProvider, IDataProvider dataProvider)
+            {
+            }
+
+            public IFactorProvider Get(Symbol symbol)
+            {
+                return FactorFile;
+            }
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/EnqueableEnumeratorTests.cs

@@ -0,0 +1,136 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using NUnit.Framework;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class EnqueableEnumeratorTests
+    {
+        [Test]
+        public void PassesTicksStraightThrough()
+        {
+            var enumerator = new EnqueueableEnumerator<Tick>();
+
+            // add some ticks
+            var currentTime = new DateTime(2015, 10, 08);
+
+            // returns true even if no data present until stop is called
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) {Quantity = 10};
+            enumerator.Enqueue(tick1);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(tick1, enumerator.Current);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick2 = new Tick(currentTime, Symbols.SPY, 199.56m, 199.21m, 200.02m) {Quantity = 5};
+            enumerator.Enqueue(tick2);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(tick2, enumerator.Current);
+
+            enumerator.Stop();
+
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void RecordsInternalQueueCount()
+        {
+            var enumerator = new EnqueueableEnumerator<Tick>();
+
+            var currentTime = new DateTime(2015, 12, 01);
+            var tick = new Tick(currentTime, Symbols.SPY, 100, 101);
+            enumerator.Enqueue(tick);
+            Assert.AreEqual(1, enumerator.Count);
+
+            tick = new Tick(currentTime, Symbols.SPY, 100, 101);
+            enumerator.Enqueue(tick);
+            Assert.AreEqual(2, enumerator.Count);
+
+            enumerator.MoveNext();
+            Assert.AreEqual(1, enumerator.Count);
+
+            enumerator.MoveNext();
+            Assert.AreEqual(0, enumerator.Count);
+
+            enumerator.Dispose();
+        }
+
+        [Test, Category("TravisExclude")]
+        public void MoveNextBlocks()
+        {
+            using var finished = new ManualResetEvent(false);
+            var enumerator = new EnqueueableEnumerator<Tick>(true);
+
+            // producer
+            int count = 0;
+            Task.Run(() =>
+            {
+                while (!finished.WaitOne(TimeSpan.FromMilliseconds(50)))
+                {
+                    enumerator.Enqueue(new Tick(DateTime.Now, Symbols.SPY, 100, 101));
+                    count++;
+
+                    // 5 data points is plenty
+                    if (count > 5)
+                    {
+                        finished.Set();
+                        enumerator.Stop();
+                    }
+                }
+            });
+
+            // consumer
+            int dequeuedCount = 0;
+            bool encounteredError = false;
+            using var consumerTaskFinished = new ManualResetEvent(false);
+            Task.Run(() =>
+            {
+                while (enumerator.MoveNext())
+                {
+                    dequeuedCount++;
+                    if (enumerator.Current == null)
+                    {
+                        encounteredError = true;
+                    }
+                }
+                consumerTaskFinished.Set();
+            });
+
+            finished.WaitOne(Timeout.Infinite);
+            consumerTaskFinished.WaitOne(Timeout.Infinite);
+
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsFalse(encounteredError);
+            Assert.AreEqual(count, dequeuedCount);
+
+            enumerator.Dispose();
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataCollectionSubscriptionEnumeratorFactoryTests.cs

@@ -0,0 +1,127 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using NUnit.Framework;
+using QuantConnect.Brokerages;
+using QuantConnect.Data;
+using QuantConnect.Data.Fundamental;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
+{
+    [TestFixture]
+    public class BaseDataCollectionSubscriptionEnumeratorFactoryTests
+    {
+        // This test reports higher memory usage when ran with Travis, so we exclude it for now
+        [Test, Category("TravisExclude")]
+        public void DoesNotLeakMemory()
+        {
+            var symbolFactory = new FundamentalUniverse();
+            var symbol = symbolFactory.UniverseSymbol();
+            var config = new SubscriptionDataConfig(typeof(FundamentalUniverse), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
+            var security = new Security(
+                SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
+                config,
+                new Cash(Currencies.USD, 0, 1),
+                SymbolProperties.GetDefault(Currencies.USD),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCache()
+            );
+
+            var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
+            var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
+            using var universe = new CoarseFundamentalUniverse(universeSettings, x => new List<Symbol>{ Symbols.AAPL });
+
+            var factory = new BaseDataCollectionSubscriptionEnumeratorFactory(null);
+
+            GC.Collect();
+            var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;
+
+            var date = new DateTime(2014, 3, 25);
+
+            const int iterations = 1000;
+            for (var i = 0; i < iterations; i++)
+            {
+                var request = new SubscriptionRequest(true, universe, security, config, date, date);
+                using (var enumerator = factory.CreateEnumerator(request, TestGlobals.DataProvider))
+                {
+                    enumerator.MoveNext();
+                }
+            }
+
+            GC.Collect();
+            var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed;
+
+            Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB");
+
+            Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10);
+        }
+
+        [Test]
+        public void ReturnsExpectedTimestamps()
+        {
+            var symbolFactory = new FundamentalUniverse();
+            var symbol = symbolFactory.UniverseSymbol();
+            var config = new SubscriptionDataConfig(typeof(FundamentalUniverse), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
+            var security = new Security(
+                SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
+                config,
+                new Cash(Currencies.USD, 0, 1),
+                SymbolProperties.GetDefault(Currencies.USD),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCache()
+            );
+
+            var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
+            var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
+            using var universe = new CoarseFundamentalUniverse(universeSettings, x => new List<Symbol> { Symbols.AAPL });
+
+            var factory = new BaseDataCollectionSubscriptionEnumeratorFactory(null);
+
+            var dateStart = new DateTime(2014, 3, 26);
+            var dateEnd = new DateTime(2014, 3, 27);
+            var days = (dateEnd - dateStart).Days + 1;
+
+            var request = new SubscriptionRequest(true, universe, security, config, dateStart, dateEnd);
+
+            using (var enumerator = factory.CreateEnumerator(request, TestGlobals.DataProvider))
+            {
+                for (var i = 0; i < days; i++)
+                {
+                    Assert.IsTrue(enumerator.MoveNext());
+
+                    var current = enumerator.Current as BaseDataCollection;
+                    Assert.IsNotNull(current);
+                    Assert.AreEqual(dateStart.AddDays(i), current.Time);
+                    Assert.AreEqual(dateStart.AddDays(i), current.EndTime);
+                    Assert.AreEqual(dateStart.AddDays(i - 1), current.Data[0].Time);
+                    Assert.AreEqual(dateStart.AddDays(i), current.Data[0].EndTime);
+                }
+
+                Assert.IsFalse(enumerator.MoveNext());
+                Assert.IsNotNull(enumerator.Current);
+            }
+        }
+    }
+}
+;

Tests/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactoryTests.cs

@@ -0,0 +1,613 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Moq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
+using QuantConnect.Securities;
+using QuantConnect.Securities.Equity;
+using QuantConnect.Util;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
+{
+    [TestFixture]
+    public class LiveCustomDataSubscriptionEnumeratorFactoryTests
+    {
+        [TestFixture]
+        public class WhenCreatingEnumeratorForRestData
+        {
+            private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
+            private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
+
+            private ManualTimeProvider _timeProvider;
+            private IEnumerator<BaseData> _enumerator;
+            private Mock<ISubscriptionDataSourceReader> _dataSourceReader;
+
+            [SetUp]
+            public void Given()
+            {
+                _timeProvider = new ManualTimeProvider(_referenceUtc);
+
+                _dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
+                _dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
+                        sds.Source == "rest.source" &&
+                        sds.TransportMedium == SubscriptionTransportMedium.Rest &&
+                        sds.Format == FileFormat.Csv))
+                    )
+                    .Returns(Enumerable.Range(0, 100)
+                        .Select(i => new RestData
+                        {
+                            EndTime = _referenceLocal.AddSeconds(i)
+                        }))
+                        .Verifiable();
+
+                var config = new SubscriptionDataConfig(typeof(RestData), Symbols.SPY, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
+                var request = GetSubscriptionRequest(config, _referenceUtc.AddSeconds(-1), _referenceUtc.AddDays(1));
+
+                var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _dataSourceReader.Object);
+                _enumerator = factory.CreateEnumerator(request, null);
+            }
+
+            [TearDown]
+            public void TearDown()
+            {
+                _enumerator?.DisposeSafely();
+            }
+
+            [Test]
+            public void YieldsDataEachSecondAsTimePasses()
+            {
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNotNull(_enumerator.Current);
+                Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+
+                _timeProvider.AdvanceSeconds(1);
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNotNull(_enumerator.Current);
+                Assert.AreEqual(_referenceLocal.AddSeconds(1), _enumerator.Current.EndTime);
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+
+                VerifyGetSourceInvocationCount(_dataSourceReader, 1, "rest.source", SubscriptionTransportMedium.Rest, FileFormat.Csv);
+            }
+        }
+
+        [TestFixture]
+        public class WhenCreatingEnumeratorForRestCollectionData
+        {
+            private const int DataPerTimeStep = 3;
+            private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
+            private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
+
+            private ManualTimeProvider _timeProvider;
+            private IEnumerator<BaseData> _enumerator;
+            private Mock<ISubscriptionDataSourceReader> _dataSourceReader;
+
+            [SetUp]
+            public void Given()
+            {
+                _timeProvider = new ManualTimeProvider(_referenceUtc);
+
+                _dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
+                _dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
+                        sds.Source == "rest.collection.source" &&
+                        sds.TransportMedium == SubscriptionTransportMedium.Rest &&
+                        sds.Format == FileFormat.UnfoldingCollection))
+                    )
+                    .Returns(Enumerable.Range(0, 100)
+                        .Select(i => new BaseDataCollection(_referenceLocal.AddSeconds(i), Symbols.SPY, Enumerable.Range(0, DataPerTimeStep)
+                            .Select(_ => new RestCollectionData {EndTime = _referenceLocal.AddSeconds(i)})))
+                    )
+                    .Verifiable();
+
+                var config = new SubscriptionDataConfig(typeof(RestCollectionData), Symbols.SPY, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
+                var request = GetSubscriptionRequest(config, _referenceUtc.AddSeconds(-4), _referenceUtc.AddDays(1));
+
+                var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _dataSourceReader.Object);
+                _enumerator = factory.CreateEnumerator(request, null);
+            }
+
+            [TearDown]
+            public void TearDown()
+            {
+                _enumerator?.DisposeSafely();
+            }
+
+            [Test]
+            public void YieldsGroupOfDataEachSecond()
+            {
+                for (int i = 0; i < DataPerTimeStep; i++)
+                {
+                    Assert.IsTrue(_enumerator.MoveNext());
+                    Assert.IsNotNull(_enumerator.Current, $"Index {i} is null.");
+                    Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
+                }
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+
+                _timeProvider.AdvanceSeconds(1);
+
+                for (int i = 0; i < DataPerTimeStep; i++)
+                {
+                    Assert.IsTrue(_enumerator.MoveNext());
+                    Assert.IsNotNull(_enumerator.Current);
+                    Assert.AreEqual(_referenceLocal.AddSeconds(1), _enumerator.Current.EndTime);
+                }
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+
+                VerifyGetSourceInvocationCount(_dataSourceReader, 1, "rest.collection.source", SubscriptionTransportMedium.Rest, FileFormat.UnfoldingCollection);
+            }
+        }
+
+        [TestFixture]
+        public class WhenCreatingEnumeratorForRemoteCollectionData
+        {
+            private const int DataPerTimeStep = 3;
+            private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
+            private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
+
+            private ManualTimeProvider _timeProvider;
+            private IEnumerator<BaseData> _enumerator;
+
+            [SetUp]
+            public void Given()
+            {
+                _timeProvider = new ManualTimeProvider(_referenceUtc);
+                var dataSourceReader = new TestISubscriptionDataSourceReader
+                {
+                    TimeProvider = _timeProvider
+                };
+
+                var config = new SubscriptionDataConfig(typeof(RemoteCollectionData), Symbols.SPY, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
+                var request = GetSubscriptionRequest(config, _referenceUtc.AddSeconds(-4), _referenceUtc.AddDays(1));
+
+                var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, dataSourceReader);
+                _enumerator = factory.CreateEnumerator(request, null);
+            }
+
+            private class TestISubscriptionDataSourceReader : ISubscriptionDataSourceReader
+            {
+                public ManualTimeProvider TimeProvider;
+                public event EventHandler<InvalidSourceEventArgs> InvalidSource;
+
+                public IEnumerable<BaseData> Read(SubscriptionDataSource source)
+                {
+                    var currentLocalTime = TimeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
+                    var data = Enumerable.Range(0, DataPerTimeStep).Select(_ => new RemoteCollectionData { EndTime = currentLocalTime });
+
+                    // let's add some old data which should be ignored
+                    data = data.Concat(Enumerable.Range(0, DataPerTimeStep).Select(_ => new RemoteCollectionData { EndTime = currentLocalTime.AddSeconds(-1) }));
+                    return new BaseDataCollection(currentLocalTime, Symbols.SPY, data);
+                }
+            }
+
+            [TearDown]
+            public void TearDown()
+            {
+                _enumerator?.DisposeSafely();
+            }
+
+            [Test]
+            public void YieldsGroupOfDataEachSecond()
+            {
+                for (int i = 0; i < DataPerTimeStep; i++)
+                {
+                    Assert.IsTrue(_enumerator.MoveNext());
+                    Assert.IsNotNull(_enumerator.Current, $"Index {i} is null.");
+                    Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
+                }
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+
+                _timeProvider.AdvanceSeconds(1);
+
+                for (int i = 0; i < DataPerTimeStep; i++)
+                {
+                    Assert.IsTrue(_enumerator.MoveNext());
+                    Assert.IsNotNull(_enumerator.Current);
+                    Assert.AreEqual(_referenceLocal.AddSeconds(1), _enumerator.Current.EndTime);
+                }
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+            }
+        }
+
+        [TestFixture]
+        public class WhenCreatingEnumeratorForSecondRemoteFileData
+        {
+            private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
+            private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
+
+            private ManualTimeProvider _timeProvider;
+            private IEnumerator<BaseData> _enumerator;
+            private Mock<ISubscriptionDataSourceReader> _dataSourceReader;
+
+            [SetUp]
+            public void Given()
+            {
+                _timeProvider = new ManualTimeProvider(_referenceUtc);
+
+                _dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
+                _dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
+                        sds.Source == "remote.file.source" &&
+                        sds.TransportMedium == SubscriptionTransportMedium.RemoteFile &&
+                        sds.Format == FileFormat.Csv))
+                    )
+                    .Returns(Enumerable.Range(0, 100)
+                        .Select(i => new RemoteFileData
+                        {
+                            // include past data
+                            EndTime = _referenceLocal.AddSeconds(i - 95)
+                        }))
+                    .Verifiable();
+
+                var config = new SubscriptionDataConfig(typeof(RemoteFileData), Symbols.SPY, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
+                var request = GetSubscriptionRequest(config, _referenceUtc.AddSeconds(-6), _referenceUtc.AddDays(1));
+
+                var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _dataSourceReader.Object);
+                _enumerator = factory.CreateEnumerator(request, null);
+            }
+
+            [TearDown]
+            public void TearDown()
+            {
+                _enumerator?.DisposeSafely();
+            }
+
+            [Test]
+            public void YieldsDataEachSecondAsTimePasses()
+            {
+                // most recent 5 seconds of data
+                for (int i = 5; i > 0; i--)
+                {
+                    Assert.IsTrue(_enumerator.MoveNext());
+                    Assert.IsNotNull(_enumerator.Current);
+                    Assert.AreEqual(_referenceLocal.AddSeconds(-i), _enumerator.Current.EndTime);
+                }
+
+                // first data point
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNotNull(_enumerator.Current);
+                Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
+
+                _timeProvider.AdvanceSeconds(1);
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNotNull(_enumerator.Current);
+                Assert.AreEqual(_referenceLocal.AddSeconds(1), _enumerator.Current.EndTime);
+
+                VerifyGetSourceInvocationCount(_dataSourceReader, 1, "remote.file.source", SubscriptionTransportMedium.RemoteFile, FileFormat.Csv);
+            }
+        }
+
+        [TestFixture]
+        public class WhenCreatingEnumeratorForDailyRemoteFileData
+        {
+            private int _dataPointsAfterReference = 1;
+            private readonly DateTime _referenceLocal = new DateTime(2017, 10, 12);
+            private readonly DateTime _referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
+
+            private ManualTimeProvider _timeProvider;
+            private IEnumerator<BaseData> _enumerator;
+            private Mock<ISubscriptionDataSourceReader> _dataSourceReader;
+
+            [SetUp]
+            public void Given()
+            {
+                _timeProvider = new ManualTimeProvider(_referenceUtc);
+
+                _dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
+                _dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
+                        sds.Source == "remote.file.source" &&
+                        sds.TransportMedium == SubscriptionTransportMedium.RemoteFile &&
+                        sds.Format == FileFormat.Csv))
+                    )
+                    .Returns(() => Enumerable.Range(0, 100)
+                        .Select(i => new RemoteFileData
+                        {
+                            // include past data
+                            EndTime = _referenceLocal.Add(TimeSpan.FromDays(i - (100 - _dataPointsAfterReference - 1)))
+                        }))
+                    .Verifiable();
+
+                var config = new SubscriptionDataConfig(typeof(RemoteFileData), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
+                var request = GetSubscriptionRequest(config, _referenceUtc.AddDays(-2), _referenceUtc.AddDays(1));
+
+                var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, _dataSourceReader.Object);
+                _enumerator = factory.CreateEnumerator(request, null);
+            }
+
+            [TearDown]
+            public void TearDown()
+            {
+                _enumerator?.DisposeSafely();
+            }
+
+            [Test]
+            public void YieldsDataEachDayAsTimePasses()
+            {
+                // previous point is exactly one resolution step behind, so it emits
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNotNull(_enumerator.Current);
+                Assert.AreEqual(_referenceLocal.AddDays(-1), _enumerator.Current.EndTime);
+                VerifyGetSourceInvocation(1);
+
+                // yields the data for the current time
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNotNull(_enumerator.Current);
+                Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime);
+                VerifyGetSourceInvocation(0);
+
+                _timeProvider.Advance(Time.OneDay);
+
+                // now we can yield the next data point as it has passed frontier time
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNotNull(_enumerator.Current);
+                Assert.AreEqual(_referenceLocal.AddDays(1), _enumerator.Current.EndTime);
+                VerifyGetSourceInvocation(0);
+
+                // this call exhaused the enumerator stack and yields a null result
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+                VerifyGetSourceInvocation(0);
+
+                // this call refrshes the enumerator stack but finds no data ahead of the frontier
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+                VerifyGetSourceInvocation(1);
+
+                _timeProvider.Advance(TimeSpan.FromMinutes(30));
+
+                // time advances 30 minutes so we'll try to refresh again
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+                VerifyGetSourceInvocation(1);
+
+                _timeProvider.Advance(Time.OneDay);
+
+                // now to the next day, we'll try again and get data
+                _dataPointsAfterReference++;
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNotNull(_enumerator.Current);
+                Assert.AreEqual(_referenceLocal.AddDays(2), _enumerator.Current.EndTime);
+                VerifyGetSourceInvocation(1);
+
+                _timeProvider.Advance(TimeSpan.FromHours(1));
+
+                // out of data
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+                VerifyGetSourceInvocation(0);
+
+                _timeProvider.Advance(TimeSpan.FromHours(1));
+
+                // time advanced so we'll try to refresh the souce again, but exhaust the stack because no data
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+                VerifyGetSourceInvocation(1);
+
+                // move forward to next whole day, midnight
+                _timeProvider.Advance(Time.OneDay.Subtract(TimeSpan.FromHours(2.5)));
+
+                // the day elapsed but there's still no data available
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+                VerifyGetSourceInvocation(1);
+
+                // this is rate limited by the 30 minute guard for daily data
+                _timeProvider.Advance(TimeSpan.FromMinutes(29));
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+                VerifyGetSourceInvocation(0);
+
+                // another 30 minutes elapsed and now there's data available
+                _dataPointsAfterReference++;
+                _timeProvider.Advance(TimeSpan.FromMinutes(1));
+
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNotNull(_enumerator.Current);
+                Assert.AreEqual(_referenceLocal.AddDays(3), _enumerator.Current.EndTime);
+                VerifyGetSourceInvocation(1);
+
+                // exhausted the stack
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+                VerifyGetSourceInvocation(0);
+
+                // rate limited
+                Assert.IsTrue(_enumerator.MoveNext());
+                Assert.IsNull(_enumerator.Current);
+                VerifyGetSourceInvocation(0);
+            }
+
+            private int _runningCount;
+            private void VerifyGetSourceInvocation(int count)
+            {
+                _runningCount += count;
+                VerifyGetSourceInvocationCount(_dataSourceReader, _runningCount, "remote.file.source", SubscriptionTransportMedium.RemoteFile, FileFormat.Csv);
+            }
+        }
+
+        [TestCase(10)]
+        [TestCase(60)]
+        [TestCase(0)]
+        public void AllowsSpecifyingIntervalCheck(int intervalCheck)
+        {
+            var referenceLocal = new DateTime(2017, 10, 12);
+            var referenceUtc = new DateTime(2017, 10, 12).ConvertToUtc(TimeZones.NewYork);
+
+            var timeProvider = new ManualTimeProvider(referenceUtc);
+
+            var callCount = 0;
+            var dataSourceReader = new Mock<ISubscriptionDataSourceReader>();
+            dataSourceReader.Setup(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
+                    sds.Source == "local.file.source" &&
+                    sds.TransportMedium == SubscriptionTransportMedium.LocalFile &&
+                    sds.Format == FileFormat.Csv))
+                )
+                .Returns(() => new []{ new LocalFileData { EndTime = referenceLocal.AddSeconds(++callCount) } })
+                .Verifiable();
+
+            var config = new SubscriptionDataConfig(typeof(LocalFileData), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
+            var request = GetSubscriptionRequest(config, referenceUtc.AddSeconds(-1), referenceUtc.AddDays(1));
+
+            var intervalCalls = intervalCheck == 0 ? (TimeSpan?) null : TimeSpan.FromMinutes(intervalCheck);
+
+            var factory = new TestableLiveCustomDataSubscriptionEnumeratorFactory(timeProvider, dataSourceReader.Object, intervalCalls);
+            var enumerator = factory.CreateEnumerator(request, null);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current);
+            Assert.AreEqual(referenceLocal.AddSeconds(callCount), enumerator.Current.EndTime);
+
+            VerifyGetSourceInvocationCount(dataSourceReader, 1, "local.file.source", SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
+
+            // time didn't pass so should refresh
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+            VerifyGetSourceInvocationCount(dataSourceReader, 1, "local.file.source", SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
+
+            var expectedInterval = intervalCalls ?? TimeSpan.FromMinutes(30);
+
+            timeProvider.Advance(expectedInterval.Add(-TimeSpan.FromSeconds(2)));
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+            VerifyGetSourceInvocationCount(dataSourceReader, 1, "local.file.source", SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
+
+            timeProvider.Advance(TimeSpan.FromSeconds(2));
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current);
+            Assert.AreEqual(referenceLocal.AddSeconds(callCount), enumerator.Current.EndTime);
+            VerifyGetSourceInvocationCount(dataSourceReader, 2, "local.file.source", SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
+        }
+
+        private static void VerifyGetSourceInvocationCount(Mock<ISubscriptionDataSourceReader> dataSourceReader, int count, string source, SubscriptionTransportMedium medium, FileFormat fileFormat)
+        {
+            dataSourceReader.Verify(dsr => dsr.Read(It.Is<SubscriptionDataSource>(sds =>
+                sds.Source == source && sds.TransportMedium == medium && sds.Format == fileFormat)), Times.Exactly(count));
+        }
+
+        private static SubscriptionRequest GetSubscriptionRequest(SubscriptionDataConfig config, DateTime startTime, DateTime endTime)
+        {
+            var quoteCurrency = new Cash(Currencies.USD, 0, 1);
+            var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, Symbols.SPY, SecurityType.Equity);
+            var security = new Equity(
+                Symbols.SPY,
+                exchangeHours,
+                quoteCurrency,
+                SymbolProperties.GetDefault(Currencies.USD),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCache()
+            );
+            return new SubscriptionRequest(false, null, security, config, startTime, endTime);
+        }
+
+
+        class RestData : BaseData
+        {
+            public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+            {
+                return new SubscriptionDataSource("rest.source", SubscriptionTransportMedium.Rest);
+            }
+        }
+
+        class RemoteCollectionData : BaseData
+        {
+            public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+            {
+                return new SubscriptionDataSource("remote.collection.source", SubscriptionTransportMedium.RemoteFile, FileFormat.UnfoldingCollection);
+            }
+        }
+
+        class RestCollectionData : BaseData
+        {
+            public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+            {
+                return new SubscriptionDataSource("rest.collection.source", SubscriptionTransportMedium.Rest, FileFormat.UnfoldingCollection);
+            }
+        }
+
+        class RemoteFileData : BaseData
+        {
+            public override DateTime EndTime
+            {
+                get { return Time + QuantConnect.Time.OneDay; }
+                set { Time = value - QuantConnect.Time.OneDay; }
+            }
+
+            public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+            {
+                return new SubscriptionDataSource("remote.file.source", SubscriptionTransportMedium.RemoteFile);
+            }
+        }
+
+        class LocalFileData : BaseData
+        {
+            public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+            {
+                return new SubscriptionDataSource("local.file.source", SubscriptionTransportMedium.LocalFile);
+            }
+        }
+
+        class TestableLiveCustomDataSubscriptionEnumeratorFactory : LiveCustomDataSubscriptionEnumeratorFactory
+        {
+            private readonly ISubscriptionDataSourceReader _dataSourceReader;
+
+            public TestableLiveCustomDataSubscriptionEnumeratorFactory(ITimeProvider timeProvider, ISubscriptionDataSourceReader dataSourceReader, TimeSpan? minimumIntervalCheck = null)
+                : base(timeProvider, null, minimumIntervalCheck: minimumIntervalCheck)
+            {
+                _dataSourceReader = dataSourceReader;
+            }
+
+            protected override ISubscriptionDataSourceReader GetSubscriptionDataSourceReader(SubscriptionDataSource source,
+                IDataCacheProvider dataCacheProvider,
+                SubscriptionDataConfig config,
+                DateTime date,
+                BaseData baseData,
+                IDataProvider dataProvider)
+            {
+                return _dataSourceReader;
+            }
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/FastForwardEnumeratorTests.cs

@@ -0,0 +1,111 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using NUnit.Framework;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class FastForwardEnumeratorTests
+    {
+        [Test]
+        public void FastForwardsOldData()
+        {
+            var start = new DateTime(2015, 10, 10, 13, 0, 0);
+            var data = new List<Tick>
+            {
+                new Tick {Time = start.AddMinutes(-1)},
+                new Tick {Time = start.AddSeconds(-1)},
+                new Tick {Time = start.AddSeconds(0)},
+                new Tick {Time = start.AddSeconds(1)},
+            };
+
+            var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
+            var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(0.5));
+
+            Assert.IsTrue(fastForward.MoveNext());
+            Assert.AreEqual(start, fastForward.Current.Time);
+            fastForward.Dispose();
+        }
+        [Test]
+        public void FastForwardsOldDataAllowsEquals()
+        {
+            var start = new DateTime(2015, 10, 10, 13, 0, 0);
+            var data = new List<Tick>
+            {
+                new Tick {Time = start.AddMinutes(-1)},
+                new Tick {Time = start.AddSeconds(-1)},
+                new Tick {Time = start.AddSeconds(0)},
+                new Tick {Time = start.AddSeconds(1)},
+            };
+
+            var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
+            var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(1));
+
+            Assert.IsTrue(fastForward.MoveNext());
+            Assert.AreEqual(start.AddSeconds(-1), fastForward.Current.Time);
+            fastForward.Dispose();
+        }
+        [Test]
+        public void FiltersOutPastData()
+        {
+            var start = new DateTime(2015, 10, 10, 13, 0, 0);
+            var data = new List<Tick>
+            {
+                new Tick {Time = start.AddMinutes(-1)},
+                new Tick {Time = start.AddSeconds(-1)},
+                new Tick {Time = start.AddSeconds(1)},
+                new Tick {Time = start.AddSeconds(0)},
+                new Tick {Time = start.AddSeconds(2)}
+            };
+
+            var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
+            var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(0.5));
+
+            Assert.IsTrue(fastForward.MoveNext());
+            Assert.AreEqual(start.AddSeconds(1), fastForward.Current.Time);
+
+            Assert.IsTrue(fastForward.MoveNext());
+            Assert.AreEqual(start.AddSeconds(2), fastForward.Current.Time);
+            fastForward.Dispose();
+        }
+
+        [Test]
+        public void CurrentIsNullWhenEnumeratorReturnsFalse()
+        {
+            var start = new DateTime(2015, 10, 10, 13, 0, 0);
+            var data = new List<Tick>
+            {
+                new Tick {Time = start.AddSeconds(0)}
+            };
+
+            var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
+            var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(0.5));
+
+            Assert.IsTrue(fastForward.MoveNext());
+            Assert.IsNotNull(fastForward.Current);
+
+            Assert.IsFalse(fastForward.MoveNext());
+            Assert.IsNull(fastForward.Current);
+            fastForward.Dispose();
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/FrontierAwareEnumeratorTests.cs

@@ -0,0 +1,99 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using NodaTime;
+using NUnit.Framework;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class FrontierAwareEnumeratorTests
+    {
+        [Test]
+        public void ReturnsTrueWhenNextDataIsAheadOfFrontier()
+        {
+            var currentTime = new DateTime(2015, 10, 13);
+            var timeProvider = new ManualTimeProvider(currentTime);
+            var underlying = new List<Tick>
+            {
+                new Tick {Time = currentTime.AddSeconds(1)}
+            };
+
+            var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
+            var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
+
+            Assert.IsTrue(frontierAware.MoveNext());
+            Assert.IsNull(frontierAware.Current);
+            frontierAware.Dispose();
+        }
+
+        [Test]
+        public void YieldsDataWhenFrontierPasses()
+        {
+            var currentTime = new DateTime(2015, 10, 13);
+            var timeProvider = new ManualTimeProvider(currentTime);
+            var underlying = new List<Tick>
+            {
+                new Tick {Time = currentTime.AddSeconds(1)}
+            };
+
+            var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
+            var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
+
+            timeProvider.AdvanceSeconds(1);
+
+            Assert.IsTrue(frontierAware.MoveNext());
+            Assert.IsNotNull(frontierAware.Current);
+            Assert.AreEqual(underlying[0], frontierAware.Current);
+            frontierAware.Dispose();
+        }
+
+        [Test]
+        public void YieldsFutureDataAtCorrectTime()
+        {
+            var currentTime = new DateTime(2015, 10, 13);
+            var timeProvider = new ManualTimeProvider(currentTime);
+            var underlying = new List<Tick>
+            {
+                new Tick {Time = currentTime.AddSeconds(10)}
+            };
+
+            var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
+            var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
+
+            for (int i = 0; i < 10; i++)
+            {
+                timeProvider.AdvanceSeconds(1);
+                Assert.IsTrue(frontierAware.MoveNext());
+                if (i < 9)
+                {
+                    Assert.IsNull(frontierAware.Current);
+                }
+                else
+                {
+                    Assert.IsNotNull(frontierAware.Current);
+                    Assert.AreEqual(underlying[0], frontierAware.Current);
+                }
+            }
+            frontierAware.Dispose();
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/LiveAuxiliaryDataEnumeratorTests.cs

@@ -0,0 +1,306 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using System.Globalization;
+using QuantConnect.Securities;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class LiveAuxiliaryDataEnumeratorTests
+    {
+        [TestCase(DataMappingMode.OpenInterest, "20130616", false)]
+        [TestCase(DataMappingMode.FirstDayMonth, "20130602", false)]
+        [TestCase(DataMappingMode.LastTradingDay, "20130623", false)]
+        [TestCase(DataMappingMode.OpenInterest, "20130616", true)]
+        [TestCase(DataMappingMode.FirstDayMonth, "20130602", true)]
+        [TestCase(DataMappingMode.LastTradingDay, "20130623", true)]
+        public void EmitsMappingEventsBasedOnCurrentMapFileAndTime(DataMappingMode dataMappingMode, string mappingDate, bool delayed)
+        {
+            var config = new SubscriptionDataConfig(typeof(TradeBar),
+                Symbols.ES_Future_Chain,
+                Resolution.Daily,
+                TimeZones.NewYork,
+                TimeZones.NewYork,
+                true,
+                true,
+                false,
+                dataMappingMode: dataMappingMode);
+            var symbolMaps = new List<SubscriptionDataConfig.NewSymbolEventArgs>();
+            config.NewSymbol += (sender, args) => symbolMaps.Add(args);
+            var time = new DateTime(2013, 05, 28);
+            var cache = new SecurityCache();
+
+            cache.AddData(new Tick(time, config.Symbol, 20, 10));
+            var timeProvider = new ManualTimeProvider(time);
+
+            var futureTicker1 = "es vhle2yxr5blt";
+            TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, new[]
+            {
+                new MapFileRow(Time.BeginningOfTime, Futures.Indices.SP500EMini, Exchange.CME),
+                new MapFileRow(new DateTime(2013,06,01), futureTicker1, Exchange.CME, DataMappingMode.FirstDayMonth),
+                new MapFileRow(new DateTime(2013,06,15), futureTicker1, Exchange.CME, DataMappingMode.OpenInterest),
+                new MapFileRow(new DateTime(2013,06,22), futureTicker1, Exchange.CME, DataMappingMode.LastTradingDay),
+            });
+
+            IEnumerator<BaseData> enumerator;
+            Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, cache, new TestMapFileProvider(), TestGlobals.FactorFileProvider, time, out enumerator));
+
+            // get's mapped right away!
+            Assert.AreEqual(futureTicker1.ToUpper(), config.MappedSymbol);
+
+            Assert.AreEqual(1, symbolMaps.Count);
+            Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].Old);
+            Assert.AreEqual(Futures.Indices.SP500EMini, symbolMaps[0].Old.ID.Symbol);
+            Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].New);
+            Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[0].New.Underlying.ID.ToString());
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var expectedMappingDate = DateTime.ParseExact(mappingDate, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
+            if (delayed)
+            {
+                // we advance to the mapping date, without any new mapFile!
+                timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
+            }
+            else
+            {
+                // just advance a day to show nothing happens until mapping time
+                timeProvider.Advance(TimeSpan.FromDays(1));
+            }
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var futureTicker2 = "es vk2zrh843z7l";
+            TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, TestMapFileResolver.MapFile.Concat(
+                new[]
+                {
+                    new MapFileRow(new DateTime(2013,09,01), futureTicker2, Exchange.CME, DataMappingMode.FirstDayMonth),
+                    new MapFileRow(new DateTime(2013,09,14), futureTicker2, Exchange.CME, DataMappingMode.OpenInterest),
+                    new MapFileRow(new DateTime(2013,09,21), futureTicker2, Exchange.CME, DataMappingMode.LastTradingDay),
+                }));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            if (delayed)
+            {
+                // we got a new mapFile! advance the date and expect mapping to have happened
+                timeProvider.Advance(TimeSpan.FromDays(1));
+            }
+            else
+            {
+                // we advance to the mapping date
+                timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
+            }
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current);
+
+            Assert.AreEqual(2, symbolMaps.Count);
+            Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].Old);
+            Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[1].Old.Underlying.ID.ToString());
+            Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].New);
+            Assert.AreEqual(futureTicker2.ToUpper(), symbolMaps[1].New.Underlying.ID.ToString());
+
+            Assert.AreEqual(futureTicker2.ToUpper(), config.MappedSymbol);
+
+            Assert.AreEqual(futureTicker2.ToUpper(), (enumerator.Current as SymbolChangedEvent).NewSymbol);
+            Assert.AreEqual(futureTicker1.ToUpper(), (enumerator.Current as SymbolChangedEvent).OldSymbol);
+            Assert.AreEqual(config.Symbol, (enumerator.Current as SymbolChangedEvent).Symbol);
+            Assert.AreEqual(timeProvider.GetUtcNow().Date, (enumerator.Current as SymbolChangedEvent).Time);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void EmitsDelistingEventsBasedOnCurrentTime()
+        {
+            var config = new SubscriptionDataConfig(typeof(TradeBar),
+                Symbols.SPY_C_192_Feb19_2016,
+                Resolution.Daily,
+                TimeZones.NewYork,
+                TimeZones.NewYork,
+                true,
+                true,
+                false);
+            var delistingDate = config.Symbol.GetDelistingDate();
+            var time = delistingDate.AddDays(-10);
+            var cache = new SecurityCache();
+            cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 20, 10));
+            var timeProvider = new ManualTimeProvider(time);
+
+            IEnumerator<BaseData> enumerator;
+            Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, cache, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, config.Symbol.ID.Date, out enumerator));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            // advance until delisting date, take into account 5 hour offset of NY + TradableDateOffset
+            timeProvider.Advance(TimeSpan.FromDays(10));
+            timeProvider.Advance(TimeSpan.FromHours(5));
+            timeProvider.Advance(Time.LiveAuxiliaryDataOffset);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
+            Assert.AreEqual(delistingDate, (enumerator.Current as Delisting).Time);
+            Assert.AreEqual(15, (enumerator.Current as Delisting).Price);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            // when the day ends the delisted event will pass through, respecting the offset
+            timeProvider.Advance(TimeSpan.FromDays(1));
+
+            cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 40, 20));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
+            Assert.AreEqual(delistingDate.AddDays(1), (enumerator.Current as Delisting).Time);
+            Assert.AreEqual(30, (enumerator.Current as Delisting).Price);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+            enumerator.Dispose();
+        }
+
+        [TestCase(false)]
+        [TestCase(true)]
+        public void EquityEmitsDelistingEventsBasedOnCurrentTime(bool delayed)
+        {
+            var config = new SubscriptionDataConfig(typeof(TradeBar),
+                Symbols.SPY,
+                Resolution.Daily,
+                TimeZones.NewYork,
+                TimeZones.NewYork,
+                true,
+                true,
+                false);
+            var time = new DateTime(2013, 05, 28);
+            var cache = new SecurityCache();
+
+            cache.AddData(new Tick(time, config.Symbol, 20, 10));
+            var timeProvider = new ManualTimeProvider(time);
+
+            TestMapFileResolver.MapFile = new MapFile(config.Symbol.ID.Symbol, new[]
+            {
+                new MapFileRow(Time.BeginningOfTime, config.Symbol.ID.Symbol),
+                new MapFileRow(Time.EndOfTime, config.Symbol.ID.Symbol),
+            });
+
+            IEnumerator<BaseData> enumerator;
+            Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, cache, new TestMapFileProvider(), TestGlobals.FactorFileProvider, time, out enumerator));
+
+            // get's mapped right away!
+            Assert.AreEqual("SPY", config.MappedSymbol);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var delistingDate = time.AddDays(2);
+            var delistedMapFile = new MapFile(config.Symbol.ID.Symbol, new[]
+            {
+                new MapFileRow(Time.BeginningOfTime, config.Symbol.ID.Symbol),
+                new MapFileRow(delistingDate, config.Symbol.ID.Symbol),
+            });
+
+            // just advance a day to show nothing happens until mapping time
+            timeProvider.Advance(TimeSpan.FromDays(1));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            if (!delayed)
+            {
+                TestMapFileResolver.MapFile = delistedMapFile;
+            }
+
+            // we advance to the mapping date, without any new mapFile!
+            timeProvider.Advance(delistingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
+
+            if (delayed)
+            {
+                // nothing happens
+                Assert.IsTrue(enumerator.MoveNext());
+                Assert.IsNull(enumerator.Current);
+
+                TestMapFileResolver.MapFile = delistedMapFile;
+
+                timeProvider.Advance(Time.OneDay);
+            }
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current);
+
+            var delisted = enumerator.Current as Delisting;
+            Assert.IsNotNull(delisted);
+            Assert.AreEqual(DelistingType.Warning, delisted.Type);
+
+            if (!delayed)
+            {
+                Assert.IsTrue(enumerator.MoveNext());
+                Assert.IsNull(enumerator.Current);
+
+                // delisting passed
+                timeProvider.Advance(TimeSpan.FromDays(1));
+            }
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current);
+
+            delisted = enumerator.Current as Delisting;
+            Assert.IsNotNull(delisted);
+            Assert.AreEqual(DelistingType.Delisted, delisted.Type);
+            enumerator.Dispose();
+        }
+
+        private class TestMapFileProvider : IMapFileProvider
+        {
+            public void Initialize(IDataProvider dataProvider)
+            {
+            }
+
+            public MapFileResolver Get(AuxiliaryDataKey auxiliaryDataKey)
+            {
+                return new TestMapFileResolver();
+            }
+        }
+
+        private class TestMapFileResolver : MapFileResolver
+        {
+            public static MapFile MapFile { get; set; }
+            public TestMapFileResolver()
+                : base(new[] { MapFile })
+            {
+            }
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/LiveEquityDataSynchronizingEnumeratorTests.cs

@@ -0,0 +1,80 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using NodaTime;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using QuantConnect.Logging;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture, Parallelizable(ParallelScope.All)]
+    public class LiveEquityDataSynchronizingEnumeratorTests
+    {
+        // this test case generates data points in the past, will complete very quickly
+        [TestCase(-15, 1)]
+        // this test case generates data points in the future, will require at least 10 seconds to complete
+        [TestCase(0, 11)]
+        public void SynchronizesData(int timeOffsetSeconds, int testTimeSeconds)
+        {
+            var start = DateTime.UtcNow;
+            var end = start.AddSeconds(testTimeSeconds);
+
+            var time = start;
+            var tickList1 = Enumerable.Range(0, 10).Select(x => new Tick { Time = time.AddSeconds(x * 1 + timeOffsetSeconds), Value = x }).ToList();
+            var tickList2 = Enumerable.Range(0, 5).Select(x => new Tick { Time = time.AddSeconds(x * 2 + timeOffsetSeconds), Value = x + 100 }).ToList();
+            var stream1 = tickList1.GetEnumerator();
+            var stream2 = tickList2.GetEnumerator();
+
+            var count1 = 0;
+            var count2 = 0;
+            var previous = DateTime.MinValue;
+            var synchronizer = new LiveAuxiliaryDataSynchronizingEnumerator(new RealTimeProvider(), DateTimeZone.Utc, stream1, new List<IEnumerator<BaseData>> { stream2 });
+            while (synchronizer.MoveNext() && DateTime.UtcNow < end)
+            {
+                if (synchronizer.Current != null)
+                {
+                    if (synchronizer.Current.Value < 100)
+                    {
+                        Assert.AreEqual(count1, synchronizer.Current.Value);
+                        count1++;
+                    }
+                    else
+                    {
+                        Assert.AreEqual(count2 + 100, synchronizer.Current.Value);
+                        count2++;
+                    }
+
+                    Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous));
+                    previous = synchronizer.Current.EndTime;
+
+                    Log.Trace($"Data point emitted: {synchronizer.Current.EndTime:O} - {synchronizer.Current}");
+                }
+            }
+
+            Log.Trace($"Total point count: {count1 + count2}");
+
+            Assert.AreEqual(tickList1.Count, count1);
+            Assert.AreEqual(tickList2.Count, count2);
+            synchronizer.Dispose();
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/LiveFillForwardEnumeratorTests.cs

@@ -0,0 +1,577 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Securities;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class LiveFillForwardEnumeratorTests
+    {
+        [Test]
+        public void FillsForwardOnNulls()
+        {
+            var reference = new DateTime(2015, 10, 08);
+            var period = Time.OneSecond;
+            var underlying = new List<BaseData>
+            {
+                // 0 seconds
+                new TradeBar(reference, Symbols.SPY, 10, 20, 5, 15, 123456, period),
+                // 1 seconds
+                null,
+                // 3 seconds
+                new TradeBar(reference.AddSeconds(2), Symbols.SPY, 100, 200, 50, 150, 1234560, period),
+                null,
+                null,
+                null,
+                null
+            };
+
+            var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
+            timeProvider.SetCurrentTime(reference);
+            var exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
+            var fillForward = new LiveFillForwardEnumerator(timeProvider, underlying.GetEnumerator(), exchange, Ref.Create(Time.OneSecond), false, reference, Time.EndOfTime, Resolution.Second, exchange.TimeZone, false);
+
+            // first point is always emitted
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.AreEqual(underlying[0], fillForward.Current);
+            Assert.AreEqual(123456, ((TradeBar)fillForward.Current).Volume);
+
+            // stepping again without advancing time does nothing, but we'll still
+            // return true as per IEnumerator contract
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.IsNull(fillForward.Current);
+
+            timeProvider.SetCurrentTime(reference.AddSeconds(2));
+
+            // non-null next will fill forward in between
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.AreEqual(underlying[0].EndTime, fillForward.Current.Time);
+            Assert.AreEqual(underlying[0].Value, fillForward.Current.Value);
+            Assert.IsTrue(fillForward.Current.IsFillForward);
+            Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
+
+            // even without stepping the time this will advance since non-null data is ready
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.AreEqual(underlying[2], fillForward.Current);
+            Assert.AreEqual(1234560, ((TradeBar)fillForward.Current).Volume);
+
+            // step ahead into null data territory
+            timeProvider.SetCurrentTime(reference.AddSeconds(4));
+
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
+            Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork).RoundDown(Time.OneSecond), fillForward.Current.EndTime);
+            Assert.IsTrue(fillForward.Current.IsFillForward);
+            Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
+
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.IsNull(fillForward.Current);
+
+            timeProvider.SetCurrentTime(reference.AddSeconds(5));
+
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
+            Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork).RoundDown(Time.OneSecond), fillForward.Current.EndTime);
+            Assert.IsTrue(fillForward.Current.IsFillForward);
+            Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
+
+            timeProvider.SetCurrentTime(reference.AddSeconds(6));
+
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
+            Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork).RoundDown(Time.OneSecond), fillForward.Current.EndTime);
+            Assert.IsTrue(fillForward.Current.IsFillForward);
+            Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
+
+            fillForward.Dispose();
+        }
+
+        [Test]
+        public void HandlesDaylightSavingTimeChange()
+        {
+            // In 2018, Daylight Saving Time (DST) began at 2 AM on Sunday, March 11
+            // This means that clocks were moved forward one hour on March 11
+            var reference = new DateTime(2018, 3, 10);
+            var period = Time.OneDay;
+            var underlying = new List<TradeBar>
+            {
+                new TradeBar(reference, Symbols.SPY, 10, 20, 5, 15, 123456, period),
+                // Daylight Saving Time change, the data still goes from midnight to midnight
+                new TradeBar(reference.AddDays(1), Symbols.SPY, 100, 200, 50, 150, 1234560, period)
+            };
+
+            var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
+            timeProvider.SetCurrentTime(reference);
+            var exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
+            var fillForward = new LiveFillForwardEnumerator(
+                timeProvider,
+                underlying.GetEnumerator(),
+                exchange,
+                Ref.Create(Time.OneDay),
+                false,
+                reference,
+                Time.EndOfTime,
+                Resolution.Daily,
+                exchange.TimeZone, false);
+
+            // first point is always emitted
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.IsFalse(fillForward.Current.IsFillForward);
+            Assert.AreEqual(underlying[0], fillForward.Current);
+            //Assert.AreEqual(underlying[0].EndTime, fillForward.Current.EndTime);
+            Assert.AreEqual(123456, ((TradeBar)fillForward.Current).Volume);
+
+            // Daylight Saving Time change -> add 1 hour
+            timeProvider.SetCurrentTime(reference.AddDays(1).AddHours(1));
+
+            // second data point emitted
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.IsFalse(fillForward.Current.IsFillForward);
+            Assert.AreEqual(underlying[1], fillForward.Current);
+            //Assert.AreEqual(underlying[1].EndTime, fillForward.Current.EndTime);
+            Assert.AreEqual(1234560, ((TradeBar)fillForward.Current).Volume);
+
+            Assert.IsTrue(fillForward.MoveNext());
+            Assert.IsTrue(fillForward.Current.IsFillForward);
+            Assert.AreEqual(underlying[1].EndTime, fillForward.Current.Time);
+            Assert.AreEqual(underlying[1].Value, fillForward.Current.Value);
+            Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
+
+            fillForward.Dispose();
+        }
+
+        [Test]
+        public void LiveFillForwardEnumeratorDoesNotStall()
+        {
+            var reference = new DateTime(2020, 5, 21, 9, 40, 0, 100);
+            var timeProvider = new ManualTimeProvider(reference, TimeZones.NewYork);
+
+            using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, reference.Date, Resolution.Minute, out var enqueueableEnumerator, false);
+            var openingBar = new TradeBar
+            {
+                Open = 0.01m,
+                High = 0.01m,
+                Low = 0.01m,
+                Close = 0.01m,
+                Volume = 1,
+                EndTime = new DateTime(2020, 5, 21, 9, 40, 0),
+                Symbol = Symbols.AAPL
+            };
+            var secondBar = new TradeBar
+            {
+                Open = 1m,
+                High = 2m,
+                Low = 1m,
+                Close = 2m,
+                Volume = 100,
+                EndTime = new DateTime(2020, 5, 21, 9, 42, 0),
+                Symbol = Symbols.AAPL
+            };
+
+
+            // Enqueue the first point, which will be emitted ASAP.
+            enqueueableEnumerator.Enqueue(openingBar);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.NotNull(fillForwardEnumerator.Current);
+            Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+            Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
+
+            // Advance the time, we expect a fill-forward bar.
+            timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 41, 0, 100) + LiveFillForwardEnumerator.GetMaximumDataTimeout(Resolution.Minute));
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
+            Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+            Assert.AreEqual(openingBar.EndTime.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
+
+            // Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
+            timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 42, 0, 100));
+            enqueueableEnumerator.Enqueue(secondBar);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
+            enqueueableEnumerator.Dispose();
+        }
+
+        private static IEnumerable<TestCaseData> TimeOutTestCases
+        {
+            get
+            {
+                // Hour resolution, fill forward to market open
+                yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 10, 0, 0), true, true);
+                yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 10, 0, 0), false, false);
+                yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 10, 0, 0), null, true, false);
+                yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 10, 0, 0), null, false, false);
+                // over a weekend (22th is a Friday and 25th Monday is a holiday)
+                yield return new(Resolution.Hour, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 10, 0, 0), true, true);
+                yield return new(Resolution.Hour, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 10, 0, 0), false, false);
+                // market close
+                yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 15, 0, 0), null, true, true);
+                yield return new(Resolution.Hour, new DateTime(2020, 5, 21, 15, 0, 0), null, false, false);
+
+                // Minute resolution, fill forward to market open
+                yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 9, 31, 0), true, true);
+                yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 9, 31, 0), false, false);
+                yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 9, 31, 0), null, true, false);
+                yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 9, 31, 0), null, false, false);
+                // over a weekend
+                yield return new(Resolution.Minute, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 9, 31, 0), true, true);
+                yield return new(Resolution.Minute, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 9, 31, 0), false, false);
+                // market close
+                yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 15, 59, 0), null, true, true);
+                yield return new(Resolution.Minute, new DateTime(2020, 5, 21, 15, 59, 0), null, false, false);
+
+                // Second resolution, fill forward to market open
+                yield return new(Resolution.Second, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 9, 30, 1), true, true);
+                yield return new(Resolution.Second, new DateTime(2020, 5, 21, 16, 0, 0), new DateTime(2020, 5, 22, 9, 30, 1), false, false);
+                yield return new(Resolution.Second, new DateTime(2020, 5, 21, 9, 30, 1), null, true, false);
+                yield return new(Resolution.Second, new DateTime(2020, 5, 21, 9, 30, 1), null, false, false);
+                // over a weekend
+                yield return new(Resolution.Second, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 9, 30, 1), true, true);
+                yield return new(Resolution.Second, new DateTime(2020, 5, 22, 16, 0, 0), new DateTime(2020, 5, 26, 9, 30, 1), false, false);
+                // market close
+                yield return new(Resolution.Second, new DateTime(2020, 5, 21, 15, 59, 59), null, true, true);
+                yield return new(Resolution.Second, new DateTime(2020, 5, 21, 15, 59, 59), null, false, false);
+            }
+        }
+
+        [TestCaseSource(nameof(TimeOutTestCases))]
+        public void TakesIntoAccountTimeOut(Resolution resolution, DateTime previousDataEndTime, DateTime? expectedNextBarEndTime, bool dataArrivedLate, bool shouldHaveTimeout)
+        {
+            var timeProvider = new ManualTimeProvider(previousDataEndTime, TimeZones.NewYork);
+
+            using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, previousDataEndTime.Date, resolution, out var enqueueableEnumerator, dailyStrictEndTimeEnabled: false);
+            var period = resolution.ToTimeSpan();
+            var openingBar = new TradeBar(previousDataEndTime.Subtract(period), Symbols.AAPL, 0.01m, 0.01m, 0.01m, 0.01m, 1, period);
+
+            expectedNextBarEndTime ??= previousDataEndTime.Add(resolution.ToTimeSpan());
+            var secondBar = new TradeBar(openingBar.EndTime, Symbols.AAPL, 1m, 2m, 1m, 2m, 100, period);
+
+            // Enqueue the first point, which will be emitted ASAP.
+            enqueueableEnumerator.Enqueue(openingBar);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.NotNull(fillForwardEnumerator.Current);
+            Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+            Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
+
+            // Advance the time, we don't expect a fill-forward bar because the timeout amount has not passed yet
+            timeProvider.SetCurrentTime(expectedNextBarEndTime.Value);
+            if (dataArrivedLate)
+            {
+                Assert.IsTrue(fillForwardEnumerator.MoveNext());
+
+                if (shouldHaveTimeout)
+                {
+                    Assert.IsNull(fillForwardEnumerator.Current);
+
+                    // Advance the time, including the expected timout, we expect a fill-forward bar.
+                    timeProvider.Advance(LiveFillForwardEnumerator.GetMaximumDataTimeout(resolution));
+
+                    Assert.IsTrue(fillForwardEnumerator.MoveNext());
+                }
+
+                Assert.IsNotNull(fillForwardEnumerator.Current);
+                Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
+                Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+                Assert.AreEqual(expectedNextBarEndTime, fillForwardEnumerator.Current.EndTime);
+
+                Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
+                Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+                Assert.AreEqual(expectedNextBarEndTime, fillForwardEnumerator.Current.EndTime);
+
+            }
+
+            // Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
+            enqueueableEnumerator.Enqueue(secondBar);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
+            enqueueableEnumerator.Dispose();
+        }
+
+        [Test]
+        public void TakesIntoAccountTimeOutWhenThereAreBigGaps([Values(Resolution.Second, Resolution.Minute, Resolution.Hour)] Resolution resolution)
+        {
+            var start = new DateTime(2020, 5, 20, 12, 0, 0);
+            var end = new DateTime(2020, 5, 22, 16, 0, 0);
+
+            var timeProvider = new ManualTimeProvider(start, TimeZones.NewYork);
+
+            using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, start, resolution, out var enqueueableEnumerator, dailyStrictEndTimeEnabled: false);
+            var period = resolution.ToTimeSpan();
+            var openingBar = new TradeBar(start.Subtract(period), Symbols.AAPL, 0.01m, 0.01m, 0.01m, 0.01m, 1, period);
+
+            // Enqueue the first point, which will be emitted ASAP.
+            enqueueableEnumerator.Enqueue(openingBar);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.NotNull(fillForwardEnumerator.Current);
+            Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+            Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
+
+            timeProvider.Advance(period);
+
+            var previous = (TradeBar)fillForwardEnumerator.Current;
+            var currentIsMarketOpen = false;
+            var currentIsMarketClose = false;
+            while (previous.EndTime < end)
+            {
+                var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
+
+                Assert.IsTrue(fillForwardEnumerator.MoveNext(), $"Previous: {previous.EndTime}");
+
+                if (currentIsMarketOpen || currentIsMarketClose)
+                {
+                    Assert.IsNull(fillForwardEnumerator.Current, $"Previous: {previous.EndTime}");
+
+                    // Advance the time, including the expected timout, we expect a fill-forward bar.
+                    timeProvider.Advance(LiveFillForwardEnumerator.GetMaximumDataTimeout(resolution));
+                    Assert.IsTrue(fillForwardEnumerator.MoveNext(), $"Previous: {previous.EndTime}");
+                }
+
+                Assert.IsNotNull(fillForwardEnumerator.Current, $"Previous: {previous.EndTime}");
+
+                var current = (TradeBar)fillForwardEnumerator.Current;
+                Assert.IsTrue(current.IsFillForward, $"Current: {previous.EndTime}");
+                Assert.AreEqual(previous.Open, current.Open, $"Current: {previous.EndTime}");
+                Assert.AreEqual(previous.Price, current.Price, $"Current: {previous.EndTime}");
+
+                var expectedEndTime = previous.EndTime.Add(period);
+                if (currentIsMarketOpen)
+                {
+                    expectedEndTime = expectedEndTime.Date.AddDays(1).AddHours(9).AddMinutes(30).Add(period);
+                    if (resolution == Resolution.Hour)
+                    {
+                        expectedEndTime = expectedEndTime.RoundDown(period);
+                    }
+                }
+                Assert.AreEqual(expectedEndTime, current.EndTime, $"Current: {previous.EndTime}");
+
+                currentIsMarketOpen = current.EndTime.TimeOfDay == TimeSpan.FromHours(16);
+                currentIsMarketClose = current.EndTime.TimeOfDay == TimeSpan.FromHours(16) - period;
+                previous = current;
+
+                // Advance the time provider
+                var nextTime = current.EndTime.Add(period);
+                if (nextTime.TimeOfDay > TimeSpan.FromHours(16))
+                {
+                    // If the next time is after market close, we need to advance to the next day
+                    nextTime = nextTime.Date.AddDays(1).AddHours(9).AddMinutes(30).Add(period);
+                    if (resolution == Resolution.Hour)
+                    {
+                        nextTime = nextTime.RoundDown(period);
+                    }
+                }
+                timeProvider.SetCurrentTime(nextTime);
+            }
+
+            enqueueableEnumerator.Dispose();
+        }
+
+        [TestCase(true, true)]
+        [TestCase(false, true)]
+        [TestCase(true, false)]
+        [TestCase(false, false)]
+        public void TakesIntoAccountTimeOutDaily(bool dailyStrictEndTimeEnabled, bool dataArrivedLate)
+        {
+            var resolution = Resolution.Daily;
+            var referenceOpenTime = new DateTime(2020, 5, 21, 9, 30, 0);
+            var referenceTime = new DateTime(2020, 5, 21, 16, 0, 0);
+            if (!dailyStrictEndTimeEnabled)
+            {
+                referenceOpenTime = referenceOpenTime.Date;
+                referenceTime = referenceTime.Date.AddDays(1);
+            }
+            var timeProvider = new ManualTimeProvider(referenceTime, TimeZones.NewYork);
+            using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, referenceTime.Date, resolution, out var enqueueableEnumerator, dailyStrictEndTimeEnabled);
+            var openingBar = new TradeBar
+            {
+                Open = 0.01m,
+                High = 0.01m,
+                Low = 0.01m,
+                Close = 0.01m,
+                Volume = 1,
+                Time = referenceOpenTime,
+                EndTime = referenceTime,
+                Symbol = Symbols.AAPL
+            };
+            var secondBar = new TradeBar
+            {
+                Open = 1m,
+                High = 2m,
+                Low = 1m,
+                Close = 2m,
+                Volume = 100,
+                Time = referenceOpenTime.AddDays(1),
+                EndTime = referenceTime.AddDays(1),
+                Symbol = Symbols.AAPL
+            };
+
+            // Enqueue the first point, which will be emitted ASAP.
+            enqueueableEnumerator.Enqueue(openingBar);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.NotNull(fillForwardEnumerator.Current);
+            Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+            Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
+
+            // Advance the time, we don't expect a fill-forward bar because the timeout amount has not passed yet
+            timeProvider.SetCurrentTime(secondBar.EndTime);
+
+            if (dataArrivedLate)
+            {
+                Assert.IsTrue(fillForwardEnumerator.MoveNext());
+                Assert.IsNull(fillForwardEnumerator.Current);
+
+                // Advance the time, including the expected timout, we expect a fill-forward bar.
+                timeProvider.Advance(LiveFillForwardEnumerator.GetMaximumDataTimeout(resolution));
+                Assert.IsTrue(fillForwardEnumerator.MoveNext());
+                Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
+                Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+                Assert.AreEqual(secondBar.EndTime, fillForwardEnumerator.Current.EndTime);
+            }
+
+            // Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
+            enqueueableEnumerator.Enqueue(secondBar);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
+            Assert.AreEqual(secondBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+            Assert.AreEqual(secondBar.EndTime, fillForwardEnumerator.Current.EndTime);
+            enqueueableEnumerator.Dispose();
+        }
+
+        [TestCase(Resolution.Minute)]
+        [TestCase(Resolution.Hour)]
+        public void MultiResolutionSmallerFillForwardResolution(Resolution resolution)
+        {
+            var ffResolution = Resolution.Second;
+            var referenceOpenTime = new DateTime(2020, 5, 21, 14, 0, 0);
+            if (resolution == Resolution.Minute)
+            {
+                referenceOpenTime = new DateTime(2020, 5, 21, 15, 59, 0);
+            }
+            var referenceTime = new DateTime(2020, 5, 21, 15, 0, 0);
+
+            var timeProvider = new ManualTimeProvider(referenceTime, TimeZones.NewYork);
+            using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, referenceTime.Date, resolution, out var enqueueableEnumerator, true, ffResolution);
+            var openingBar = new TradeBar
+            {
+                Open = 0.01m,
+                High = 0.01m,
+                Low = 0.01m,
+                Close = 0.01m,
+                Volume = 1,
+                Time = referenceOpenTime,
+                EndTime = referenceTime,
+                Symbol = Symbols.AAPL
+            };
+
+            // Enqueue the first point, which will be emitted ASAP.
+            enqueueableEnumerator.Enqueue(openingBar);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.NotNull(fillForwardEnumerator.Current);
+            Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+            Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
+
+            // Advance the time, we expect a fill-forward bar
+
+            for (var i = 0; i < 60; i++)
+            {
+                timeProvider.Advance(Time.OneSecond);
+
+                Assert.IsTrue(fillForwardEnumerator.MoveNext());
+                Assert.NotNull(fillForwardEnumerator.Current);
+                Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+                Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork), fillForwardEnumerator.Current.EndTime);
+            }
+
+            enqueueableEnumerator.Dispose();
+        }
+
+        [TestCase(Resolution.Daily, Resolution.Minute)]
+        [TestCase(Resolution.Hour, Resolution.Minute)]
+        [TestCase(Resolution.Daily, Resolution.Second)]
+        [TestCase(Resolution.Hour, Resolution.Second)]
+        public void MultiResolutionMarketClose(Resolution resolution, Resolution ffResolution)
+        {
+            var referenceOpenTime = new DateTime(2020, 5, 21, 9, 30, 0);
+            if (resolution == Resolution.Hour)
+            {
+                referenceOpenTime = new DateTime(2020, 5, 21, 15, 0, 0);
+            }
+            var referenceTime = new DateTime(2020, 5, 21, 16, 0, 0);
+            var timeProvider = new ManualTimeProvider(referenceTime, TimeZones.NewYork);
+            using var fillForwardEnumerator = GetLiveFillForwardEnumerator(timeProvider, referenceTime.Date, resolution, out var enqueueableEnumerator, true, ffResolution);
+            var openingBar = new TradeBar
+            {
+                Open = 0.01m,
+                High = 0.01m,
+                Low = 0.01m,
+                Close = 0.01m,
+                Volume = 1,
+                Time = referenceOpenTime,
+                EndTime = referenceTime,
+                Symbol = Symbols.AAPL
+            };
+
+            // Enqueue the first point, which will be emitted ASAP.
+            enqueueableEnumerator.Enqueue(openingBar);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.NotNull(fillForwardEnumerator.Current);
+            Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
+            Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
+
+            for (var i = 0; i < 600; i++)
+            {
+                // Advance the time, we don't expect a fill-forward bar, we've emitted our daily bar already and market is closed
+                timeProvider.Advance(ffResolution.ToTimeSpan());
+                Assert.IsTrue(fillForwardEnumerator.MoveNext());
+                Assert.IsNull(fillForwardEnumerator.Current);
+            }
+
+            enqueueableEnumerator.Dispose();
+        }
+
+        private static LiveFillForwardEnumerator GetLiveFillForwardEnumerator(ITimeProvider timeProvider, DateTime startTime, Resolution resolution,
+            out EnqueueableEnumerator<BaseData> enqueueableEnumerator, bool dailyStrictEndTimeEnabled, Resolution? ffResolution = null)
+        {
+            enqueueableEnumerator = new EnqueueableEnumerator<BaseData>();
+            var fillForwardEnumerator = new LiveFillForwardEnumerator(
+                timeProvider,
+                enqueueableEnumerator,
+                new SecurityExchange(MarketHoursDatabase.FromDataFolder()
+                    .ExchangeHoursListing
+                    .First(kvp => kvp.Key.Market == Market.USA && kvp.Key.SecurityType == SecurityType.Equity)
+                    .Value
+                    .ExchangeHours),
+                Ref.CreateReadOnly(() => (ffResolution ?? resolution).ToTimeSpan()),
+                false,
+                startTime,
+                Time.EndOfTime,
+                resolution,
+                TimeZones.NewYork,
+                dailyStrictEndTimeEnabled: dailyStrictEndTimeEnabled
+            );
+
+            return fillForwardEnumerator;
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/LiveSubscriptionEnumeratorTests.cs

@@ -0,0 +1,116 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using NodaTime;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Packets;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture, Parallelizable(ParallelScope.All)]
+    public class LiveSubscriptionEnumeratorTests
+    {
+        [Test]
+        public void HandlesSymbolMapping()
+        {
+            var canonical = Symbols.Fut_SPY_Feb19_2016.Canonical;
+            var dataQueue = new TestDataQueueHandler
+            {
+                DataPerSymbol = new Dictionary<Symbol, List<BaseData>>
+                {
+                    { Symbols.Fut_SPY_Feb19_2016,
+                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Feb19_2016, 1, 1)} },
+                    { Symbols.Fut_SPY_Mar19_2016,
+                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Mar19_2016, 2, 2)} },
+                }
+            };
+            var config = new SubscriptionDataConfig(typeof(Tick), canonical, Resolution.Tick,
+                DateTimeZone.Utc, DateTimeZone.Utc, false, false, false)
+            {
+                MappedSymbol = Symbols.Fut_SPY_Feb19_2016.ID.ToString()
+            };
+
+            var compositeDataQueueHandler = new TestDataQueueHandlerManager(new AlgorithmSettings());
+            compositeDataQueueHandler.ExposedDataHandlers.Add(dataQueue);
+            var data = new LiveSubscriptionEnumerator(config, compositeDataQueueHandler, (_, _) => {}, (_) => false);
+
+            Assert.IsTrue(data.MoveNext());
+            Assert.AreEqual(1, (data.Current as Tick).AskPrice);
+            Assert.AreEqual(canonical, (data.Current as Tick).Symbol);
+
+            Assert.IsFalse(data.MoveNext());
+            Assert.IsNull(data.Current);
+
+            Assert.AreEqual(2, dataQueue.DataPerSymbol.Count);
+            config.MappedSymbol = Symbols.Fut_SPY_Mar19_2016.ID.ToString();
+            Assert.AreEqual(1, dataQueue.DataPerSymbol.Count);
+
+            Assert.IsTrue(data.MoveNext());
+            Assert.AreEqual(2, (data.Current as Tick).AskPrice);
+            Assert.AreEqual(canonical, (data.Current as Tick).Symbol);
+            Assert.AreNotEqual(canonical, dataQueue.DataPerSymbol[Symbols.Fut_SPY_Mar19_2016].Single().Symbol);
+
+            Assert.IsFalse(data.MoveNext());
+            Assert.IsNull(data.Current);
+
+            Assert.AreEqual(1, dataQueue.DataPerSymbol.Count);
+
+            data.Dispose();
+            compositeDataQueueHandler.Dispose();
+        }
+
+        internal class TestDataQueueHandler : IDataQueueHandler
+        {
+            public bool IsConnected => true;
+
+            public Dictionary<Symbol, List<BaseData>> DataPerSymbol;
+
+            public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
+            {
+                if (DataPerSymbol.TryGetValue(dataConfig.Symbol, out var baseDatas))
+                {
+                    return baseDatas.GetEnumerator();
+                }
+                throw new Exception($"Failed to find a data enumerator for symbol {dataConfig.Symbol}!");
+            }
+            public void Unsubscribe(SubscriptionDataConfig dataConfig)
+            {
+                DataPerSymbol.Remove(dataConfig.Symbol);
+            }
+            public void SetJob(LiveNodePacket job)
+            {
+            }
+            public void Dispose()
+            {
+            }
+        }
+
+        private class TestDataQueueHandlerManager : DataQueueHandlerManager
+        {
+            public List<IDataQueueHandler> ExposedDataHandlers => DataHandlers;
+            public TestDataQueueHandlerManager(IAlgorithmSettings settings) : base(settings)
+            {
+            }
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/MappingEventProviderTests.cs

@@ -0,0 +1,84 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class MappingEventProviderTests
+    {
+        private SubscriptionDataConfig _config;
+
+        [SetUp]
+        public void SetUp()
+        {
+            var symbol = Symbol.Create("TFCFA", SecurityType.Equity, Market.USA);
+
+            _config = new SubscriptionDataConfig(typeof(TradeBar),
+                symbol,
+                Resolution.Daily,
+                TimeZones.NewYork,
+                TimeZones.NewYork,
+                true,
+                true,
+                false);
+        }
+
+        [Test]
+        public void InitialMapping()
+        {
+            var provider = new MappingEventProvider();
+
+            Assert.AreEqual("TFCFA", _config.MappedSymbol);
+
+            provider.Initialize(_config,
+                null,
+                TestGlobals.MapFileProvider,
+                new DateTime(2006, 1, 1));
+
+            Assert.AreEqual("NWSA", _config.MappedSymbol);
+        }
+
+        [Test]
+        public void MappingEvent()
+        {
+            var provider = new MappingEventProvider();
+            provider.Initialize(_config,
+                null,
+                TestGlobals.MapFileProvider,
+                new DateTime(2006, 1, 1));
+
+            Assert.AreEqual("NWSA", _config.MappedSymbol);
+
+            var symbolEvent = (SymbolChangedEvent)provider
+                .GetEvents(new NewTradableDateEventArgs(
+                    new DateTime(2013, 6, 29),
+                    null,
+                    _config.Symbol,
+                    null)).Single();
+
+            Assert.AreEqual("FOXA", symbolEvent.NewSymbol);
+            Assert.AreEqual("NWSA", symbolEvent.OldSymbol);
+            Assert.AreEqual("FOXA", _config.MappedSymbol);
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/PriceScaleFactorEnumeratorTests.cs

@@ -0,0 +1,362 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NUnit.Framework;
+using QuantConnect.Data;
+using System.Collections;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using Tick = QuantConnect.Data.Market.Tick;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using System.Linq;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class PriceScaleFactorEnumeratorTests
+    {
+        private SubscriptionDataConfig _config;
+        private RawDataEnumerator _rawDataEnumerator;
+
+        [SetUp]
+        public void Setup()
+        {
+            _config = GetConfig(Symbols.SPY, Resolution.Daily);
+            _rawDataEnumerator = new RawDataEnumerator();
+        }
+
+        [Test]
+        public void EquityTradeBar()
+        {
+            var enumerator = new PriceScaleFactorEnumerator(
+                _rawDataEnumerator,
+                _config,
+                TestGlobals.FactorFileProvider);
+            _rawDataEnumerator.CurrentValue = new TradeBar(
+                new DateTime(2018, 1, 1),
+                _config.Symbol,
+                10,
+                10,
+                10,
+                10,
+                100);
+            Assert.IsTrue(enumerator.MoveNext());
+            var tradeBar = enumerator.Current as TradeBar;
+            var expectedValue = 10 * _config.PriceScaleFactor;
+
+            Assert.Less(expectedValue, 10);
+            Assert.AreEqual(expectedValue, tradeBar.Price);
+            Assert.AreEqual(expectedValue, tradeBar.Open);
+            Assert.AreEqual(expectedValue, tradeBar.Close);
+            Assert.AreEqual(expectedValue, tradeBar.High);
+            Assert.AreEqual(expectedValue, tradeBar.Low);
+            Assert.AreEqual(expectedValue, tradeBar.Value);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void EquityQuoteBar()
+        {
+            var enumerator = new PriceScaleFactorEnumerator(
+                _rawDataEnumerator,
+                _config,
+                TestGlobals.FactorFileProvider);
+            _rawDataEnumerator.CurrentValue = new QuoteBar(
+                new DateTime(2018, 1, 1),
+                _config.Symbol,
+                new Bar(10, 10, 10, 10),
+                100,
+                new Bar(10, 10, 10, 10),
+                100);
+            Assert.IsTrue(enumerator.MoveNext());
+            var quoteBar = enumerator.Current as QuoteBar;
+            var expectedValue = 10 * _config.PriceScaleFactor;
+
+            Assert.Less(expectedValue, 10);
+
+            Assert.AreEqual(expectedValue, quoteBar.Price);
+            Assert.AreEqual(expectedValue, quoteBar.Value);
+            Assert.AreEqual(expectedValue, quoteBar.Open);
+            Assert.AreEqual(expectedValue, quoteBar.Close);
+            Assert.AreEqual(expectedValue, quoteBar.High);
+            Assert.AreEqual(expectedValue, quoteBar.Low);
+            // bid
+            Assert.AreEqual(expectedValue, quoteBar.Bid.Open);
+            Assert.AreEqual(expectedValue, quoteBar.Bid.Close);
+            Assert.AreEqual(expectedValue, quoteBar.Bid.High);
+            Assert.AreEqual(expectedValue, quoteBar.Bid.Low);
+            // ask
+            Assert.AreEqual(expectedValue, quoteBar.Ask.Open);
+            Assert.AreEqual(expectedValue, quoteBar.Ask.Close);
+            Assert.AreEqual(expectedValue, quoteBar.Ask.High);
+            Assert.AreEqual(expectedValue, quoteBar.Ask.Low);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void EquityTick()
+        {
+            var enumerator = new PriceScaleFactorEnumerator(
+                _rawDataEnumerator,
+                _config,
+                TestGlobals.FactorFileProvider);
+            _rawDataEnumerator.CurrentValue = new Tick(
+                new DateTime(2018, 1, 1),
+                _config.Symbol,
+                10,
+                10,
+                10);
+            Assert.IsTrue(enumerator.MoveNext());
+            var tick = enumerator.Current as Tick;
+            var expectedValue = 10 * _config.PriceScaleFactor;
+
+            Assert.Less(expectedValue, 10);
+            Assert.AreEqual(expectedValue, tick.Price);
+            Assert.AreEqual(expectedValue, tick.Value);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void FactorFileIsNull()
+        {
+            var enumerator = new PriceScaleFactorEnumerator(
+                _rawDataEnumerator,
+                _config,
+                null);
+            _rawDataEnumerator.CurrentValue = new Tick(
+                new DateTime(2018, 1, 1),
+                _config.Symbol,
+                10,
+                10,
+                10);
+            Assert.IsTrue(enumerator.MoveNext());
+            var tick = enumerator.Current as Tick;
+            Assert.AreEqual(10, tick.Price);
+            Assert.AreEqual(10, tick.Value);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void RawEnumeratorReturnsFalse()
+        {
+            var enumerator = new PriceScaleFactorEnumerator(
+                _rawDataEnumerator,
+                _config,
+                TestGlobals.FactorFileProvider);
+            _rawDataEnumerator.CurrentValue = new Tick(
+                new DateTime(2018, 1, 1),
+                _config.Symbol,
+                10,
+                10,
+                10);
+            _rawDataEnumerator.MoveNextReturnValue = false;
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.AreEqual(_rawDataEnumerator.CurrentValue, enumerator.Current);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void RawEnumeratorCurrentIsNull()
+        {
+            var enumerator = new PriceScaleFactorEnumerator(
+                _rawDataEnumerator,
+                _config,
+                TestGlobals.FactorFileProvider);
+            _rawDataEnumerator.CurrentValue = null;
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void UpdatesFactorFileCorrectly()
+        {
+            var dateBeforeUpadate = new DateTime(2018, 3, 14);
+            var dateAtUpadate = new DateTime(2018, 3, 15);
+            var dateAfterUpadate = new DateTime(2018, 3, 16);
+
+            var enumerator = new PriceScaleFactorEnumerator(
+                _rawDataEnumerator,
+                _config,
+                TestGlobals.FactorFileProvider);
+
+            // Before factor file update date (2018, 3, 15)
+            _rawDataEnumerator.CurrentValue = new Tick(
+                dateBeforeUpadate,
+                _config.Symbol,
+                10,
+                10,
+                10);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            var factorFile = TestGlobals.FactorFileProvider.Get(_config.Symbol);
+            var expectedFactor = factorFile.GetPriceFactor(dateBeforeUpadate, DataNormalizationMode.Adjusted);
+            var tick = enumerator.Current as Tick;
+            Assert.AreEqual(expectedFactor, _config.PriceScaleFactor);
+            Assert.AreEqual(10 * expectedFactor, tick.Price);
+            Assert.AreEqual(10 * expectedFactor, tick.Value);
+
+            // At factor file update date (2018, 3, 15)
+            _rawDataEnumerator.CurrentValue = new Tick(
+                dateAtUpadate,
+                _config.Symbol,
+                10,
+                10,
+                10);
+            Assert.IsTrue(enumerator.MoveNext());
+            var expectedFactor2 = factorFile.GetPriceFactor(dateAtUpadate, DataNormalizationMode.Adjusted);
+            var tick2 = enumerator.Current as Tick;
+            Assert.AreEqual(expectedFactor2, _config.PriceScaleFactor);
+            Assert.AreEqual(10 * expectedFactor2, tick2.Price);
+            Assert.AreEqual(10 * expectedFactor2, tick2.Value);
+
+            // After factor file update date (2018, 3, 15)
+            _rawDataEnumerator.CurrentValue = new Tick(
+                dateAfterUpadate,
+                _config.Symbol,
+                10,
+                10,
+                10);
+            Assert.IsTrue(enumerator.MoveNext());
+            var expectedFactor3 = factorFile.GetPriceFactor(dateAfterUpadate, DataNormalizationMode.Adjusted);
+            var tick3 = enumerator.Current as Tick;
+            Assert.AreEqual(expectedFactor3, _config.PriceScaleFactor);
+            Assert.AreEqual(10 * expectedFactor3, tick3.Price);
+            Assert.AreEqual(10 * expectedFactor3, tick3.Value);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void PricesAreProperlyAdjustedForLookAheadScaledRawDataNormalizationMode()
+        {
+            var factorFileEntries = new[]
+            {
+                new DateTime(2005, 02, 25),
+                new DateTime(2012, 08, 08),
+                new DateTime(2013, 05, 08),
+                new DateTime(2014, 08, 06),
+                new DateTime(2015, 08, 05)
+            };
+            var endDate = factorFileEntries.Last().AddDays(1);
+
+            var config = GetConfig(Symbols.AAPL, Resolution.Daily);
+            config.DataNormalizationMode = DataNormalizationMode.ScaledRaw;
+
+            using var enumerator = new PriceScaleFactorEnumerator(
+                _rawDataEnumerator,
+                config,
+                TestGlobals.FactorFileProvider,
+                endDate: endDate);
+
+            var price = 100m;
+            var factorFile = TestGlobals.FactorFileProvider.Get(config.Symbol);
+            var endDateFactor = factorFile.GetPriceFactor(endDate, config.DataNormalizationMode);
+
+            var performAssertions = (DateTime date) =>
+            {
+                var expectedFactor = factorFile.GetPriceFactor(date, config.DataNormalizationMode);
+                Assert.AreEqual(expectedFactor / endDateFactor, config.PriceScaleFactor);
+
+                var tradeBar = enumerator.Current as TradeBar;
+                var expectedValue = price * config.PriceScaleFactor;
+                Assert.AreEqual(expectedValue, tradeBar.Price);
+                Assert.AreEqual(expectedValue, tradeBar.Open);
+                Assert.AreEqual(expectedValue, tradeBar.Close);
+                Assert.AreEqual(expectedValue, tradeBar.High);
+                Assert.AreEqual(expectedValue, tradeBar.Low);
+                Assert.AreEqual(expectedValue, tradeBar.Value);
+
+                return expectedFactor;
+            };
+
+            foreach (var factorFileDate in factorFileEntries)
+            {
+                // before split
+                var dateBeforeSplit = factorFileDate.AddDays(-1);
+                _rawDataEnumerator.CurrentValue = new TradeBar(dateBeforeSplit, config.Symbol, price, price, price, price, price);
+                Assert.IsTrue(enumerator.MoveNext());
+                var expectedFactorBeforeSplit = performAssertions(dateBeforeSplit);
+
+                // at split
+                _rawDataEnumerator.CurrentValue = new TradeBar(factorFileDate, config.Symbol, price, price, price, price, price);
+                Assert.IsTrue(enumerator.MoveNext());
+                var expectedFactorAtSplit = performAssertions(factorFileDate);
+                Assert.AreEqual(expectedFactorBeforeSplit, expectedFactorAtSplit);
+
+                // after split
+                var dateAfterSplit = factorFileDate.AddDays(1);
+                _rawDataEnumerator.CurrentValue = new TradeBar(dateAfterSplit, config.Symbol, price, price, price, price, price);
+                Assert.IsTrue(enumerator.MoveNext());
+                var expectedFactorAfterSplit = performAssertions(dateAfterSplit);
+                Assert.AreNotEqual(expectedFactorAtSplit, expectedFactorAfterSplit);
+
+                if (factorFileDate == factorFileEntries.Last())
+                {
+                    // prices should have been adjusted to the end date prices, instead of the latest factor file entry (today),
+                    // So the last factor should be 1.
+                    Assert.AreEqual(1m, config.PriceScaleFactor);
+                }
+            }
+        }
+
+        private static SubscriptionDataConfig GetConfig(Symbol symbol, Resolution resolution)
+        {
+            return new SubscriptionDataConfig(typeof(TradeBar),
+                symbol,
+                resolution,
+                TimeZones.NewYork,
+                TimeZones.NewYork,
+                true,
+                true,
+                false);
+        }
+
+        private class RawDataEnumerator : IEnumerator<BaseData>
+        {
+            public bool MoveNextReturnValue { get; set; }
+            public BaseData CurrentValue { get; set; }
+
+            public BaseData Current => CurrentValue;
+
+            object IEnumerator.Current => CurrentValue;
+
+            public RawDataEnumerator()
+            {
+                MoveNextReturnValue = true;
+            }
+            public bool MoveNext()
+            {
+                return MoveNextReturnValue;
+            }
+
+            public void Reset()
+            {
+                throw new NotImplementedException();
+            }
+            public void Dispose()
+            {
+            }
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/QuoteBarFillForwardEnumeratorTests.cs

@@ -0,0 +1,63 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class QuoteBarFillForwardEnumeratorTests
+    {
+        [Test]
+        public void FillsForwardBidAskBars()
+        {
+            var bar1 = new QuoteBar
+            {
+                Bid = new Bar(3m, 4m, 1m, 2m),
+                Ask = new Bar(3.1m, 4.1m, 1.1m, 2.1m),
+            };
+
+            var bar2 = new QuoteBar
+            {
+                Bid = null,
+                Ask = null,
+            };
+
+            var data = new[] { bar1, bar2 }.ToList();
+            var enumerator = data.GetEnumerator();
+
+            var fillForwardEnumerator = new QuoteBarFillForwardEnumerator(enumerator);
+
+            // 9:31
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            var quoteBar1 = (QuoteBar)fillForwardEnumerator.Current;
+            Assert.AreSame(bar1.Bid, quoteBar1.Bid);
+            Assert.AreSame(bar1.Ask, quoteBar1.Ask);
+
+            // 9:32
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            var quoteBar2 = (QuoteBar)fillForwardEnumerator.Current;
+            Assert.AreSame(quoteBar1.Bid, quoteBar2.Bid);
+            Assert.AreSame(quoteBar1.Ask, quoteBar2.Ask);
+
+            fillForwardEnumerator.Dispose();
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/RateLimitEnumeratorTests.cs

@@ -0,0 +1,62 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Globalization;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class RateLimitEnumeratorTests
+    {
+        [Test]
+        public void LimitsBasedOnTimeBetweenCalls()
+        {
+            var currentTime = new DateTime(2015, 10, 10, 13, 6, 0);
+            var timeProvider = new ManualTimeProvider(currentTime, TimeZones.Utc);
+            var data = Enumerable.Range(0, 100).Select(x => new Tick {Symbol = CreateSymbol(x)}).GetEnumerator();
+            var rateLimit = new RateLimitEnumerator<BaseData>(data, timeProvider, Time.OneSecond);
+
+            Assert.IsTrue(rateLimit.MoveNext());
+
+            while (rateLimit.MoveNext() && rateLimit.Current == null)
+            {
+                timeProvider.AdvanceSeconds(0.1);
+            }
+
+            var delta = (timeProvider.GetUtcNow() - currentTime).TotalSeconds;
+
+            Assert.AreEqual(1, delta);
+
+            Assert.AreEqual("1", data.Current.Symbol.Value);
+
+            rateLimit.Dispose();
+        }
+
+        private static Symbol CreateSymbol(int x)
+        {
+            return new Symbol(
+                SecurityIdentifier.GenerateBase(null, x.ToString(CultureInfo.InvariantCulture), Market.USA),
+                 x.ToString(CultureInfo.InvariantCulture));
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/RefreshEnumeratorTests.cs

@@ -0,0 +1,156 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Moq;
+using System.IO;
+using NUnit.Framework;
+using System.Collections.Generic;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class RefreshEnumeratorTests
+    {
+        [Test]
+        public void StaleFileHandleException()
+        {
+            var fakeEnumerator = new Mock<IEnumerator<int?>>();
+            fakeEnumerator.Setup(e => e.MoveNext()).Throws(new IOException("stale file handle"));
+            fakeEnumerator.Setup(e => e.Dispose()).Verifiable();
+            var refresher = new RefreshEnumerator<int?>(() => fakeEnumerator.Object);
+
+            // does not throw exception but disposes of enumerator
+            refresher.MoveNext();
+
+            fakeEnumerator.Verify(enumerator => enumerator.Dispose(), Times.Once);
+
+            refresher.Dispose();
+        }
+
+        [Test]
+        public void RefreshesEnumeratorOnFirstMoveNext()
+        {
+            var refreshed = false;
+            var refresher = new RefreshEnumerator<int?>(() =>
+            {
+                refreshed = true;
+                return new List<int?>().GetEnumerator();
+            });
+
+            refresher.MoveNext();
+            Assert.IsTrue(refreshed);
+
+            refresher.Dispose();
+        }
+
+        [Test]
+        public void MoveNextReturnsTrueWhenUnderlyingEnumeratorReturnsFalse()
+        {
+            var refresher = new RefreshEnumerator<int?>(() => new List<int?>().GetEnumerator());
+            Assert.IsTrue(refresher.MoveNext());
+
+            refresher.Dispose();
+        }
+
+        [Test]
+        public void CurrentIsDefault_T_WhenUnderlyingEnumeratorReturnsFalse()
+        {
+            var refresher = new RefreshEnumerator<int?>(() => new List<int?>().GetEnumerator());
+            refresher.MoveNext();
+            Assert.AreEqual(default(int?), refresher.Current);
+
+            refresher.Dispose();
+        }
+
+        [Test]
+        public void UnderlyingEnumeratorDisposed_WhenUnderlyingEnumeratorReturnsFalse()
+        {
+            var fakeEnumerator = new Mock<IEnumerator<int?>>();
+            fakeEnumerator.Setup(e => e.MoveNext()).Returns(false);
+            fakeEnumerator.Setup(e => e.Dispose()).Verifiable();
+            var refresher = new RefreshEnumerator<int?>(() => fakeEnumerator.Object);
+            refresher.MoveNext();
+
+            fakeEnumerator.Verify(enumerator => enumerator.Dispose(), Times.Once);
+
+            refresher.Dispose();
+        }
+
+        [Test]
+        public void DisposeCallsUnderlyingDispose()
+        {
+            var fakeEnumerator = new Mock<IEnumerator<int?>>();
+            fakeEnumerator.Setup(e => e.MoveNext()).Returns(true);
+            fakeEnumerator.Setup(e => e.Dispose()).Verifiable();
+            var refresher = new RefreshEnumerator<int?>(() => fakeEnumerator.Object);
+            refresher.MoveNext();
+            refresher.Dispose();
+
+            fakeEnumerator.Verify(enumerator => enumerator.Dispose(), Times.Once);
+
+            refresher.Dispose();
+        }
+
+        [Test]
+        public void ResetCallsUnderlyingReset()
+        {
+            var fakeEnumerator = new Mock<IEnumerator<int?>>();
+            fakeEnumerator.Setup(e => e.MoveNext()).Returns(true);
+            fakeEnumerator.Setup(e => e.Reset()).Verifiable();
+            var refresher = new RefreshEnumerator<int?>(() => fakeEnumerator.Object);
+            refresher.MoveNext();
+            refresher.Reset();
+
+            fakeEnumerator.Verify(enumerator => enumerator.Reset(), Times.Once);
+
+            refresher.Dispose();
+        }
+
+        [Test]
+        public void RefreshesAfterMoveNextReturnsFalse()
+        {
+            var refreshCount = 0;
+            var list = new List<int?> {1, 2};
+            var refresher = new RefreshEnumerator<int?>(() =>
+            {
+                refreshCount++;
+                return list.GetEnumerator();
+            });
+
+            Assert.IsTrue(refresher.MoveNext());
+            Assert.AreEqual(1, refreshCount);
+            Assert.AreEqual(1, refresher.Current);
+
+            Assert.IsTrue(refresher.MoveNext());
+            Assert.AreEqual(1, refreshCount);
+            Assert.AreEqual(2, refresher.Current);
+
+            Assert.IsTrue(refresher.MoveNext());
+            Assert.AreEqual(1, refreshCount);
+            Assert.AreEqual(default(int?), refresher.Current);
+
+            Assert.IsTrue(refresher.MoveNext());
+            Assert.AreEqual(2, refreshCount);
+            Assert.AreEqual(1, refresher.Current);
+
+            Assert.IsTrue(refresher.MoveNext());
+            Assert.AreEqual(2, refreshCount);
+            Assert.AreEqual(2, refresher.Current);
+
+            refresher.Dispose();
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/ScannableEnumeratorTests.cs

@@ -0,0 +1,355 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NodaTime;
+using NUnit.Framework;
+using QuantConnect.Data.Consolidators;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture, Parallelizable(ParallelScope.All)]
+    public class ScannableEnumeratorTests
+    {
+        [Test]
+        public void PassesTicksStraightThrough()
+        {
+            var currentTime = new DateTime(2000, 01, 01);
+            using var identityDataConsolidator = new IdentityDataConsolidator<Tick>();
+            var enumerator = new ScannableEnumerator<Tick>(
+                identityDataConsolidator,
+                DateTimeZone.ForOffset(Offset.FromHours(-5)),
+                new ManualTimeProvider(currentTime),
+                (s, e) => { }
+            );
+
+            // returns true even if no data present until stop is called
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) { Quantity = 10 };
+            enumerator.Update(tick1);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(tick1, enumerator.Current);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick2 = new Tick(currentTime, Symbols.SPY, 199.56m, 199.21m, 200.02m) { Quantity = 5 };
+            enumerator.Update(tick2);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(tick2, enumerator.Current);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void NewDataAvailableShouldFire()
+        {
+            var currentTime = new DateTime(2000, 01, 01);
+            var available = false;
+            using var identityDataConsolidator = new IdentityDataConsolidator<Tick>();
+            var enumerator = new ScannableEnumerator<Tick>(
+                identityDataConsolidator,
+                DateTimeZone.ForOffset(Offset.FromHours(-5)),
+                new ManualTimeProvider(currentTime),
+                (s, e) => { available = true; }
+            );
+
+            // returns true even if no data present until stop is called
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+            Assert.IsFalse(available);
+
+            var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) { Quantity = 10 };
+            enumerator.Update(tick1);
+            Assert.IsTrue(available);
+
+            enumerator.Dispose();
+        }
+
+        [Test]
+        public void AggregatesNewQuoteBarProperly()
+        {
+            var reference = DateTime.Today;
+
+            using var tickQuoteBarConsolidator = new TickQuoteBarConsolidator(4);
+            using var enumerator = new ScannableEnumerator<Data.BaseData>(
+                tickQuoteBarConsolidator,
+                DateTimeZone.ForOffset(Offset.FromHours(-5)),
+                new ManualTimeProvider(reference),
+                (s, e) => { }
+            );
+
+            var tick1 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference,
+                BidPrice = 10,
+                BidSize = 20,
+                TickType = TickType.Quote
+            };
+            enumerator.Update(tick1);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick2 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.AddMinutes(1),
+                AskPrice = 20,
+                AskSize = 10,
+                TickType = TickType.Quote
+            };
+
+            var badTick = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.AddMinutes(1),
+                AskPrice = 25,
+                AskSize = 100,
+                BidPrice = -100,
+                BidSize = 2,
+                Value = 50,
+                Quantity = 1234,
+                TickType = TickType.Trade
+            };
+            enumerator.Update(badTick);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            enumerator.Update(tick2);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick3 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.AddMinutes(2),
+                BidPrice = 12,
+                BidSize = 50,
+                TickType = TickType.Quote
+            };
+            enumerator.Update(tick3);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick4 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.AddMinutes(3),
+                AskPrice = 17,
+                AskSize = 15,
+                TickType = TickType.Quote
+            };
+            enumerator.Update(tick4);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current);
+
+            QuoteBar quoteBar = enumerator.Current as QuoteBar;
+            Assert.IsNotNull(quoteBar);
+
+            Assert.AreEqual(Symbols.SPY, quoteBar.Symbol);
+            Assert.AreEqual(tick1.Time, quoteBar.Time);
+            Assert.AreEqual(tick4.EndTime, quoteBar.EndTime);
+            Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open);
+            Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Low);
+            Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.High);
+            Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.Close);
+            Assert.AreEqual(tick3.BidSize, quoteBar.LastBidSize);
+
+            Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open);
+            Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Low);
+            Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.High);
+            Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Close);
+            Assert.AreEqual(tick4.AskSize, quoteBar.LastAskSize);
+        }
+
+        [Test]
+        public void ForceScanQuoteBar()
+        {
+            var reference = new DateTime(2020, 2, 2, 1, 0, 0);
+            var timeProvider = new ManualTimeProvider(reference);
+            var dateTimeZone = DateTimeZone.ForOffset(Offset.FromHours(-5));
+            using var tickQuoteBarConsolidator = new TickQuoteBarConsolidator(TimeSpan.FromMinutes(1));
+            using var enumerator = new ScannableEnumerator<Data.BaseData>(
+                tickQuoteBarConsolidator,
+                dateTimeZone,
+                timeProvider,
+                (s, e) => { }
+            );
+
+            var tick1 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.ConvertFromUtc(dateTimeZone),
+                BidPrice = 10,
+                BidSize = 20,
+                TickType = TickType.Quote
+            };
+            enumerator.Update(tick1);
+
+            var tick2 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.AddSeconds(1).ConvertFromUtc(dateTimeZone),
+                AskPrice = 20,
+                AskSize = 10,
+                TickType = TickType.Quote
+            };
+
+            enumerator.Update(tick2);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick3 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.AddSeconds(2).ConvertFromUtc(dateTimeZone),
+                BidPrice = 12,
+                BidSize = 50,
+                TickType = TickType.Quote
+            };
+            enumerator.Update(tick3);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick4 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.AddSeconds(3).ConvertFromUtc(dateTimeZone),
+                AskPrice = 17,
+                AskSize = 15,
+                TickType = TickType.Quote
+            };
+            enumerator.Update(tick4);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            timeProvider.AdvanceSeconds(120);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current);
+            QuoteBar quoteBar = enumerator.Current as QuoteBar;
+            Assert.IsNotNull(quoteBar);
+
+            Assert.AreEqual(Symbols.SPY, quoteBar.Symbol);
+            Assert.AreEqual(tick1.Time, quoteBar.Time);
+            Assert.AreNotEqual(tick4.EndTime, quoteBar.EndTime);
+            Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open);
+            Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Low);
+            Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.High);
+            Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.Close);
+            Assert.AreEqual(tick3.BidSize, quoteBar.LastBidSize);
+
+            Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open);
+            Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Low);
+            Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.High);
+            Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Close);
+            Assert.AreEqual(tick4.AskSize, quoteBar.LastAskSize);
+        }
+
+        [Test]
+        public void MoveNextScanQuoteBar()
+        {
+            var offset = Offset.FromHours(-5);
+            var timeZone = DateTimeZone.ForOffset(offset);
+            var utc = new DateTimeOffset(DateTime.Today);
+            var reference = utc.ToOffset(offset.ToTimeSpan());
+            var timeProvider = new ManualTimeProvider(reference.DateTime, timeZone);
+
+            using var tickQuoteBarConsolidator = new TickQuoteBarConsolidator(TimeSpan.FromMinutes(1));
+            using var enumerator = new ScannableEnumerator<Data.BaseData>(
+                tickQuoteBarConsolidator,
+                timeZone,
+                timeProvider,
+                (s, e) => { }
+            );
+
+            var tick1 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.DateTime,
+                BidPrice = 10,
+                BidSize = 20,
+                TickType = TickType.Quote
+            };
+            enumerator.Update(tick1);
+
+            var tick2 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.DateTime.AddSeconds(1),
+                AskPrice = 20,
+                AskSize = 10,
+                TickType = TickType.Quote
+            };
+
+            enumerator.Update(tick2);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick3 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.DateTime.AddSeconds(2),
+                BidPrice = 12,
+                BidSize = 50,
+                TickType = TickType.Quote
+            };
+            enumerator.Update(tick3);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var tick4 = new Tick
+            {
+                Symbol = Symbols.SPY,
+                Time = reference.DateTime.AddSeconds(3),
+                AskPrice = 17,
+                AskSize = 15,
+                TickType = TickType.Quote
+            };
+            enumerator.Update(tick4);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            timeProvider.SetCurrentTime(reference.DateTime.AddMinutes(2));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current);
+            QuoteBar quoteBar = enumerator.Current as QuoteBar;
+            Assert.IsNotNull(quoteBar);
+
+            Assert.AreEqual(Symbols.SPY, quoteBar.Symbol);
+            Assert.AreEqual(tick1.Time, quoteBar.Time);
+            Assert.AreNotEqual(tick4.EndTime, quoteBar.EndTime);
+            Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open);
+            Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Low);
+            Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.High);
+            Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.Close);
+            Assert.AreEqual(tick3.BidSize, quoteBar.LastBidSize);
+
+            Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open);
+            Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Low);
+            Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.High);
+            Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Close);
+            Assert.AreEqual(tick4.AskSize, quoteBar.LastAskSize);
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/ScheduledEnumeratorTests.cs

@@ -0,0 +1,393 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture, Parallelizable(ParallelScope.All)]
+    public class ScheduledEnumeratorTests
+    {
+        private readonly DateTime _referenceTime = new DateTime(2019, 1, 1);
+
+        [TestCase(true)]
+        [TestCase(false)]
+        public void RespectsPredicateTimeProvider(bool newDataArrivedInTime)
+        {
+            var scheduledDate = _referenceTime.AddDays(1);
+            using var underlyingEnumerator = new TestEnumerator
+            {
+                MoveNextReturn = true,
+                MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
+                {
+                    new Tick(scheduledDate.AddDays(-1), Symbols.SPY, 1, 1)
+                })
+            };
+            var timeProvider = new ManualTimeProvider(_referenceTime);
+
+            using var enumerator = new ScheduledEnumerator(
+                underlyingEnumerator,
+                new List<DateTime> { scheduledDate },
+                new PredicateTimeProvider(timeProvider, (currentDateTime) => {
+                    // will only let time advance after it's passed the 7/8 hour frontier
+                    return currentDateTime.TimeOfDay > TimeSpan.FromMinutes(7 * 60 + DateTime.UtcNow.Second);
+                }),
+                TimeZones.Utc,
+                DateTime.MinValue);
+
+            // still null since frontier is still behind schedule
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            timeProvider.SetCurrentTimeUtc(scheduledDate);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            timeProvider.SetCurrentTimeUtc(scheduledDate.AddHours(2));
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            if (newDataArrivedInTime)
+            {
+                // New data comes in!
+                underlyingEnumerator.MoveNextNewValues.Enqueue(new Tick(scheduledDate, Symbols.SPY, 10, 10));
+            }
+
+            timeProvider.SetCurrentTimeUtc(scheduledDate.AddHours(8));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(scheduledDate, enumerator.Current.Time);
+            Assert.AreEqual(newDataArrivedInTime ? 10 : 1, (enumerator.Current as Tick).BidPrice);
+
+            // schedule ended so enumerator will end too
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        [Test]
+        public void ScheduleSkipsOldDates()
+        {
+            using var testEnumerator = new TestEnumerator();
+            using var enumerator = new ScheduledEnumerator(
+                testEnumerator,
+                new List<DateTime> { _referenceTime },
+                new ManualTimeProvider(_referenceTime),
+                TimeZones.Utc,
+                _referenceTime.AddDays(1));
+
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        [Test]
+        public void EmptyScheduleThrowsNoException()
+        {
+            ScheduledEnumerator enumerator = null;
+            Assert.DoesNotThrow(() => enumerator = new ScheduledEnumerator(
+                new TestEnumerator(),
+                new List<DateTime>(),
+                new ManualTimeProvider(_referenceTime),
+                TimeZones.Utc,
+                DateTime.MinValue));
+
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        [Test]
+        public void ReturnsTrueEvenIfUnderlyingIsNullButReturnsTrue()
+        {
+            using var underlyingEnumerator = new TestEnumerator { MoveNextReturn = true };
+            var timeProvider = new ManualTimeProvider(_referenceTime);
+
+            using var enumerator = new ScheduledEnumerator(
+                underlyingEnumerator,
+                new List<DateTime> { _referenceTime.AddDays(1) },
+                timeProvider,
+                TimeZones.Utc,
+                DateTime.MinValue);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        [Test]
+        public void ReturnsFalseWhenUnderlyingReturnsFalse()
+        {
+            using var underlyingEnumerator = new TestEnumerator { MoveNextReturn = false };
+            var timeProvider = new ManualTimeProvider(_referenceTime);
+
+            using var enumerator = new ScheduledEnumerator(
+                underlyingEnumerator,
+                new List<DateTime> { _referenceTime.AddDays(1) },
+                timeProvider,
+                TimeZones.Utc,
+                DateTime.MinValue);
+
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        [Test]
+        public void ForwardsDataToFitSchedule()
+        {
+            var scheduledDate = _referenceTime.AddDays(1);
+            using var underlyingEnumerator = new TestEnumerator
+            {
+                MoveNextReturn = true,
+                MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
+                {
+                    new Tick(scheduledDate, Symbols.SPY, 1, 1),
+                    // way in the future compared with the schedule
+                    new Tick(scheduledDate.AddYears(1), Symbols.SPY, 10, 10)
+                })
+            };
+            var timeProvider = new ManualTimeProvider(_referenceTime);
+
+            using var enumerator = new ScheduledEnumerator(
+                underlyingEnumerator,
+                new List<DateTime> { scheduledDate, scheduledDate.AddDays(1) },
+                timeProvider,
+                TimeZones.Utc,
+                DateTime.MinValue);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            // still null since frontier is still behind schedule
+            Assert.IsNull(enumerator.Current);
+
+            timeProvider.SetCurrentTimeUtc(scheduledDate);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(scheduledDate, enumerator.Current.Time);
+            Assert.AreEqual(1, (enumerator.Current as Tick).BidPrice);
+
+            // it will forward previous available value to fit the schedule
+            timeProvider.SetCurrentTimeUtc(scheduledDate.AddDays(1));
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(scheduledDate.AddDays(1), enumerator.Current.Time);
+            Assert.AreEqual(1, (enumerator.Current as Tick).BidPrice);
+
+            // schedule ended so enumerator will end too
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        [Test]
+        public void UpdatesCurrentBasedOnSchedule()
+        {
+            var scheduledDate = _referenceTime.AddDays(1);
+            using var underlyingEnumerator = new TestEnumerator
+            {
+                MoveNextReturn = true,
+                MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
+                {
+                    new Tick(scheduledDate, Symbols.SPY, 1, 1)
+                })
+            };
+            var timeProvider = new ManualTimeProvider(_referenceTime);
+
+            using var enumerator = new ScheduledEnumerator(
+                underlyingEnumerator,
+                new List<DateTime> { scheduledDate },
+                timeProvider,
+                TimeZones.Utc,
+                DateTime.MinValue);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            // still null since frontier is still behind schedule
+            Assert.IsNull(enumerator.Current);
+
+            timeProvider.SetCurrentTimeUtc(scheduledDate);
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(scheduledDate, enumerator.Current.Time);
+            Assert.AreEqual(1, (enumerator.Current as Tick).BidPrice);
+
+            // schedule ended so enumerator will end too
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        [Test]
+        public void WillUseLatestDataPoint()
+        {
+            using var underlyingEnumerator = new TestEnumerator
+            {
+                MoveNextReturn = true,
+                MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
+                {
+                    new Tick(new DateTime(2019, 1, 15), Symbols.SPY, 1, 1),
+                    new Tick(new DateTime(2019, 1, 20), Symbols.SPY, 2, 1),
+                    new Tick(new DateTime(2019, 1, 25), Symbols.SPY, 3, 1)
+                })
+            };
+            var timeProvider = new ManualTimeProvider(_referenceTime);
+
+            using var enumerator = new ScheduledEnumerator(
+                underlyingEnumerator,
+                new List<DateTime> { new DateTime(2019, 2, 1) },
+                timeProvider,
+                TimeZones.Utc,
+                DateTime.MinValue);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            // still null since frontier is still behind schedule
+            Assert.IsNull(enumerator.Current);
+
+            // frontier is now a month after the scheduled time!
+            timeProvider.SetCurrentTimeUtc(new DateTime(2019, 3, 1));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            // it uses the last available data point in the enumerator
+            Assert.AreEqual(new DateTime(2019, 2, 1), enumerator.Current.Time);
+            Assert.AreEqual(3, (enumerator.Current as Tick).BidPrice);
+
+            Assert.IsNull(underlyingEnumerator.Current);
+        }
+
+        [Test]
+        public void WillUseLatestDataPointOnlyIfBeforeOrAtSchedule()
+        {
+            using var underlyingEnumerator = new TestEnumerator
+            {
+                MoveNextReturn = true,
+                MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
+                {
+                    new Tick(new DateTime(2019, 1, 20), Symbols.SPY, 2, 1),
+                    new Tick(new DateTime(2019, 1, 25), Symbols.SPY, 3, 1),
+                    // this guys is in 2020
+                    new Tick(new DateTime(2020, 1, 1), Symbols.SPY, 4, 1)
+                })
+            };
+            var timeProvider = new ManualTimeProvider(_referenceTime);
+
+            using var enumerator = new ScheduledEnumerator(
+                underlyingEnumerator,
+                new List<DateTime> { new DateTime(2019, 2, 1), new DateTime(2020, 2, 1) },
+                timeProvider,
+                TimeZones.Utc,
+                DateTime.MinValue);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            // still null since frontier is still behind schedule
+            Assert.IsNull(enumerator.Current);
+
+            // frontier is now a month after the scheduled time!
+            timeProvider.SetCurrentTimeUtc(new DateTime(2019, 3, 1));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            // it uses the last available data point in the enumerator that is before the schedule
+            Assert.AreEqual(new DateTime(2019, 2, 1), enumerator.Current.Time);
+            Assert.AreEqual(3, (enumerator.Current as Tick).BidPrice);
+
+            // the underlying enumerator hold the next data point
+            Assert.AreEqual(new DateTime(2020, 1, 1), underlyingEnumerator.Current.Time);
+
+            // now lets test fetching the last data point
+            timeProvider.SetCurrentTimeUtc(new DateTime(2021, 3, 1));
+
+            // the underlying will end but should still emit the data point it has
+            underlyingEnumerator.MoveNextReturn = false;
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(new DateTime(2020, 2, 1), enumerator.Current.Time);
+            Assert.AreEqual(4, (enumerator.Current as Tick).BidPrice);
+
+            Assert.IsNull(underlyingEnumerator.Current);
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        [Test]
+        public void NoTimeProvider()
+        {
+            using var underlyingEnumerator = new TestEnumerator
+            {
+                MoveNextReturn = true,
+                MoveNextNewValues = new Queue<BaseData>(new List<BaseData>
+                {
+                    new Tick(new DateTime(2019, 1, 20), Symbols.SPY, 2, 1),
+                    new Tick(new DateTime(2019, 1, 25), Symbols.SPY, 3, 1),
+
+                    new Tick(new DateTime(2020, 1, 1), Symbols.SPY, 4, 1)
+                })
+            };
+            using var enumerator = new ScheduledEnumerator(
+                underlyingEnumerator,
+                new List<DateTime> { new DateTime(2019, 2, 1), new DateTime(2020, 2, 1) },
+                null,
+                TimeZones.Utc,
+                DateTime.MinValue);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            // it uses the last available data point in the enumerator that is before the schedule
+            Assert.AreEqual(new DateTime(2019, 2, 1), enumerator.Current.Time);
+            Assert.AreEqual(3, (enumerator.Current as Tick).BidPrice);
+
+            // the underlying enumerator hold the next data point
+            Assert.AreEqual(new DateTime(2020, 1, 1), underlyingEnumerator.Current.Time);
+
+            // the underlying will end but should still emit the data point it has
+            underlyingEnumerator.MoveNextReturn = false;
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(new DateTime(2020, 2, 1), enumerator.Current.Time);
+            Assert.AreEqual(4, (enumerator.Current as Tick).BidPrice);
+
+            Assert.IsNull(underlyingEnumerator.Current);
+            Assert.IsFalse(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        private class TestEnumerator : IEnumerator<BaseData>
+        {
+            public Queue<BaseData> MoveNextNewValues { get; set; }
+            public BaseData Current { get; private set; }
+
+            object IEnumerator.Current => Current;
+
+            public bool MoveNextReturn { get; set; }
+
+            public bool MoveNext()
+            {
+                if (MoveNextNewValues != null && MoveNextNewValues.Count > 0)
+                {
+                    Current = MoveNextNewValues.Dequeue();
+                }
+                else
+                {
+                    Current = null;
+                }
+                return MoveNextReturn;
+            }
+
+            public void Reset()
+            {}
+            public void Dispose()
+            {}
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/SubscriptionDataEnumeratorTests.cs

@@ -0,0 +1,89 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class SubscriptionDataEnumeratorTests
+    {
+
+        [TestCase(typeof(TradeBar), true)]
+        [TestCase(typeof(OpenInterest), false)]
+        [TestCase(typeof(QuoteBar), false)]
+        public void EnumeratorEmitsAuxData(Type typeOfConfig, bool shouldReceiveAuxData)
+        {
+            var config = CreateConfig(Resolution.Hour, typeOfConfig);
+            var security = GetSecurity(config);
+            var time = new DateTime(2010, 1, 1);
+            var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, time, time.AddDays(1));
+
+
+            // Make a aux data stream; for this testing case we will just use delisting data points
+            var totalPoints = 8;
+            var stream = Enumerable.Range(0, totalPoints).Select(x => new Delisting { Time = time.AddHours(x) }).GetEnumerator();
+            using var enumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, tzOffsetProvider, stream, false, false);
+
+            // Test our SubscriptionDataEnumerator to see if it emits the aux data
+            int dataReceivedCount = 0;
+            while (enumerator.MoveNext())
+            {
+                dataReceivedCount++;
+                if (enumerator.Current != null && enumerator.Current.Data.DataType == MarketDataType.Auxiliary)
+                {
+                    Assert.IsTrue(shouldReceiveAuxData);
+                }
+            }
+
+            // If it should receive aux data it should have emitted all points
+            // otherwise none should have been emitted
+            if (shouldReceiveAuxData)
+            {
+                Assert.AreEqual(totalPoints, dataReceivedCount);
+            }
+            else
+            {
+                Assert.AreEqual(0, dataReceivedCount);
+            }
+        }
+
+
+        private static Security GetSecurity(SubscriptionDataConfig config)
+        {
+            return new Security(
+                SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
+                config,
+                new Cash(Currencies.USD, 0, 1m),
+                SymbolProperties.GetDefault(Currencies.USD),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCache()
+            );
+        }
+
+        private static SubscriptionDataConfig CreateConfig(Resolution resolution, Type type)
+        {
+            return new SubscriptionDataConfig(type, Symbols.SPY, resolution, TimeZones.NewYork, TimeZones.NewYork, true, true, false);
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/SynchronizingBaseDataEnumeratorTests.cs

@@ -0,0 +1,147 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class SynchronizingBaseDataEnumeratorTests
+    {
+        [Test]
+        public void SynchronizesData()
+        {
+            var time = new DateTime(2016, 03, 03, 12, 05, 00);
+            var stream1 = Enumerable.Range(0, 10).Select(x => new Tick {Time = time.AddSeconds(x * 1)}).GetEnumerator();
+            var stream2 = Enumerable.Range(0, 5).Select(x => new Tick {Time = time.AddSeconds(x * 2)}).GetEnumerator();
+            var stream3 = Enumerable.Range(0, 20).Select(x => new Tick {Time = time.AddSeconds(x * 0.5)}).GetEnumerator();
+
+            var previous = DateTime.MinValue;
+            var synchronizer = new SynchronizingBaseDataEnumerator(stream1, stream2, stream3);
+            while (synchronizer.MoveNext())
+            {
+                Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous));
+                previous = synchronizer.Current.EndTime;
+            }
+
+            synchronizer.Dispose();
+        }
+
+        [Test]
+        public void WontRemoveEnumeratorsReturningTrueWithCurrentNull()
+        {
+            var time = new DateTime(2016, 03, 03, 12, 05, 00);
+            var stream1 = Enumerable.Range(0, 20)
+                // return null except the last value and check if its emitted
+                .Select(x => x == 19 ? new Tick {Time = time.AddSeconds(x * 100), Quantity = 998877} : null
+            ).GetEnumerator();
+            var stream2 = Enumerable.Range(0, 5).Select(x => new Tick { Time = time.AddSeconds(x * 2) }).GetEnumerator();
+            var stream3 = Enumerable.Range(0, 20).Select(x => new Tick { Time = time.AddSeconds(x * 0.5) }).GetEnumerator();
+
+            var previous = new Tick { Time = DateTime.MinValue };
+            var synchronizer = new SynchronizingBaseDataEnumerator(stream1, stream2, stream3);
+            while (synchronizer.MoveNext())
+            {
+                Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous.EndTime));
+                previous = synchronizer.Current as Tick;
+            }
+            Assert.AreEqual(998877, previous.Quantity);
+
+            synchronizer.Dispose();
+        }
+
+        [Test]
+        public void WillRemoveEnumeratorsReturningFalse()
+        {
+            var time = new DateTime(2016, 03, 03, 12, 05, 00);
+            var stream1 = new TestEnumerator { MoveNextReturnValue = false };
+            var stream2 = Enumerable.Range(0, 10).Select(x => new Tick { Time = time.AddSeconds(x * 2) }).GetEnumerator();
+            var synchronizer = new SynchronizingBaseDataEnumerator(stream1, stream2);
+            var emitted = false;
+            while (synchronizer.MoveNext())
+            {
+                emitted = true;
+            }
+            Assert.IsTrue(emitted);
+            Assert.IsTrue(stream1.MoveNextWasCalled);
+            Assert.AreEqual(1, stream1.MoveNextCallCount);
+
+            synchronizer.Dispose();
+        }
+
+        [Test]
+        public void WillStopIfAllEnumeratorsCurrentIsNullAndReturningTrue()
+        {
+            var stream1 = new TestEnumerator { MoveNextReturnValue = true };
+            var synchronizer = new SynchronizingBaseDataEnumerator(stream1);
+            while (synchronizer.MoveNext())
+            {
+                Assert.Fail();
+            }
+            Assert.IsTrue(stream1.MoveNextWasCalled);
+            Assert.Pass();
+
+            synchronizer.Dispose();
+        }
+
+        [Test]
+        public void WillStopIfAllEnumeratorsCurrentIsNullAndReturningFalse()
+        {
+            var stream1 = new TestEnumerator { MoveNextReturnValue = false };
+            var synchronizer = new SynchronizingBaseDataEnumerator(stream1);
+            while (synchronizer.MoveNext())
+            {
+                Assert.Fail();
+            }
+            Assert.IsTrue(stream1.MoveNextWasCalled);
+            Assert.Pass();
+
+            synchronizer.Dispose();
+        }
+
+        private class TestEnumerator : IEnumerator<BaseData>
+        {
+            public int MoveNextCallCount { get; set; }
+            public bool MoveNextReturnValue { get; set; }
+            public bool MoveNextWasCalled { get; set; }
+
+            public TestEnumerator()
+            {
+                MoveNextWasCalled = false;
+            }
+
+            public bool MoveNext()
+            {
+                MoveNextCallCount++;
+                MoveNextWasCalled = true;
+                return MoveNextReturnValue;
+            }
+
+            public BaseData Current { get; }
+
+            object IEnumerator.Current => Current;
+            public void Dispose() { }
+            public void Reset() { }
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/SynchronizingSliceEnumeratorTests.cs

@@ -0,0 +1,146 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class SynchronizingSliceEnumeratorTests
+    {
+        [Test]
+        public void SynchronizesData()
+        {
+            var time = new DateTime(2016, 03, 03, 12, 05, 00);
+            var stream1 = Enumerable.Range(0, 10).Select(x => new Slice(time.AddSeconds(x * 1), new List<BaseData>(), utcTime: time.AddSeconds(x * 1))).GetEnumerator();
+            var stream2 = Enumerable.Range(0, 5).Select(x => new Slice(time.AddSeconds(x * 2), new List<BaseData>(), utcTime: time.AddSeconds(x * 2))).GetEnumerator();
+            var stream3 = Enumerable.Range(0, 20).Select(x => new Slice(time.AddSeconds(x * 0.5), new List<BaseData>(), utcTime: time.AddSeconds(x * 0.5))).GetEnumerator();
+
+            var previous = DateTime.MinValue;
+            var synchronizer = new SynchronizingSliceEnumerator(stream1, stream2, stream3);
+            while (synchronizer.MoveNext())
+            {
+                Assert.That(synchronizer.Current.UtcTime, Is.GreaterThanOrEqualTo(previous));
+                previous = synchronizer.Current.UtcTime;
+            }
+            synchronizer.Dispose();
+        }
+
+        [Test]
+        public void WontRemoveEnumeratorsReturningTrueWithCurrentNull()
+        {
+            var time = new DateTime(2016, 03, 03, 12, 05, 00);
+            var tradeBar1 = new TradeBar { Symbol = Symbols.SPY, Time = time };
+            var tradeBar2 = new TradeBar { Symbol = Symbols.AAPL, Time = time, Open = 23 };
+            var stream1 = Enumerable.Range(0, 20)
+                // return null except the last value and check if its emitted
+                .Select(x => x == 19 ? new Slice(time.AddSeconds(x * 1), new BaseData[] { tradeBar1, tradeBar2 }, utcTime: time.AddSeconds(x * 1)) : null
+            ).GetEnumerator();
+            var stream2 = Enumerable.Range(0, 5).Select(x => new Slice(time.AddSeconds(x * 2), new List<BaseData>(), utcTime: time.AddSeconds(x * 2))).GetEnumerator();
+            var stream3 = Enumerable.Range(0, 20).Select(x => new Slice(time.AddSeconds(x * 0.5), new List<BaseData>(), utcTime: time.AddSeconds(x * 0.5))).GetEnumerator();
+
+            var previous = new Slice(DateTime.MinValue, new List<BaseData>(), DateTime.MinValue);
+            var synchronizer = new SynchronizingSliceEnumerator(stream1, stream2, stream3);
+            while (synchronizer.MoveNext())
+            {
+                Assert.That(synchronizer.Current.UtcTime, Is.GreaterThanOrEqualTo(previous.UtcTime));
+                previous = synchronizer.Current;
+            }
+            Assert.AreEqual(2, previous.Bars.Count);
+            synchronizer.Dispose();
+        }
+
+        [Test]
+        public void WillRemoveEnumeratorsReturningFalse()
+        {
+            var time = new DateTime(2016, 03, 03, 12, 05, 00);
+            var stream1 = new TestEnumerator { MoveNextReturnValue = false };
+            var stream2 = Enumerable.Range(0, 10).Select(x => new Slice(time.AddSeconds(x * 0.5), new List<BaseData>(), utcTime: time.AddSeconds(x * 0.5))).GetEnumerator();
+            var synchronizer = new SynchronizingSliceEnumerator(stream1, stream2);
+            var emitted = false;
+            while (synchronizer.MoveNext())
+            {
+                emitted = true;
+            }
+            Assert.IsTrue(emitted);
+            Assert.IsTrue(stream1.MoveNextWasCalled);
+            Assert.AreEqual(1, stream1.MoveNextCallCount);
+            synchronizer.Dispose();
+        }
+
+        [Test]
+        public void WillStopIfAllEnumeratorsCurrentIsNullAndReturningTrue()
+        {
+            var stream1 = new TestEnumerator { MoveNextReturnValue = true };
+            var synchronizer = new SynchronizingSliceEnumerator(stream1);
+            while (synchronizer.MoveNext())
+            {
+                Assert.Fail();
+            }
+            Assert.IsTrue(stream1.MoveNextWasCalled);
+            synchronizer.Dispose();
+        }
+
+        [Test]
+        public void WillStopIfAllEnumeratorsCurrentIsNullAndReturningFalse()
+        {
+            var stream1 = new TestEnumerator { MoveNextReturnValue = false };
+            var synchronizer = new SynchronizingSliceEnumerator(stream1);
+            while (synchronizer.MoveNext())
+            {
+                Assert.Fail();
+            }
+            Assert.IsTrue(stream1.MoveNextWasCalled);
+            synchronizer.Dispose();
+        }
+
+        private class TestEnumerator : IEnumerator<Slice>
+        {
+            public int MoveNextCallCount { get; set; }
+            public bool MoveNextReturnValue { get; set; }
+            public bool MoveNextWasCalled { get; set; }
+
+            public TestEnumerator()
+            {
+                MoveNextWasCalled = false;
+            }
+
+            public bool MoveNext()
+            {
+                MoveNextCallCount++;
+                MoveNextWasCalled = true;
+                return MoveNextReturnValue;
+            }
+
+            public Slice Current { get; }
+
+            object IEnumerator.Current => Current;
+
+            public void Dispose()
+            { }
+
+            public void Reset()
+            { }
+        }
+    }
+}