chore: import upstream snapshot with attribution

Common/Util/BaseExtendedDictionary.cs

@@ -0,0 +1,256 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using QuantConnect;
+using QuantConnect.Python;
+
+namespace Common.Util
+{
+    /// <summary>
+    /// Provides a generic implementation of ExtendedDictionary with specific dictionary type
+    /// </summary>
+    [PandasNonExpandable]
+    public class BaseExtendedDictionary<TKey, TValue, TDictionary> : ExtendedDictionary<TKey, TValue>, IDictionary<TKey, TValue>
+        where TDictionary : IDictionary<TKey, TValue>, new()
+    {
+        /// <summary>
+        /// The dictionary instance
+        /// </summary>
+        protected TDictionary Dictionary { get; }
+
+        /// <summary>
+        /// Initializes a new instance of the BaseExtendedDictionary class that is empty
+        /// </summary>
+        public BaseExtendedDictionary()
+        {
+            Dictionary = new TDictionary();
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the BaseExtendedDictionary class that contains elements copied from the specified dictionary
+        /// </summary>
+        /// <param name="dictionary">The dictionary whose elements are copied to the new dictionary</param>
+        public BaseExtendedDictionary(TDictionary dictionary)
+        {
+            Dictionary = dictionary;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the BaseExtendedDictionary class
+        /// using the specified <paramref name="data"/> as a data source
+        /// </summary>
+        /// <param name="data">The data source for this dictionary</param>
+        /// <param name="keySelector">Delegate used to select a key from the value</param>
+        public BaseExtendedDictionary(IEnumerable<TValue> data, Func<TValue, TKey> keySelector)
+            : this()
+        {
+            foreach (var datum in data)
+            {
+                Dictionary[keySelector(datum)] = datum;
+            }
+        }
+
+        /// <summary>
+        /// Gets the number of elements contained in the dictionary
+        /// </summary>
+        public override int Count => Dictionary.Count;
+
+        /// <summary>
+        /// Gets a value indicating whether the dictionary is read-only
+        /// </summary>
+        public override bool IsReadOnly => Dictionary.IsReadOnly;
+
+        /// <summary>
+        /// Gets the value associated with the specified key
+        /// </summary>
+        /// <param name="key">The key whose value to get</param>
+        /// <param name="value">When this method returns, the value associated with the specified key</param>
+        /// <returns>true if the key was found; otherwise, false</returns>
+        public override bool TryGetValue(TKey key, out TValue value)
+        {
+            return Dictionary.TryGetValue(key, out value);
+        }
+
+        /// <summary>
+        /// Gets all the items in the dictionary
+        /// </summary>
+        /// <returns>All the items in the dictionary</returns>
+        public override IEnumerable<KeyValuePair<TKey, TValue>> GetItems()
+        {
+            return Dictionary;
+        }
+
+        /// <summary>
+        /// Gets a collection containing the keys in the dictionary
+        /// </summary>
+        protected override IEnumerable<TKey> GetKeys => Dictionary.Keys;
+
+        /// <summary>
+        /// Gets a collection containing the values in the dictionary
+        /// </summary>
+        protected override IEnumerable<TValue> GetValues => Dictionary.Values;
+
+        /// <summary>
+        /// Gets a collection containing the keys of the dictionary
+        /// </summary>
+        public virtual ICollection<TKey> Keys => Dictionary.Keys;
+
+        /// <summary>
+        /// Gets a collection containing the values of the dictionary
+        /// </summary>
+        public virtual ICollection<TValue> Values => Dictionary.Values;
+
+        /// <summary>
+        /// Gets or sets the value associated with the specified key
+        /// </summary>
+        /// <param name="key">The key of the value to get or set</param>
+        /// <returns>The value associated with the specified key</returns>
+        public override TValue this[TKey key]
+        {
+            get => Dictionary[key];
+            set => Dictionary[key] = value;
+        }
+
+        /// <summary>
+        /// Removes all items from the dictionary
+        /// </summary>
+        public override void Clear()
+        {
+            Dictionary.Clear();
+        }
+
+        /// <summary>
+        /// Removes the value with the specified key
+        /// </summary>
+        /// <param name="key">The key of the element to remove</param>
+        /// <returns>true if the element was successfully found and removed; otherwise, false</returns>
+        public override bool Remove(TKey key)
+        {
+            return Dictionary.Remove(key);
+        }
+
+        /// <summary>
+        /// Adds an element with the provided key and value to the dictionary
+        /// </summary>
+        /// <param name="key">The key of the element to add</param>
+        /// <param name="value">The value of the element to add</param>
+        public virtual void Add(TKey key, TValue value)
+        {
+            Dictionary.Add(key, value);
+        }
+
+        /// <summary>
+        /// Adds an element with the provided key-value pair to the dictionary
+        /// </summary>
+        /// <param name="item">The key-value pair to add</param>
+        public virtual void Add(KeyValuePair<TKey, TValue> item)
+        {
+            Dictionary.Add(item);
+        }
+
+        /// <summary>
+        /// Determines whether the dictionary contains the specified key
+        /// </summary>
+        /// <param name="key">The key to locate in the dictionary</param>
+        /// <returns>true if the dictionary contains an element with the specified key; otherwise, false</returns>
+        public override bool ContainsKey(TKey key)
+        {
+            return Dictionary.ContainsKey(key);
+        }
+
+        /// <summary>
+        /// Determines whether the dictionary contains a specific key-value pair
+        /// </summary>
+        /// <param name="item">The key-value pair to locate</param>
+        /// <returns>true if the key-value pair was found; otherwise, false</returns>
+        public virtual bool Contains(KeyValuePair<TKey, TValue> item)
+        {
+            return Dictionary.Contains(item);
+        }
+
+        /// <summary>
+        /// Copies the elements of the dictionary to an array, starting at a particular array index
+        /// </summary>
+        /// <param name="array">The array to copy to</param>
+        /// <param name="arrayIndex">The starting index in the array</param>
+        public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
+        {
+            Dictionary.CopyTo(array, arrayIndex);
+        }
+
+        /// <summary>
+        /// Removes the first occurrence of a specific object from the dictionary
+        /// </summary>
+        /// <param name="item">The key-value pair to remove</param>
+        /// <returns>true if the key-value pair was successfully removed; otherwise, false</returns>
+        public virtual bool Remove(KeyValuePair<TKey, TValue> item)
+        {
+            return Dictionary.Remove(item);
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through the dictionary
+        /// </summary>
+        /// <returns>An enumerator for the dictionary</returns>
+        public virtual IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
+        {
+            return Dictionary.GetEnumerator();
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through the dictionary
+        /// </summary>
+        /// <returns>An enumerator for the dictionary</returns>
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+    }
+
+    /// <summary>
+    /// Provides a default implementation of ExtendedDictionary using Dictionary{TKey, TValue}
+    /// </summary>
+    [PandasNonExpandable]
+    public class BaseExtendedDictionary<TKey, TValue> : BaseExtendedDictionary<TKey, TValue, Dictionary<TKey, TValue>>
+    {
+        /// <summary>
+        /// Initializes a new instance of the BaseExtendedDictionary class that is empty
+        /// </summary>
+        public BaseExtendedDictionary() : base()
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the BaseExtendedDictionary class that contains elements copied from the specified dictionary
+        /// </summary>
+        /// <param name="dictionary">The dictionary whose elements are copied to the new dictionary</param>
+        public BaseExtendedDictionary(IDictionary<TKey, TValue> dictionary) : base(new Dictionary<TKey, TValue>(dictionary))
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the BaseExtendedDictionary class
+        /// using the specified <paramref name="data"/> as a data source
+        /// </summary>
+        /// <param name="data">The data source for this dictionary</param>
+        /// <param name="keySelector">Delegate used to select a key from the value</param>
+        public BaseExtendedDictionary(IEnumerable<TValue> data, Func<TValue, TKey> keySelector) : base(data, keySelector)
+        {
+        }
+    }
+}

Common/Util/BusyBlockingCollection.cs

@@ -0,0 +1,214 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Threading;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// A small wrapper around <see cref="BlockingCollection{T}"/> used to communicate busy state of the items
+    /// being processed
+    /// </summary>
+    /// <typeparam name="T">The item type being processed</typeparam>
+    public class BusyBlockingCollection<T> : IBusyCollection<T>
+    {
+        private readonly BlockingCollection<T> _collection;
+        private readonly ManualResetEventSlim _processingCompletedEvent;
+        private readonly object _lock = new object();
+
+        /// <summary>
+        /// Gets a wait handle that can be used to wait until this instance is done
+        /// processing all of it's item
+        /// </summary>
+        public WaitHandle WaitHandle
+        {
+            get { return _processingCompletedEvent.WaitHandle; }
+        }
+
+        /// <summary>
+        /// Gets the number of items held within this collection
+        /// </summary>
+        public int Count
+        {
+            get { return _collection.Count; }
+        }
+
+        /// <summary>
+        /// Returns true if processing, false otherwise
+        /// </summary>
+        public bool IsBusy
+        {
+            get
+            {
+                lock (_lock)
+                {
+                    return _collection.Count > 0 || !_processingCompletedEvent.IsSet;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="BusyBlockingCollection{T}"/> class
+        /// with a bounded capacity of <see cref="int.MaxValue"/>
+        /// </summary>
+        public BusyBlockingCollection()
+            : this(int.MaxValue)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="BusyBlockingCollection{T}"/> class
+        /// with the specified <paramref name="boundedCapacity"/>
+        /// </summary>
+        /// <param name="boundedCapacity">The maximum number of items allowed in the collection</param>
+        public BusyBlockingCollection(int boundedCapacity)
+        {
+            _collection = new BlockingCollection<T>(boundedCapacity);
+
+            // initialize as not busy
+            _processingCompletedEvent = new ManualResetEventSlim(true);
+        }
+
+        /// <summary>
+        /// Adds the items to this collection
+        /// </summary>
+        /// <param name="item">The item to be added</param>
+        public void Add(T item)
+        {
+            Add(item, CancellationToken.None);
+        }
+
+        /// <summary>
+        /// Adds the items to this collection
+        /// </summary>
+        /// <param name="item">The item to be added</param>
+        /// <param name="cancellationToken">A cancellation token to observer</param>
+        public void Add(T item, CancellationToken cancellationToken)
+        {
+            bool added;
+            lock (_lock)
+            {
+                // we're adding work to be done, mark us as busy
+                _processingCompletedEvent.Reset();
+                added = _collection.TryAdd(item, 0, cancellationToken);
+            }
+
+            if (!added)
+            {
+                _collection.Add(item, cancellationToken);
+            }
+        }
+
+        /// <summary>
+        /// Marks the <see cref="BusyBlockingCollection{T}"/> as not accepting any more additions
+        /// </summary>
+        public void CompleteAdding()
+        {
+            _collection.CompleteAdding();
+        }
+
+        /// <summary>
+        /// Provides a consuming enumerable for items in this collection.
+        /// </summary>
+        /// <returns>An enumerable that removes and returns items from the collection</returns>
+        public IEnumerable<T> GetConsumingEnumerable()
+        {
+            return GetConsumingEnumerable(CancellationToken.None);
+        }
+
+        /// <summary>
+        /// Provides a consuming enumerable for items in this collection.
+        /// </summary>
+        /// <param name="cancellationToken">A cancellation token to observer</param>
+        /// <returns>An enumerable that removes and returns items from the collection</returns>
+        public IEnumerable<T> GetConsumingEnumerable(CancellationToken cancellationToken)
+        {
+            while (!_collection.IsCompleted)
+            {
+                T item;
+
+                // check to see if something is immediately available
+                bool tookItem;
+
+                try
+                {
+                    tookItem = _collection.TryTake(out item, 0, cancellationToken);
+                }
+                catch (OperationCanceledException)
+                {
+                    // if the operation was canceled, just bail on the enumeration
+                    yield break;
+                }
+
+                if (tookItem)
+                {
+                    // something was immediately available, emit it
+                    yield return item;
+                    continue;
+                }
+
+
+                // we need to lock this with the Add method since we need to model the act of
+                // taking/flipping the switch and adding/flipping the switch as one operation
+                lock (_lock)
+                {
+                    // double check that there's nothing in the collection within a lock, it's possible
+                    // that between the TryTake above and this statement, the Add method was called, so we
+                    // don't want to flip the switch if there's something in the collection
+                    if (_collection.Count == 0)
+                    {
+                        // nothing was immediately available, mark us as idle
+                        _processingCompletedEvent.Set();
+                    }
+                }
+
+                try
+                {
+                    // now block until something is available
+                    tookItem = _collection.TryTake(out item, Timeout.Infinite, cancellationToken);
+                }
+                catch (OperationCanceledException)
+                {
+                    // if the operation was canceled, just bail on the enumeration
+                    yield break;
+                }
+
+                if (tookItem)
+                {
+                    // emit the item we found
+                    yield return item;
+                }
+            }
+
+            // no more items to process
+            _processingCompletedEvent.Set();
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _collection.Dispose();
+            _processingCompletedEvent.Dispose();
+        }
+    }
+}

Common/Util/BusyCollection.cs

@@ -0,0 +1,132 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Threading;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// A non blocking <see cref="IBusyCollection{T}"/> implementation
+    /// </summary>
+    /// <typeparam name="T">The item type being processed</typeparam>
+    public class BusyCollection<T> : IBusyCollection<T>
+    {
+        private readonly ConcurrentQueue<T> _collection = new ConcurrentQueue<T>();
+        private readonly ManualResetEventSlim _processingCompletedEvent = new ManualResetEventSlim(true);
+        private bool _completedAdding;
+
+        /// <summary>
+        /// Gets a wait handle that can be used to wait until this instance is done
+        /// processing all of it's item
+        /// </summary>
+        public WaitHandle WaitHandle => _processingCompletedEvent.WaitHandle;
+
+        /// <summary>
+        /// Gets the number of items held within this collection
+        /// </summary>
+        public int Count => _collection.Count;
+
+        /// <summary>
+        /// Returns true if processing, false otherwise
+        /// </summary>
+        public bool IsBusy => !_collection.IsEmpty || !_processingCompletedEvent.IsSet;
+
+        /// <summary>
+        /// Adds the items to this collection
+        /// </summary>
+        /// <param name="item">The item to be added</param>
+        public void Add(T item)
+        {
+            Add(item, CancellationToken.None);
+        }
+
+        /// <summary>
+        /// Adds the items to this collection
+        /// </summary>
+        /// <param name="item">The item to be added</param>
+        /// <param name="cancellationToken">A cancellation token to observer</param>
+        public void Add(T item, CancellationToken cancellationToken)
+        {
+            if (_completedAdding)
+            {
+                throw new InvalidOperationException("Collection has already been marked as not " +
+                    $"accepting more additions, see {nameof(CompleteAdding)}");
+            }
+
+            // locking to avoid race condition with GetConsumingEnumerable()
+            lock (_processingCompletedEvent)
+            {
+                // we're adding work to be done, mark us as busy
+                _processingCompletedEvent.Reset();
+                _collection.Enqueue(item);
+            }
+        }
+
+        /// <summary>
+        /// Provides a consuming enumerable for items in this collection.
+        /// </summary>
+        /// <returns>An enumerable that removes and returns items from the collection</returns>
+        public IEnumerable<T> GetConsumingEnumerable()
+        {
+            return GetConsumingEnumerable(CancellationToken.None);
+        }
+
+        /// <summary>
+        /// Provides a consuming enumerable for items in this collection.
+        /// </summary>
+        /// <param name="cancellationToken">A cancellation token to observer</param>
+        /// <returns>An enumerable that removes and returns items from the collection</returns>
+        public IEnumerable<T> GetConsumingEnumerable(CancellationToken cancellationToken)
+        {
+            T item;
+            while (_collection.TryDequeue(out item))
+            {
+                yield return item;
+            }
+
+            // locking to avoid race condition with Add()
+            lock (_processingCompletedEvent)
+            {
+                if (!_collection.TryPeek(out item))
+                {
+                    _processingCompletedEvent.Set();
+                }
+            }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _collection.Clear();
+            _processingCompletedEvent.Dispose();
+        }
+
+        /// <summary>
+        /// Marks the collection as not accepting any more additions
+        /// </summary>
+        public void CompleteAdding()
+        {
+            _completedAdding = true;
+        }
+    }
+}

Common/Util/CandlestickJsonConverter.cs

@@ -0,0 +1,97 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Candlestick Json Converter
+    /// </summary>
+    public class CandlestickJsonConverter : JsonConverter
+    {
+        /// <summary>
+        /// Write Series to Json
+        /// </summary>
+        /// <param name="writer">The Json Writer to use</param>
+        /// <param name="value">The value to written to Json</param>
+        /// <param name="serializer">The Json Serializer to use</param>
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            // Candlesticks will be written as a single array of 5 values: [time, open, high, low, close]
+
+            var candlestick = value as Candlestick;
+            if (candlestick == null)
+            {
+                return;
+            }
+
+            writer.WriteStartArray();
+
+            writer.WriteValue(candlestick.LongTime);
+            writer.WriteValue(candlestick.Open);
+            writer.WriteValue(candlestick.High);
+            writer.WriteValue(candlestick.Low);
+            writer.WriteValue(candlestick.Close);
+
+            writer.WriteEndArray();
+        }
+
+        /// <summary>
+        /// Json reader implementation which handles backwards compatiblity for old equity chart points
+        /// </summary>
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            if(reader.TokenType == JsonToken.StartObject)
+            {
+                var chartPoint = serializer.Deserialize<ChartPoint>(reader);
+                if(chartPoint == null)
+                {
+                    return null;
+                }
+                return new Candlestick(chartPoint.X, chartPoint.Y, chartPoint.Y, chartPoint.Y, chartPoint.Y);
+            }
+            var jArray = JArray.Load(reader);
+            if(jArray.Count <= 2)
+            {
+                var chartPoint = jArray.ToObject<ChartPoint>();
+                if (chartPoint == null)
+                {
+                    return null;
+                }
+                return new Candlestick(chartPoint.X, chartPoint.Y, chartPoint.Y, chartPoint.Y, chartPoint.Y);
+            }
+            return new Candlestick(jArray[0].Value<long>(), jArray[1].Value<decimal?>(), jArray[2].Value<decimal?>(),
+                jArray[3].Value<decimal?>(), jArray[4].Value<decimal?>());
+        }
+
+        /// <summary>
+        /// Determine if this Converter can convert this type
+        /// </summary>
+        /// <param name="objectType">Type that we would like to convert</param>
+        /// <returns>True if <see cref="Series"/></returns>
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(Candlestick);
+        }
+
+        /// <summary>
+        /// This converter wont be used to read JSON. Will throw exception if manually called.
+        /// </summary>
+        public override bool CanRead => true;
+    }
+}

Common/Util/CashAmountUtil.cs

@@ -0,0 +1,38 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Securities;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides utility methods for working with <see cref="CashAmount"/> instances
+    /// </summary>
+    public static class CashAmountUtil
+    {
+        /// <summary>
+        /// Determines if a cash balance should be added to the cash book
+        /// </summary>
+        /// <param name="balance">The cash balance to check</param>
+        /// <param name="accountCurrency">The algorithm's account currency</param>
+        /// <returns>True if the balance should be added, false otherwise</returns>
+        public static bool ShouldAddCashBalance(CashAmount balance, string accountCurrency)
+        {
+            // Don't add zero quantity currencies except the account currency
+            // we do add 'BNFCR' even if zero as it's used to track brokerage fees, we need lean to setup conversion rates for it
+            return balance.Amount != 0 || balance.Currency == accountCurrency || balance.Currency == "BNFCR";
+        }
+    }
+}

Common/Util/CastingEnumerable.cs

@@ -0,0 +1,81 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Defines a list that casts the elements of a source list to a derived type.
+    /// This is useful to avoid materializing another list after using, for example, the <see cref="Enumerable.Cast{TResult}(IEnumerable)"/> LINQ method.
+    /// </summary>
+    /// <typeparam name="TBase">The base type of the elements in the source enumerable.</typeparam>
+    /// <typeparam name="TDerived">The type to cast the elements to.</typeparam>
+    public class CastingEnumerable<TBase, TDerived> : IReadOnlyList<TDerived>
+        where TDerived : class, TBase
+    {
+        private IReadOnlyList<TBase> _data;
+
+        /// <summary>
+        /// Gets the count of items in the enumerable.
+        /// </summary>
+        public int Count => _data.Count;
+
+        /// <summary>
+        /// Gets the element at the specified index.
+        /// </summary>
+        /// <param name="index">The zero-based index of the element to get.</param>
+        /// <returns>The element at the specified index.</returns>
+        public TDerived this[int index] => (TDerived)_data[index];
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CastingEnumerable{TBase, TDerived}"/> class
+        /// </summary>
+        public CastingEnumerable(IReadOnlyList<TBase> data)
+        {
+            _data = data;
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through the collection.
+        /// </summary>
+        /// <returns>
+        /// An enumerator that can be used to iterate through the collection.
+        /// </returns>
+        /// <filterpriority>1</filterpriority>
+        public IEnumerator<TDerived> GetEnumerator()
+        {
+            foreach (var item in _data)
+            {
+                yield return (TDerived)item;
+            }
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through a collection.
+        /// </summary>
+        /// <returns>
+        /// An enumerator object that can be used to iterate through the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+    }
+}

Common/Util/ChartPointJsonConverter.cs

@@ -0,0 +1,82 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Json Converter for ChartPoint which handles special reading
+    /// </summary>
+    public class ChartPointJsonConverter : JsonConverter
+    {
+        /// <summary>
+        /// Determine if this Converter can convert this type
+        /// </summary>
+        /// <param name="objectType">Type that we would like to convert</param>
+        /// <returns>True if <see cref="Series"/></returns>
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(ChartPoint);
+        }
+
+        /// <summary>
+        /// Reads series from Json
+        /// </summary>
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            if (reader.TokenType == JsonToken.StartObject)
+            {
+                var jObject = JObject.Load(reader);
+                var x = jObject["x"];
+
+                if (!jObject.ContainsKey("y"))
+                {
+                    return new ChartPoint(x.Value<long>(), 0);
+                }
+
+                var y = jObject["y"];
+                if (y != null && (y.Type == JTokenType.Float || y.Type == JTokenType.Integer))
+                {
+                    return new ChartPoint(x.Value<long>(), y.Value<decimal>());
+                }
+
+                if (y.Type == JTokenType.Null)
+                {
+                    return new ChartPoint(x.Value<long>(), null);
+                }
+
+                return null;
+            }
+
+            var jArray = JArray.Load(reader);
+            return new ChartPoint(jArray[0].Value<long>(), jArray[1].Value<decimal?>());
+        }
+
+        /// <summary>
+        /// Write point to Json
+        /// </summary>
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            var chartPoint = (ChartPoint)value;
+            writer.WriteStartArray();
+            writer.WriteValue(chartPoint.X);
+            writer.WriteValue(chartPoint.Y);
+            writer.WriteEndArray();
+        }
+    }
+}

Common/Util/CircularQueue.cs

@@ -0,0 +1,87 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// A never ending queue that will dequeue and reenqueue the same item
+    /// </summary>
+    public class CircularQueue<T>
+    {
+        private readonly T _head;
+        private readonly Queue<T> _queue;
+
+        /// <summary>
+        /// Fired when we do a full circle
+        /// </summary>
+        public event EventHandler CircleCompleted;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CircularQueue{T}"/> class
+        /// </summary>
+        /// <param name="items">The items in the queue</param>
+        public CircularQueue(params T[] items)
+            : this((IEnumerable<T>)items)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CircularQueue{T}"/> class
+        /// </summary>
+        /// <param name="items">The items in the queue</param>
+        public CircularQueue(IEnumerable<T> items)
+        {
+            _queue = new Queue<T>();
+
+            var first = true;
+            foreach (var item in items)
+            {
+                if (first)
+                {
+                    first = false;
+                    _head = item;
+                }
+                _queue.Enqueue(item);
+            }
+        }
+
+        /// <summary>
+        /// Dequeues the next item
+        /// </summary>
+        /// <returns>The next item</returns>
+        public T Dequeue()
+        {
+            var item = _queue.Dequeue();
+            if (item.Equals(_head))
+            {
+                OnCircleCompleted();
+            }
+            _queue.Enqueue(item);
+            return item;
+        }
+
+        /// <summary>
+        /// Event invocator for the <see cref="CircleCompleted"/> evet
+        /// </summary>
+        protected virtual void OnCircleCompleted()
+        {
+            var handler = CircleCompleted;
+            if (handler != null) handler(this, EventArgs.Empty);
+        }
+    }
+}

Common/Util/ColorJsonConverter.cs

@@ -0,0 +1,85 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Drawing;
+using System.Globalization;
+using Newtonsoft.Json;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// A <see cref="JsonConverter" /> implementation that serializes a <see cref="Color" /> as a string.
+    /// If Color is empty, string is also empty and vice-versa. Meaning that color is autogen.
+    /// </summary>
+    public class ColorJsonConverter : TypeChangeJsonConverter<Color, string>
+    {
+        /// <summary>
+        /// Converts a .NET Color to a hexadecimal as a string
+        /// </summary>
+        /// <param name="value">The input value to be converted before serialization</param>
+        /// <returns>Hexadecimal number as a string. If .NET Color is null, returns default #000000</returns>
+        protected override string Convert(Color value)
+        {
+            return value.IsEmpty ? string.Empty : $"#{value.R.ToStringInvariant("X2")}{value.G.ToStringInvariant("X2")}{value.B.ToStringInvariant("X2")}";
+        }
+
+        /// <summary>
+        /// Converts the input string to a .NET Color object
+        /// </summary>
+        /// <param name="value">The deserialized value that needs to be converted to T</param>
+        /// <returns>The converted value</returns>
+        protected override Color Convert(string value)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                return Color.Empty;
+            }
+            if (value.Length != 7)
+            {
+                var message = $"Unable to convert '{value}' to a Color. Requires string length of 7 including the leading hashtag.";
+                throw new FormatException(message);
+            }
+
+            var red = HexToInt(value.Substring(1, 2));
+            var green = HexToInt(value.Substring(3, 2));
+            var blue = HexToInt(value.Substring(5, 2));
+            return Color.FromArgb(red, green, blue);
+        }
+
+        /// <summary>
+        /// Converts hexadecimal number to integer
+        /// </summary>
+        /// <param name="hexValue">Hexadecimal number</param>
+        /// <returns>Integer representation of the hexadecimal</returns>
+        private int HexToInt(string hexValue)
+        {
+            if (hexValue.Length != 2)
+            {
+                var message = $"Unable to convert '{hexValue}' to an Integer. Requires string length of 2.";
+                throw new FormatException(message);
+            }
+
+            int result;
+            if (!int.TryParse(hexValue, NumberStyles.HexNumber, null, out result))
+            {
+                throw new FormatException($"Invalid hex number: {hexValue}");
+            }
+
+            return result;
+        }
+    }
+}

Common/Util/ComparisonOperator.cs

@@ -0,0 +1,54 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Utility Comparison Operator class
+    /// </summary>
+    public static class ComparisonOperator
+    {
+        /// <summary>
+        /// Compares two values using given operator
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="op">Comparison operator</param>
+        /// <param name="arg1">The first value</param>
+        /// <param name="arg2">The second value</param>
+        /// <returns>Returns true if its left-hand operand meets the operator value to its right-hand operand, false otherwise</returns>
+        public static bool Compare<T>(ComparisonOperatorTypes op, T arg1, T arg2) where T : IComparable
+        {
+            switch (op)
+            {
+                case ComparisonOperatorTypes.Equals:
+                    return arg1.CompareTo(arg2) == 0;
+                case ComparisonOperatorTypes.NotEqual:
+                    return arg1.CompareTo(arg2) != 0;
+                case ComparisonOperatorTypes.Greater:
+                    return arg1.CompareTo(arg2) == 1;
+                case ComparisonOperatorTypes.GreaterOrEqual:
+                    return arg1.CompareTo(arg2) >= 0;
+                case ComparisonOperatorTypes.Less:
+                    return arg1.CompareTo(arg2) == -1;
+                case ComparisonOperatorTypes.LessOrEqual:
+                    return arg1.CompareTo(arg2) <= 0;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(op), $"Operator '{op}' is not supported.");
+            }
+        }
+    }
+}

Common/Util/ComparisonOperatorTypes.cs

@@ -0,0 +1,57 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Comparison operators
+    /// </summary>
+    [JsonConverter(typeof(StringEnumConverter), true)]
+    public enum ComparisonOperatorTypes
+    {
+        /// <summary>
+        /// Check if their operands are equal
+        /// </summary>
+        Equals,
+
+        /// <summary>
+        /// Check if their operands are not equal
+        /// </summary>
+        NotEqual,
+
+        /// <summary>
+        /// Checks left-hand operand is greater than its right-hand operand
+        /// </summary>
+        Greater,
+
+        /// <summary>
+        /// Checks left-hand operand is greater or equal to its right-hand operand
+        /// </summary>
+        GreaterOrEqual,
+
+        /// <summary>
+        /// Checks left-hand operand is less than its right-hand operand
+        /// </summary>
+        Less,
+
+        /// <summary>
+        /// Checks left-hand operand is less or equal to its right-hand operand
+        /// </summary>
+        LessOrEqual
+    }
+}

Common/Util/Composer.cs

@@ -0,0 +1,411 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.ComponentModel.Composition;
+using System.ComponentModel.Composition.Hosting;
+using System.ComponentModel.Composition.Primitives;
+using System.ComponentModel.Composition.ReflectionModel;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using QuantConnect.Configuration;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides methods for obtaining exported MEF instances
+    /// </summary>
+    public class Composer
+    {
+        private static string PluginDirectory;
+        private static readonly Lazy<Composer> LazyComposer = new Lazy<Composer>(
+            () =>
+            {
+                PluginDirectory = Config.Get("plugin-directory");
+                return new Composer();
+            });
+
+        /// <summary>
+        /// Gets the singleton instance
+        /// </summary>
+        /// <remarks>Intentionally using a property so that when its gotten it will
+        /// trigger the lazy construction which will be after the right configuration
+        /// is loaded. See GH issue 3258</remarks>
+        public static Composer Instance => LazyComposer.Value;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Composer"/> class. This type
+        /// is a light wrapper on top of an MEF <see cref="CompositionContainer"/>
+        /// </summary>
+        public Composer()
+        {
+            // Determine what directory to grab our assemblies from if not defined by 'composer-dll-directory' configuration key
+            var dllDirectoryString = Config.Get("composer-dll-directory");
+            if (string.IsNullOrWhiteSpace(dllDirectoryString))
+            {
+                // Check our appdomain directory for QC Dll's, for most cases this will be true and fine to use
+                if (!string.IsNullOrEmpty(AppDomain.CurrentDomain.BaseDirectory) && Directory.EnumerateFiles(AppDomain.CurrentDomain.BaseDirectory, "QuantConnect.*.dll").Any())
+                {
+                    dllDirectoryString = AppDomain.CurrentDomain.BaseDirectory;
+                }
+                else
+                {
+                    // Otherwise check out our parent and current working directory
+                    // this is helpful for research because kernel appdomain defaults to kernel location
+                    var currentDirectory = Directory.GetCurrentDirectory();
+                    var parentDirectory = Directory.GetParent(currentDirectory)?.FullName ?? currentDirectory; // If parent == null will just use current
+
+                    // If our parent directory contains QC Dlls use it, otherwise default to current working directory
+                    // In cloud and CLI research cases we expect the parent directory to contain the Dlls; but locally it's likely current directory
+                    dllDirectoryString = Directory.EnumerateFiles(parentDirectory, "QuantConnect.*.dll").Any() ? parentDirectory : currentDirectory;
+                }
+            }
+
+            // Resolve full path name just to be safe
+            var primaryDllLookupDirectory = new DirectoryInfo(dllDirectoryString).FullName;
+            Log.Trace($"Composer(): Loading Assemblies from {primaryDllLookupDirectory}");
+
+            var loadFromPluginDir = !string.IsNullOrWhiteSpace(PluginDirectory)
+                && Directory.Exists(PluginDirectory) &&
+                new DirectoryInfo(PluginDirectory).FullName != primaryDllLookupDirectory;
+            var fileNames = Directory.EnumerateFiles(primaryDllLookupDirectory, "*.dll");
+            if (loadFromPluginDir)
+            {
+                fileNames = fileNames.Concat(Directory.EnumerateFiles(PluginDirectory, "*.dll"));
+            }
+            LoadPartsSafely(fileNames.DistinctBy(Path.GetFileName));
+        }
+
+        private CompositionContainer _compositionContainer;
+        private IReadOnlyList<Type> _exportedTypes;
+        private List<ComposablePartDefinition> _composableParts;
+        private readonly object _exportedValuesLockObject = new object();
+        private readonly Dictionary<Type, IEnumerable> _exportedValues = new Dictionary<Type, IEnumerable>();
+
+        /// <summary>
+        /// Gets the export matching the predicate
+        /// </summary>
+        /// <param name="predicate">Function used to pick which imported instance to return, if null the first instance is returned</param>
+        /// <returns>The only export matching the specified predicate</returns>
+        public T Single<T>(Func<T, bool> predicate)
+        {
+            if (predicate == null)
+            {
+                throw new ArgumentNullException(nameof(predicate));
+            }
+
+            return GetExportedValues<T>().Single(predicate);
+        }
+
+        /// <summary>
+        /// Adds the specified instance to this instance to allow it to be recalled via GetExportedValueByTypeName
+        /// </summary>
+        /// <typeparam name="T">The contract type</typeparam>
+        /// <param name="instance">The instance to add</param>
+        public void AddPart<T>(T instance)
+        {
+            lock (_exportedValuesLockObject)
+            {
+                IEnumerable values;
+                if (_exportedValues.TryGetValue(typeof(T), out values))
+                {
+                    ((IList<T>)values).Add(instance);
+                }
+                else
+                {
+                    values = new List<T> { instance };
+                    _exportedValues[typeof(T)] = values;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets the first type T instance if any
+        /// </summary>
+        /// <typeparam name="T">The contract type</typeparam>
+        public T GetPart<T>()
+        {
+            return GetPart<T>(null);
+        }
+
+        /// <summary>
+        /// Gets the first type T instance if any
+        /// </summary>
+        /// <typeparam name="T">The contract type</typeparam>
+        public T GetPart<T>(Func<T, bool> filter)
+        {
+            return GetParts<T>().Where(x => filter == null || filter(x)).FirstOrDefault();
+        }
+
+        /// <summary>
+        /// Gets all parts of type T instance if any
+        /// </summary>
+        /// <typeparam name="T">The contract type</typeparam>
+        public IEnumerable<T> GetParts<T>()
+        {
+            lock (_exportedValuesLockObject)
+            {
+                IEnumerable values;
+                if (_exportedValues.TryGetValue(typeof(T), out values))
+                {
+                    return ((IEnumerable<T>)values).ToList();
+                }
+                return Enumerable.Empty<T>();
+            }
+        }
+
+        /// <summary>
+        /// Will return all loaded types that are assignable to T type
+        /// </summary>
+        public IEnumerable<Type> GetExportedTypes<T>() where T : class
+        {
+            var type = typeof(T);
+            return _exportedTypes.Where(type1 =>
+                {
+                    try
+                    {
+                        return type.IsAssignableFrom(type1);
+                    }
+                    catch
+                    {
+                        return false;
+                    }
+                });
+        }
+
+        /// <summary>
+        /// Extension method to searches the composition container for an export that has a matching type name. This function
+        /// will first try to match on Type.AssemblyQualifiedName, then Type.FullName, and finally on Type.Name
+        ///
+        /// This method will not throw if multiple types are found matching the name, it will just return the first one it finds.
+        /// </summary>
+        /// <typeparam name="T">The type of the export</typeparam>
+        /// <param name="typeName">The name of the type to find. This can be an assembly qualified name, a full name, or just the type's name</param>
+        /// <param name="forceTypeNameOnExisting">When false, if any existing instance of type T is found, it will be returned even if type name doesn't match.
+        /// This is useful in cases where a single global instance is desired, like for <see cref="IDataAggregator"/></param>
+        /// <returns>The export instance</returns>
+        public T GetExportedValueByTypeName<T>(string typeName, bool forceTypeNameOnExisting = true)
+            where T : class
+        {
+            try
+            {
+                // if we've already loaded this part, then just return the same one
+                var instance = GetParts<T>().FirstOrDefault(x => !forceTypeNameOnExisting || x.GetType().MatchesTypeName(typeName));
+                if (instance != null)
+                {
+                    return instance;
+                }
+
+                var type = typeof(T);
+                var typeT = _exportedTypes.Where(type1 =>
+                    {
+                        try
+                        {
+                            return type.IsAssignableFrom(type1) && type1.MatchesTypeName(typeName);
+                        }
+                        catch
+                        {
+                            return false;
+                        }
+                    })
+                .FirstOrDefault();
+
+                if (typeT != null)
+                {
+                    instance = (T)Activator.CreateInstance(typeT);
+                }
+
+                if (instance == null)
+                {
+                    // we want to get the requested part without instantiating each one of that type
+                    var selectedPart = _composableParts
+                        .Where(x =>
+                            {
+                                try
+                                {
+                                    var xType = ReflectionModelServices.GetPartType(x).Value;
+                                    return type.IsAssignableFrom(xType) && xType.MatchesTypeName(typeName);
+                                }
+                                catch
+                                {
+                                    return false;
+                                }
+                            }
+                        )
+                        .FirstOrDefault();
+
+                    if (selectedPart == null)
+                    {
+                        throw new ArgumentException(
+                            $"Unable to locate any exports matching the requested typeName: {typeName}. Type: {type}", nameof(typeName));
+                    }
+
+                    var exportDefinition =
+                        selectedPart.ExportDefinitions.First(
+                            x => x.ContractName == AttributedModelServices.GetContractName(type));
+                    instance = (T)selectedPart.CreatePart().GetExportedValue(exportDefinition);
+                }
+
+                var exportedParts = instance.GetType().GetInterfaces()
+                    .Where(interfaceType => interfaceType.GetCustomAttribute<InheritedExportAttribute>() != null);
+
+                lock (_exportedValuesLockObject)
+                {
+                    foreach (var export in exportedParts)
+                    {
+                        var exportList = _exportedValues.SingleOrDefault(kvp => kvp.Key == export).Value;
+
+                        // cache the new value for next time
+                        if (exportList == null)
+                        {
+                            var list = (IList)Activator.CreateInstance(typeof(List<>).MakeGenericType(export));
+                            list.Add(instance);
+                            _exportedValues[export] = list;
+                        }
+                        else
+                        {
+                            ((IList)exportList).Add(instance);
+                        }
+                    }
+
+                    return instance;
+                }
+            }
+            catch (ReflectionTypeLoadException err)
+            {
+                foreach (var exception in err.LoaderExceptions)
+                {
+                    Log.Error(exception);
+                    Log.Error(exception.ToString());
+                }
+
+                if (err.InnerException != null) Log.Error(err.InnerException);
+
+                throw;
+            }
+        }
+        /// <summary>
+        /// Gets all exports of type T
+        /// </summary>
+        public IEnumerable<T> GetExportedValues<T>()
+        {
+            try
+            {
+                lock (_exportedValuesLockObject)
+                {
+                    IEnumerable values;
+                    if (_exportedValues.TryGetValue(typeof(T), out values))
+                    {
+                        return values.OfType<T>();
+                    }
+
+                    values = _compositionContainer.GetExportedValues<T>().ToList();
+                    _exportedValues[typeof(T)] = values;
+                    return values.OfType<T>();
+                }
+            }
+            catch (ReflectionTypeLoadException err)
+            {
+                foreach (var exception in err.LoaderExceptions)
+                {
+                    Log.Error(exception);
+                }
+
+                throw;
+            }
+        }
+
+        /// <summary>
+        /// Clears the cache of exported values, causing new instances to be created.
+        /// </summary>
+        public void Reset()
+        {
+            lock (_exportedValuesLockObject)
+            {
+                _exportedValues.Clear();
+            }
+        }
+
+        private void LoadPartsSafely(IEnumerable<string> files)
+        {
+            try
+            {
+                var exportedTypes = new ConcurrentBag<Type>();
+                var catalogs = new ConcurrentBag<ComposablePartCatalog>();
+                Parallel.ForEach(files, file =>
+                {
+                    try
+                    {
+                        // we need to load assemblies so that C# algorithm dependencies are resolved correctly
+                        // at the same time we need to load all QC dependencies to find all exports
+                        Assembly assembly;
+                        try
+                        {
+                            var asmName = AssemblyName.GetAssemblyName(file);
+                            assembly = Assembly.Load(asmName);
+                        }
+                        catch
+                        {
+                            // handles dependencies that are not in the probing path but might duplicate loading an already loaded assembly
+                            assembly = Assembly.LoadFrom(file);
+                        }
+
+                        if (Path.GetFileName(file).StartsWith($"{nameof(QuantConnect)}.", StringComparison.InvariantCulture))
+                        {
+                            foreach (var type in assembly.ExportedTypes.Where(type => !type.IsAbstract && !type.IsInterface && !type.IsEnum))
+                            {
+                                exportedTypes.Add(type);
+                            }
+                        }
+                        var asmCatalog = new AssemblyCatalog(assembly);
+                        var parts = asmCatalog.Parts.ToArray();
+                        if (parts.Length > 0)
+                        {
+                            catalogs.Add(asmCatalog);
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        if (!file.Contains("quickfix.fix", StringComparison.InvariantCultureIgnoreCase))
+                        {
+                            Log.Trace($"Composer.LoadPartsSafely({file}): Skipping {ex.GetType().Name}: {ex.Message}");
+                        }
+                    }
+                });
+
+                _exportedTypes = new List<Type>(exportedTypes);
+                var aggregate = new AggregateCatalog(catalogs);
+                _compositionContainer = new CompositionContainer(aggregate);
+                _composableParts = _compositionContainer.Catalog.Parts.ToList();
+            }
+            catch (Exception exception)
+            {
+                // ThreadAbortException is triggered when we shutdown ignore the error log
+                if (!(exception is ThreadAbortException))
+                {
+                    Log.Error(exception);
+                }
+            }
+        }
+    }
+}

Common/Util/ConcurrentSet.cs

@@ -0,0 +1,411 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Linq;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides a thread-safe set collection that mimics the behavior of <see cref="HashSet{T}"/>
+    /// and will be keep insertion order
+    /// </summary>
+    /// <typeparam name="T">The item type</typeparam>
+    public class ConcurrentSet<T> : ISet<T>
+    {
+        private readonly IEnumerator<T> _emptyEnumerator = Enumerable.Empty<T>().GetEnumerator();
+        private readonly OrderedDictionary _set = new OrderedDictionary();
+        // for performance we will keep a enumerator list which we will refresh only if required
+        private readonly List<T> _enumerator = new List<T>();
+        private bool _refreshEnumerator;
+
+        /// <summary>Gets the number of elements contained in the <see cref="T:System.Collections.Generic.ICollection`1" />.</summary>
+        /// <returns>The number of elements contained in the <see cref="T:System.Collections.Generic.ICollection`1" />.</returns>
+        public int Count
+        {
+            get
+            {
+                lock (_set)
+                {
+                    return _set.Count;
+                }
+            }
+        }
+
+        /// <summary>Gets a value indicating whether the <see cref="T:System.Collections.Generic.ICollection`1" /> is read-only.</summary>
+        /// <returns>true if the <see cref="T:System.Collections.Generic.ICollection`1" /> is read-only; otherwise, false.</returns>
+        public bool IsReadOnly => false;
+
+        /// <summary>Adds an item to the <see cref="T:System.Collections.Generic.ICollection`1" />.</summary>
+        /// <param name="item">The object to add to the <see cref="T:System.Collections.Generic.ICollection`1" />.</param>
+        /// <exception cref="T:System.NotSupportedException">The <see cref="T:System.Collections.Generic.ICollection`1" /> is read-only.</exception>
+        void ICollection<T>.Add(T item)
+        {
+            if (item == null)
+            {
+                throw new ArgumentNullException(nameof(item));
+            }
+
+            lock (_set)
+            {
+                AddImpl(item);
+            }
+        }
+
+        /// <summary>Modifies the current set so that it contains all elements that are present in either the current set or the specified collection.</summary>
+        /// <param name="other">The collection to compare to the current set.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public void UnionWith(IEnumerable<T> other)
+        {
+            var otherSet = other.ToHashSet();
+
+            lock (_set)
+            {
+                foreach (var item in otherSet)
+                {
+                    // wont overwrite existing references of same key
+                    AddImpl(item);
+                }
+            }
+        }
+
+        /// <summary>Modifies the current set so that it contains only elements that are also in a specified collection.</summary>
+        /// <param name="other">The collection to compare to the current set.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public void IntersectWith(IEnumerable<T> other)
+        {
+            var otherSet = other.ToHashSet();
+
+            lock (_set)
+            {
+                // remove items in '_set' that are not in 'other'
+                // enumerating this and not '_set' so its safe to modify
+                foreach (var item in this)
+                {
+                    if (!otherSet.Contains(item))
+                    {
+                        RemoveImpl(item);
+                    }
+                }
+            }
+        }
+
+        /// <summary>Removes all elements in the specified collection from the current set.</summary>
+        /// <param name="other">The collection of items to remove from the set.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public void ExceptWith(IEnumerable<T> other)
+        {
+            var otherSet = other.ToHashSet();
+
+            lock (_set)
+            {
+                // remove items from 'other'
+                foreach (var item in otherSet)
+                {
+                    RemoveImpl(item);
+                }
+            }
+        }
+
+        /// <summary>Modifies the current set so that it contains only elements that are present either in the current set or in the specified collection, but not both. </summary>
+        /// <param name="other">The collection to compare to the current set.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public void SymmetricExceptWith(IEnumerable<T> other)
+        {
+            var otherSet = other.ToHashSet();
+
+            lock (_set)
+            {
+                foreach (var item in otherSet)
+                {
+                    if (!AddImpl(item))
+                    {
+                        // remove items in both collections
+                        RemoveImpl(item);
+                    }
+                }
+            }
+        }
+
+        /// <summary>Determines whether a set is a subset of a specified collection.</summary>
+        /// <returns>true if the current set is a subset of <paramref name="other" />; otherwise, false.</returns>
+        /// <param name="other">The collection to compare to the current set.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public bool IsSubsetOf(IEnumerable<T> other)
+        {
+            var otherSet = other.ToHashSet();
+            lock (_set)
+            {
+                foreach (var item in otherSet)
+                {
+                    if (!_set.Contains(item))
+                    {
+                        return false;
+                    }
+                }
+
+                // non-strict subset can be equal
+                return _set.Keys.Count == 0;
+            }
+        }
+
+        /// <summary>Determines whether the current set is a superset of a specified collection.</summary>
+        /// <returns>true if the current set is a superset of <paramref name="other" />; otherwise, false.</returns>
+        /// <param name="other">The collection to compare to the current set.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public bool IsSupersetOf(IEnumerable<T> other)
+        {
+            var otherSet = other.ToHashSet();
+            lock (_set)
+            {
+                foreach (DictionaryEntry item in _set)
+                {
+                    if (!otherSet.Remove((T)item.Key))
+                    {
+                        return false;
+                    }
+                }
+            }
+
+            // non-strict superset can be equal
+            return true;
+        }
+
+        /// <summary>Determines whether the current set is a proper (strict) superset of a specified collection.</summary>
+        /// <returns>true if the current set is a proper superset of <paramref name="other" />; otherwise, false.</returns>
+        /// <param name="other">The collection to compare to the current set. </param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public bool IsProperSupersetOf(IEnumerable<T> other)
+        {
+            var hasOther = false;
+            var otherSet = other.ToHashSet();
+            lock (_set)
+            {
+                foreach (DictionaryEntry item in _set)
+                {
+                    if (!otherSet.Remove((T)item.Key))
+                    {
+                        return false;
+                    }
+
+                    hasOther = true;
+                }
+            }
+
+            // to be a strict superset, _set must contain extra elements and contain all of other
+            return hasOther;
+        }
+
+        /// <summary>Determines whether the current set is a proper (strict) subset of a specified collection.</summary>
+        /// <returns>true if the current set is a proper subset of <paramref name="other" />; otherwise, false.</returns>
+        /// <param name="other">The collection to compare to the current set.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public bool IsProperSubsetOf(IEnumerable<T> other)
+        {
+            var hasOther = false;
+            var otherSet = other.ToHashSet();
+            lock (_set)
+            {
+                foreach (var item in otherSet)
+                {
+                    if (!_set.Contains(item))
+                    {
+                        return false;
+                    }
+
+                    hasOther = true;
+                }
+
+                // to be a strict subset, other must contain extra elements and _set must contain all of other
+                return hasOther && _set.Keys.Count == 0;
+            }
+        }
+
+        /// <summary>Determines whether the current set overlaps with the specified collection.</summary>
+        /// <returns>true if the current set and <paramref name="other" /> share at least one common element; otherwise, false.</returns>
+        /// <param name="other">The collection to compare to the current set.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public bool Overlaps(IEnumerable<T> other)
+        {
+            var otherSet = other.ToHashSet();
+
+            lock (_set)
+            {
+                foreach (var item in otherSet)
+                {
+                    if (_set.Contains(item))
+                    {
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        /// <summary>Determines whether the current set and the specified collection contain the same elements.</summary>
+        /// <returns>true if the current set is equal to <paramref name="other" />; otherwise, false.</returns>
+        /// <param name="other">The collection to compare to the current set.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="other" /> is null.</exception>
+        public bool SetEquals(IEnumerable<T> other)
+        {
+            var otherSet = other.ToHashSet();
+            lock (_set)
+            {
+                foreach (DictionaryEntry item in _set)
+                {
+                    if (!otherSet.Remove((T) item.Key))
+                    {
+                        return false;
+                    }
+                }
+            }
+
+            return otherSet.Count == 0;
+        }
+
+        /// <summary>Adds an element to the current set and returns a value to indicate if the element was successfully added. </summary>
+        /// <returns>true if the element is added to the set; false if the element is already in the set.</returns>
+        /// <param name="item">The element to add to the set.</param>
+        public bool Add(T item)
+        {
+            lock (_set)
+            {
+                return AddImpl(item);
+            }
+        }
+
+        /// <summary>Removes all items from the <see cref="T:System.Collections.Generic.ICollection`1" />.</summary>
+        /// <exception cref="T:System.NotSupportedException">The <see cref="T:System.Collections.Generic.ICollection`1" /> is read-only. </exception>
+        public void Clear()
+        {
+            lock (_set)
+            {
+                _refreshEnumerator = true;
+                _set.Clear();
+            }
+        }
+
+        /// <summary>Determines whether the <see cref="T:System.Collections.Generic.ICollection`1" /> contains a specific value.</summary>
+        /// <returns>true if <paramref name="item" /> is found in the <see cref="T:System.Collections.Generic.ICollection`1" />; otherwise, false.</returns>
+        /// <param name="item">The object to locate in the <see cref="T:System.Collections.Generic.ICollection`1" />.</param>
+        public bool Contains(T item)
+        {
+            lock (_set)
+            {
+                return item != null && _set.Contains(item);
+            }
+        }
+
+        /// <summary>Copies the elements of the <see cref="T:System.Collections.Generic.ICollection`1" /> to an <see cref="T:System.Array" />, starting at a particular <see cref="T:System.Array" /> index.</summary>
+        /// <param name="array">The one-dimensional <see cref="T:System.Array" /> that is the destination of the elements copied from <see cref="T:System.Collections.Generic.ICollection`1" />. The <see cref="T:System.Array" /> must have zero-based indexing.</param>
+        /// <param name="arrayIndex">The zero-based index in <paramref name="array" /> at which copying begins.</param>
+        /// <exception cref="T:System.ArgumentNullException">
+        /// <paramref name="array" /> is null.</exception>
+        /// <exception cref="T:System.ArgumentOutOfRangeException">
+        /// <paramref name="arrayIndex" /> is less than 0.</exception>
+        /// <exception cref="T:System.ArgumentException">The number of elements in the source <see cref="T:System.Collections.Generic.ICollection`1" /> is greater than the available space from <paramref name="arrayIndex" /> to the end of the destination <paramref name="array" />.</exception>
+        public void CopyTo(T[] array, int arrayIndex)
+        {
+            lock (_set)
+            {
+                foreach (DictionaryEntry item in _set)
+                {
+                    array[arrayIndex++] = (T) item.Key;
+                }
+            }
+        }
+
+        /// <summary>Removes the first occurrence of a specific object from the <see cref="T:System.Collections.Generic.ICollection`1" />.</summary>
+        /// <returns>true if <paramref name="item" /> was successfully removed from the <see cref="T:System.Collections.Generic.ICollection`1" />; otherwise, false. This method also returns false if <paramref name="item" /> is not found in the original <see cref="T:System.Collections.Generic.ICollection`1" />.</returns>
+        /// <param name="item">The object to remove from the <see cref="T:System.Collections.Generic.ICollection`1" />.</param>
+        /// <exception cref="T:System.NotSupportedException">The <see cref="T:System.Collections.Generic.ICollection`1" /> is read-only.</exception>
+        public bool Remove(T item)
+        {
+            lock (_set)
+            {
+                if (item != null && _set.Contains(item))
+                {
+                    RemoveImpl(item);
+                    return true;
+                }
+                return false;
+            }
+        }
+
+        /// <summary>Returns an enumerator that iterates through the collection.</summary>
+        /// <remarks>For thread safety will return a snapshot of the collection</remarks>
+        /// <returns>A <see cref="T:System.Collections.Generic.IEnumerator`1" /> that can be used to iterate through the collection.</returns>
+        /// <filterpriority>1</filterpriority>
+        public IEnumerator<T> GetEnumerator()
+        {
+            lock (_set)
+            {
+                if (_refreshEnumerator)
+                {
+                    _enumerator.Clear();
+                    foreach (DictionaryEntry item in _set)
+                    {
+                        _enumerator.Add((T)item.Key);
+                    }
+
+                    _refreshEnumerator = false;
+                }
+
+                return _enumerator.Count == 0 ? _emptyEnumerator : _enumerator.GetEnumerator();
+            }
+        }
+
+        /// <summary>Returns an enumerator that iterates through a collection.</summary>
+        /// <remarks>For thread safety will return a snapshot of the collection</remarks>
+        /// <returns>An <see cref="T:System.Collections.IEnumerator" /> object that can be used to iterate through the collection.</returns>
+        /// <filterpriority>2</filterpriority>
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+
+        private bool AddImpl(T item)
+        {
+            if (!_set.Contains(item))
+            {
+                _refreshEnumerator = true;
+                _set.Add(item, item);
+                return true;
+            }
+
+            return false;
+        }
+
+        private void RemoveImpl(T item)
+        {
+            _refreshEnumerator = true;
+            _set.Remove(item);
+        }
+    }
+}

Common/Util/CurrencyPairUtil.cs

@@ -0,0 +1,272 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Securities;
+using QuantConnect.Securities.Cfd;
+using QuantConnect.Securities.Forex;
+using QuantConnect.Securities.Crypto;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Utility methods for decomposing and comparing currency pairs
+    /// </summary>
+    public static class CurrencyPairUtil
+    {
+        private static readonly Lazy<SymbolPropertiesDatabase> SymbolPropertiesDatabase =
+            new Lazy<SymbolPropertiesDatabase>(Securities.SymbolPropertiesDatabase.FromDataFolder);
+
+        private static readonly int[][] PotentialStableCoins = { [1, 3], [1, 2], [0, 3], [0, 2] };
+
+        /// <summary>
+        /// Tries to decomposes the specified currency pair into a base and quote currency provided as out parameters
+        /// </summary>
+        /// <param name="currencyPair">The input currency pair to be decomposed</param>
+        /// <param name="baseCurrency">The output base currency</param>
+        /// <param name="quoteCurrency">The output quote currency</param>
+        /// <returns>True if was able to decompose the currency pair</returns>
+        public static bool TryDecomposeCurrencyPair(Symbol currencyPair, out string baseCurrency, out string quoteCurrency)
+        {
+            baseCurrency = null;
+            quoteCurrency = null;
+
+            if (!IsValidSecurityType(currencyPair?.SecurityType, throwException: false))
+            {
+                return false;
+            }
+
+            try
+            {
+                DecomposeCurrencyPair(currencyPair, out baseCurrency, out quoteCurrency);
+                return true;
+            }
+            catch
+            {
+                // ignored
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Decomposes the specified currency pair into a base and quote currency provided as out parameters
+        /// </summary>
+        /// <param name="currencyPair">The input currency pair to be decomposed</param>
+        /// <param name="baseCurrency">The output base currency</param>
+        /// <param name="quoteCurrency">The output quote currency</param>
+        /// <param name="defaultQuoteCurrency">Optionally can provide a default quote currency</param>
+        public static void DecomposeCurrencyPair(Symbol currencyPair, out string baseCurrency, out string quoteCurrency, string defaultQuoteCurrency = Currencies.USD)
+        {
+            IsValidSecurityType(currencyPair?.SecurityType, throwException: true);
+            var securityType = currencyPair.SecurityType;
+
+            if (securityType == SecurityType.Forex)
+            {
+                Forex.DecomposeCurrencyPair(currencyPair.Value, out baseCurrency, out quoteCurrency);
+                return;
+            }
+
+            var symbolProperties = SymbolPropertiesDatabase.Value.GetSymbolProperties(
+                currencyPair.ID.Market,
+                currencyPair,
+                currencyPair.SecurityType,
+                defaultQuoteCurrency);
+
+            if (securityType == SecurityType.Cfd)
+            {
+                Cfd.DecomposeCurrencyPair(currencyPair, symbolProperties, out baseCurrency, out quoteCurrency);
+            }
+            else
+            {
+                Crypto.DecomposeCurrencyPair(currencyPair, symbolProperties, out baseCurrency, out quoteCurrency);
+            }
+        }
+
+        /// <summary>
+        /// Checks whether a symbol is decomposable into a base and a quote currency
+        /// </summary>
+        /// <param name="currencyPair">The pair to check for</param>
+        /// <returns>True if the pair can be decomposed into base and quote currencies, false if not</returns>
+        public static bool IsForexDecomposable(string currencyPair)
+        {
+            return !string.IsNullOrEmpty(currencyPair) && currencyPair.Length == 6;
+        }
+
+        /// <summary>
+        /// Checks whether a symbol is decomposable into a base and a quote currency
+        /// </summary>
+        /// <param name="currencyPair">The pair to check for</param>
+        /// <returns>True if the pair can be decomposed into base and quote currencies, false if not</returns>
+        public static bool IsDecomposable(Symbol currencyPair)
+        {
+            if (currencyPair == null)
+            {
+                return false;
+            }
+
+            if (currencyPair.SecurityType == SecurityType.Forex)
+            {
+                return currencyPair.Value.Length == 6;
+            }
+
+            if (currencyPair.SecurityType == SecurityType.Cfd || currencyPair.SecurityType == SecurityType.Crypto || currencyPair.SecurityType == SecurityType.CryptoFuture)
+            {
+                var symbolProperties = SymbolPropertiesDatabase.Value.GetSymbolProperties(
+                    currencyPair.ID.Market,
+                    currencyPair,
+                    currencyPair.SecurityType,
+                    Currencies.USD);
+
+                return currencyPair.Value.EndsWith(symbolProperties.QuoteCurrency);
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// You have currencyPair AB and one known symbol (A or B). This function returns the other symbol (B or A).
+        /// </summary>
+        /// <param name="currencyPair">Currency pair AB</param>
+        /// <param name="knownSymbol">Known part of the currencyPair (either A or B)</param>
+        /// <returns>The other part of currencyPair (either B or A), or null if known symbol is not part of currencyPair</returns>
+        public static string CurrencyPairDual(this Symbol currencyPair, string knownSymbol)
+        {
+            string baseCurrency;
+            string quoteCurrency;
+
+            DecomposeCurrencyPair(currencyPair, out baseCurrency, out quoteCurrency);
+
+            return CurrencyPairDual(baseCurrency, quoteCurrency, knownSymbol);
+        }
+
+        /// <summary>
+        /// You have currencyPair AB and one known symbol (A or B). This function returns the other symbol (B or A).
+        /// </summary>
+        /// <param name="baseCurrency">The base currency of the currency pair</param>
+        /// <param name="quoteCurrency">The quote currency of the currency pair</param>
+        /// <param name="knownSymbol">Known part of the currencyPair (either A or B)</param>
+        /// <returns>The other part of currencyPair (either B or A), or null if known symbol is not part of the currency pair</returns>
+        public static string CurrencyPairDual(string baseCurrency, string quoteCurrency, string knownSymbol)
+        {
+            if (baseCurrency == knownSymbol)
+            {
+                return quoteCurrency;
+            }
+
+            if (quoteCurrency == knownSymbol)
+            {
+                return baseCurrency;
+            }
+
+            return null;
+        }
+
+        /// <summary>
+        /// Represents the relation between two currency pairs
+        /// </summary>
+        public enum Match
+        {
+            /// <summary>
+            /// The two currency pairs don't match each other normally nor when one is reversed
+            /// </summary>
+            NoMatch,
+
+            /// <summary>
+            /// The two currency pairs match each other exactly
+            /// </summary>
+            ExactMatch,
+
+            /// <summary>
+            /// The two currency pairs are the inverse of each other
+            /// </summary>
+            InverseMatch
+        }
+
+        /// <summary>
+        /// Returns how two currency pairs are related to each other
+        /// </summary>
+        /// <param name="pairA">The first pair</param>
+        /// <param name="baseCurrencyB">The base currency of the second pair</param>
+        /// <param name="quoteCurrencyB">The quote currency of the second pair</param>
+        /// <returns>The <see cref="Match"/> member that represents the relation between the two pairs</returns>
+        public static Match ComparePair(this Symbol pairA, string baseCurrencyB, string quoteCurrencyB)
+        {
+            if (!TryDecomposeCurrencyPair(pairA, out var baseCurrencyA, out var quoteCurrencyA))
+            {
+                return Match.NoMatch;
+            }
+
+            // Check for a stablecoin between the currencies
+            var currencies = new string[] { baseCurrencyA, quoteCurrencyA, baseCurrencyB, quoteCurrencyB };
+            var isThereAnyMatch = false;
+
+            // Compute all the potential stablecoins
+            foreach (var pair in PotentialStableCoins)
+            {
+                if (Currencies.IsStableCoinWithoutPair(currencies[pair[0]] + currencies[pair[1]], pairA.ID.Market)
+                    || Currencies.IsStableCoinWithoutPair(currencies[pair[1]] + currencies[pair[0]], pairA.ID.Market))
+                {
+                    // If there's a stablecoin between them, assign to currency in pair A the value
+                    // of the currency in pair B 
+                    currencies[pair[0]] = currencies[pair[1]];
+                    isThereAnyMatch = true;
+                }
+            }
+
+            string pairAValue = isThereAnyMatch ? string.Concat(currencies[0], "||", currencies[1]) : string.Concat(baseCurrencyA, "||", quoteCurrencyA);
+
+            var directPair = string.Concat(baseCurrencyB, "||", quoteCurrencyB);
+            if (pairAValue == directPair)
+            {
+                return Match.ExactMatch;
+            }
+
+            var inversePair = string.Concat(quoteCurrencyB, "||", baseCurrencyB);
+            if (pairAValue == inversePair)
+            {
+                return Match.InverseMatch;
+            }
+
+            return Match.NoMatch;
+        }
+
+        public static bool IsValidSecurityType(SecurityType? securityType, bool throwException)
+        {
+            if (securityType == null)
+            {
+                if (throwException)
+                {
+                    throw new ArgumentException("Currency pair must not be null");
+                }
+                return false;
+            }
+
+            if (securityType != SecurityType.Forex &&
+                securityType != SecurityType.Cfd &&
+                securityType != SecurityType.Crypto &&
+                securityType != SecurityType.CryptoFuture)
+            {
+                if (throwException)
+                {
+                    throw new ArgumentException($"Unsupported security type: {securityType}");
+                }
+                return false;
+            }
+
+            return true;
+        }
+    }
+}

Common/Util/DateTimeJsonConverter.cs

@@ -0,0 +1,106 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using System.Collections.Generic;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides a json converter that allows defining the date time format used
+    /// </summary>
+    public class DateTimeJsonConverter : JsonConverter
+    {
+        private readonly List<IsoDateTimeConverter> _converters;
+
+        /// <summary>
+        /// True, can read a json into a date time
+        /// </summary>
+        public override bool CanRead => true;
+
+        /// <summary>
+        /// True, can write a datetime to json
+        /// </summary>
+        public override bool CanWrite => true;
+
+        /// <summary>
+        ///  Initializes a new instance of the <see cref="DateTimeJsonConverter"/> class
+        /// </summary>
+        /// <param name="format">>The date time format</param>
+        public DateTimeJsonConverter(string format)
+        {
+            _converters = [new IsoDateTimeConverter() { DateTimeFormat = format }];
+        }
+
+        /// <summary>
+        ///  Initializes a new instance of the <see cref="DateTimeJsonConverter"/> class
+        /// </summary>
+        /// <param name="format">>The date time format</param>
+        /// <param name="format2">Other format for backwards compatibility</param>
+        public DateTimeJsonConverter(string format, string format2) : this(format)
+        {
+            _converters.Add(new IsoDateTimeConverter() { DateTimeFormat = format2 });
+        }
+
+        /// <summary>
+        ///  Initializes a new instance of the <see cref="DateTimeJsonConverter"/> class
+        /// </summary>
+        /// <param name="format">>The date time format</param>
+        /// <param name="format2">Other format for backwards compatibility</param>
+        /// <param name="format3">Other format for backwards compatibility</param>
+        public DateTimeJsonConverter(string format, string format2, string format3) : this(format, format2)
+        {
+            _converters.Add(new IsoDateTimeConverter() { DateTimeFormat = format3 });
+        }
+
+        /// <summary>
+        /// True if can convert the given object type
+        /// </summary>
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(DateTime) || objectType == typeof(string) || objectType == typeof(DateTime?);
+        }
+
+        /// <summary>
+        /// Converts the given value
+        /// </summary>
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            foreach (var converter in _converters)
+            {
+                try
+                {
+                    return converter.ReadJson(reader, objectType, existingValue, serializer);
+                }
+                catch
+                {
+                }
+            }
+            throw new JsonSerializationException($"Unexpected value when converting date. Expected formats: {string.Join(",", _converters.Select(x => x.DateTimeFormat))}");
+        }
+
+        /// <summary>
+        /// Writes the given value to json
+        /// </summary>
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            _converters[0].WriteJson(writer, value, serializer);
+        }
+    }
+}

Common/Util/DecimalJsonConverter.cs

@@ -0,0 +1,99 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Globalization;
+using Newtonsoft.Json;
+
+namespace QuantConnect.Util;
+
+/// <summary>
+/// Json converter to represent decimals as strings
+/// </summary>
+public class DecimalJsonConverter : JsonConverter
+{
+    /// <summary>
+    /// Gets a value indicating whether this <see cref="JsonConverter"/> can write JSON.
+    /// </summary>
+    /// <remarks>
+    /// This property always returns <c>false</c>, indicating that this converter does not support writing JSON.
+    /// </remarks>
+    public override bool CanWrite => false;
+
+    /// <summary>
+    /// Determines whether this instance can convert the specified object type.
+    /// </summary>
+    /// <param name="objectType">Type of the object.</param>
+    /// <returns><c>true</c> if this instance can convert the specified object type; otherwise, <c>false</c>.</returns>
+    public override bool CanConvert(Type objectType)
+    {
+        return objectType == typeof(decimal) || objectType == typeof(decimal?);
+    }
+
+    /// <summary>
+    /// Writes the JSON representation of the object.
+    /// </summary>
+    /// <param name="writer">The <see cref="JsonWriter"/> to write to.</param>
+    /// <param name="value">The value.</param>
+    /// <param name="serializer">The calling serializer.</param>
+    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+    {
+        throw new NotImplementedException();
+    }
+
+    /// <summary>
+    /// Reads the JSON representation of the object.
+    /// </summary>
+    /// <param name="reader">The <see cref="JsonReader"/> to read from.</param>
+    /// <param name="objectType">Type of the object.</param>
+    /// <param name="existingValue">The existing value of object being read.</param>
+    /// <param name="serializer">The calling serializer.</param>
+    /// <returns>The object value.</returns>
+    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+    {
+        var val = reader.Value;
+
+        if (val == null)
+        {
+            if (objectType == typeof(decimal?))
+            {
+                return null;
+            }
+            throw new JsonSerializationException($"Cannot convert null value to {objectType}.");
+        }
+
+        if (val is decimal dec)
+        {
+            return dec;
+        }
+
+        if (val is double d)
+        {
+            return d.SafeDecimalCast();
+        }
+
+        if (val is string str && TryParse(str, out var res))
+        {
+            return res;
+        }
+
+        return Convert.ToDecimal(val, CultureInfo.InvariantCulture);
+    }
+
+    private static bool TryParse(string str, out decimal value)
+    {
+        return decimal.TryParse(str, NumberStyles.Any, CultureInfo.InvariantCulture, out value);
+    }
+}

Common/Util/DisposableExtensions.cs

@@ -0,0 +1,70 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Logging;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides extensions methods for <see cref="IDisposable"/>
+    /// </summary>
+    public static class DisposableExtensions
+    {
+        /// <summary>
+        /// Calls <see cref="IDisposable.Dispose"/> within a try/catch and logs any errors.
+        /// </summary>
+        /// <param name="disposable">The <see cref="IDisposable"/> to be disposed</param>
+        /// <returns>True if the object was successfully disposed, false if an error was thrown</returns>
+        public static bool DisposeSafely(this IDisposable disposable)
+        {
+            return disposable.DisposeSafely(error => Log.Error(error));
+        }
+
+        /// <summary>
+        /// Calls <see cref="IDisposable.Dispose"/> within a try/catch and invokes the
+        /// <paramref name="errorHandler"/> on any errors.
+        /// </summary>
+        /// <param name="disposable">The <see cref="IDisposable"/> to be disposed</param>
+        /// <param name="errorHandler">Error handler delegate invoked if an exception is thrown
+        /// while calling <see cref="IDisposable.Dispose"/></param> on <paramref name="disposable"/>
+        /// <returns>True if the object was successfully disposed, false if an error was thrown or
+        /// the specified disposable was null</returns>
+        public static bool DisposeSafely(this IDisposable disposable, Action<Exception> errorHandler)
+        {
+            if (disposable == null)
+            {
+                return false;
+            }
+
+            try
+            {
+                disposable.Dispose();
+                return true;
+            }
+            catch (ObjectDisposedException)
+            {
+                // we got what we wanted, the object has been disposed
+                return true;
+            }
+            catch (Exception error)
+            {
+                errorHandler(error);
+                return false;
+            }
+        }
+    }
+}

Common/Util/DoubleUnixSecondsDateTimeJsonConverter.cs

@@ -0,0 +1,71 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using Newtonsoft.Json;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Defines a <see cref="JsonConverter"/> that serializes <see cref="DateTime"/> use the number of whole and fractional seconds since unix epoch
+    /// </summary>
+    public class DoubleUnixSecondsDateTimeJsonConverter : TypeChangeJsonConverter<DateTime?, double?>
+    {
+        private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+
+        /// <summary>
+        /// Determines whether this instance can convert the specified object type.
+        /// </summary>
+        /// <param name="objectType">Type of the object.</param>
+        /// <returns>
+        /// <c>true</c> if this instance can convert the specified object type; otherwise, <c>false</c>.
+        /// </returns>
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(DateTime)
+                || objectType == typeof(DateTime?);
+        }
+
+        /// <summary>
+        /// Convert the input value to a value to be serialzied
+        /// </summary>
+        /// <param name="value">The input value to be converted before serialziation</param>
+        /// <returns>A new instance of TResult that is to be serialzied</returns>
+        protected override double? Convert(DateTime? value)
+        {
+            if (value == null)
+            {
+                return null;
+            }
+
+            return (value.Value - UnixEpoch).TotalSeconds;
+        }
+
+        /// <summary>
+        /// Converts the input value to be deserialized
+        /// </summary>
+        /// <param name="value">The deserialized value that needs to be converted to T</param>
+        /// <returns>The converted value</returns>
+        protected override DateTime? Convert(double? value)
+        {
+            if (value == null)
+            {
+                return null;
+            }
+
+            return UnixEpoch.AddSeconds(value.Value);
+        }
+    }
+}

Common/Util/EnumeratorExtensions.cs

@@ -0,0 +1,77 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides convenience of linq extension methods for <see cref="IEnumerator{T}"/> types
+    /// </summary>
+    public static class EnumeratorExtensions
+    {
+        /// <summary>
+        /// Filter the enumerator using the specified predicate
+        /// </summary>
+        public static IEnumerator<T> Where<T>(this IEnumerator<T> enumerator, Func<T, bool> predicate)
+        {
+            using (enumerator)
+            {
+                while (enumerator.MoveNext())
+                {
+                    if (predicate(enumerator.Current))
+                    {
+                        yield return enumerator.Current;
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Project the enumerator using the specified selector
+        /// </summary>
+        public static IEnumerator<TResult> Select<T, TResult>(this IEnumerator<T> enumerator, Func<T, TResult> selector)
+        {
+            using (enumerator)
+            {
+                while (enumerator.MoveNext())
+                {
+                    yield return selector(enumerator.Current);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Project the enumerator using the specified selector
+        /// </summary>
+        public static IEnumerator<TResult> SelectMany<T, TResult>(this IEnumerator<T> enumerator, Func<T, IEnumerator<TResult>> selector)
+        {
+            using (enumerator)
+            {
+                while (enumerator.MoveNext())
+                {
+                    using (var inner = selector(enumerator.Current))
+                    {
+                        while (inner.MoveNext())
+                        {
+                            yield return inner.Current;
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

Common/Util/ExpressionBuilder.cs

@@ -0,0 +1,152 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides methods for constructing expressions at runtime
+    /// </summary>
+    public static class ExpressionBuilder
+    {
+        /// <summary>
+        /// Constructs a selector of the form: x => x.propertyOrField where x is an instance of 'type'
+        /// </summary>
+        /// <param name="type">The type of the parameter in the expression</param>
+        /// <param name="propertyOrField">The name of the property or field to bind to</param>
+        /// <returns>A new lambda expression that represents accessing the property or field on 'type'</returns>
+        public static LambdaExpression MakePropertyOrFieldSelector(Type type, string propertyOrField)
+        {
+            var parameter = Expression.Parameter(type);
+            var property = Expression.PropertyOrField(parameter, propertyOrField);
+            var lambda = Expression.Lambda(property, parameter);
+            return lambda;
+        }
+
+        /// <summary>
+        /// Constructs a selector of the form: x => x.propertyOrField where x is an instance of 'type'
+        /// </summary>
+        /// <typeparam name="T">The type of the parameter in the expression</typeparam>
+        /// <typeparam name="TProperty">The type of the property or field being accessed in the expression</typeparam>
+        /// <param name="propertyOrField">The name of the property or field to bind to</param>
+        /// <returns>A new lambda expression that represents accessing the property or field on 'type'</returns>
+        public static Expression<Func<T, TProperty>> MakePropertyOrFieldSelector<T, TProperty>(string propertyOrField)
+        {
+            return (Expression<Func<T, TProperty>>) MakePropertyOrFieldSelector(typeof (T), propertyOrField);
+        }
+
+        /// <summary>
+        /// Constructs a lambda expression that accepts two parameters of type <typeparamref name="T"/> and applies
+        /// the specified binary comparison and returns the boolean result.
+        /// </summary>
+        public static Expression<Func<T, T, bool>> MakeBinaryComparisonLambda<T>(ExpressionType type)
+        {
+            if (!type.IsBinaryComparison())
+            {
+                throw new ArgumentException($"Provided ExpressionType '{type}' is not a binary comparison.");
+            }
+
+            var left = Expression.Parameter(typeof(T), "left");
+            var right = Expression.Parameter(typeof(T), "right");
+            var body = Expression.MakeBinary(type, left, right);
+            var lambda = Expression.Lambda<Func<T, T, bool>>(body, left, right);
+            return lambda;
+        }
+
+        /// <summary>
+        /// Determines whether or not the specified <paramref name="type"/> is a binary comparison.
+        /// </summary>
+        public static bool IsBinaryComparison(this ExpressionType type)
+        {
+            switch (type)
+            {
+                case ExpressionType.Equal:
+                case ExpressionType.NotEqual:
+                case ExpressionType.LessThan:
+                case ExpressionType.LessThanOrEqual:
+                case ExpressionType.GreaterThan:
+                case ExpressionType.GreaterThanOrEqual:
+                    return true;
+
+                default:
+                    return false;
+            }
+        }
+
+        /// <summary>
+        /// Converts the specified expression into an enumerable of expressions by walking the expression tree
+        /// </summary>
+        /// <param name="expression">The expression to enumerate</param>
+        /// <returns>An enumerable containing all expressions in the input expression</returns>
+        public static IEnumerable<Expression> AsEnumerable(this Expression expression)
+        {
+            var walker = new ExpressionWalker();
+            walker.Visit(expression);
+            return walker.Expressions;
+        }
+
+        /// <summary>
+        /// Returns all the expressions of the specified type in the given expression tree
+        /// </summary>
+        /// <typeparam name="T">The type of expression to search for</typeparam>
+        /// <param name="expression">The expression to search</param>
+        /// <returns>All expressions of the given type in the specified expression</returns>
+        public static IEnumerable<T> OfType<T>(this Expression expression)
+            where T : Expression
+        {
+            return expression.AsEnumerable().OfType<T>();
+        }
+
+        /// <summary>
+        /// Returns the single expression of the specified type or throws if none or more than one expression
+        /// of the specified type is contained within the expression.
+        /// </summary>
+        /// <typeparam name="T">The type of expression to search for</typeparam>
+        /// <param name="expression">The expression to search</param>
+        /// <returns>Expression of the specified type</returns>
+        public static T Single<T>(this Expression expression)
+            where T : Expression
+        {
+            return expression.AsEnumerable().OfType<T>().Single();
+        }
+
+        /// <summary>
+        /// Returns the single expression of the specified type or throws if none or more than one expression
+        /// of the specified type is contained within the expression.
+        /// </summary>
+        /// <typeparam name="T">The type of expression to search for</typeparam>
+        /// <param name="expressions">The expressions to search</param>
+        /// <returns>Expression of the specified type</returns>
+        public static T Single<T>(this IEnumerable<Expression> expressions)
+            where T : Expression
+        {
+            return expressions.OfType<T>().Single();
+        }
+
+        private class ExpressionWalker : ExpressionVisitor
+        {
+            public readonly HashSet<Expression> Expressions = new HashSet<Expression>();
+            public override Expression Visit(Expression node)
+            {
+                Expressions.Add(node);
+                return base.Visit(node);
+            }
+        }
+    }
+}

Common/Util/FixedSizeHashQueue.cs

@@ -0,0 +1,110 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides an implementation of an add-only fixed length, unique queue system
+    /// </summary>
+    public class FixedSizeHashQueue<T> : IEnumerable<T>
+    {
+        private readonly int _size;
+        private readonly Queue<T> _queue; 
+        private readonly HashSet<T> _hash; 
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FixedSizeHashQueue{T}"/> class
+        /// </summary>
+        /// <param name="size">The maximum number of items to hold</param>
+        public FixedSizeHashQueue(int size)
+        {
+            _size = size;
+            _queue = new Queue<T>(size);
+            _hash = new HashSet<T>();
+        }
+
+        /// <summary>
+        /// Returns true if the item was added and didn't already exists
+        /// </summary>
+        public bool Add(T item)
+        {
+            if (_hash.Add(item))
+            {
+                _queue.Enqueue(item);
+                if (_queue.Count > _size)
+                {
+                    // remove the item from both
+                    _hash.Remove(_queue.Dequeue());
+                }
+                return true;
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Tries to inspect the first item in the queue
+        /// </summary>
+        public bool TryPeek(out T item)
+        {
+            return _queue.TryPeek(out item);
+        }
+
+        /// <summary>
+        /// Dequeues and returns the next item in the queue
+        /// </summary>
+        public T Dequeue()
+        {
+            var item = _queue.Dequeue();
+            _hash.Remove(item);
+            return item;
+        }
+
+        /// <summary>
+        /// Returns true if the specified item exists in the collection
+        /// </summary>
+        public bool Contains(T item)
+        {
+            return _hash.Contains(item);
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through the collection.
+        /// </summary>
+        /// <returns>
+        /// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection.
+        /// </returns>
+        /// <filterpriority>1</filterpriority>
+        public IEnumerator<T> GetEnumerator()
+        {
+            return _queue.GetEnumerator();
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through a collection.
+        /// </summary>
+        /// <returns>
+        /// An <see cref="T:System.Collections.IEnumerator"/> object that can be used to iterate through the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+    }
+}

Common/Util/FixedSizeQueue.cs

@@ -0,0 +1,59 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+
+namespace QuantConnect.Util 
+{
+    /// <summary>
+    /// Helper method for a limited length queue which self-removes the extra elements.
+    /// http://stackoverflow.com/questions/5852863/fixed-size-queue-which-automatically-dequeues-old-values-upon-new-enques
+    /// </summary>
+    /// <typeparam name="T">The type of item the queue holds</typeparam>
+    public class FixedSizeQueue<T> : Queue<T>
+    {
+        private int _limit = -1;
+
+        /// <summary>
+        /// Max Length 
+        /// </summary>
+        public int Limit
+        {
+            get { return _limit; }
+            set { _limit = value; }
+        }
+
+        /// <summary>
+        /// Create a new fixed length queue:
+        /// </summary>
+        public FixedSizeQueue(int limit)
+            : base(limit)
+        {
+            Limit = limit;
+        }
+
+        /// <summary>
+        /// Enqueue a new item int the generic fixed length queue:
+        /// </summary>
+        public new void Enqueue(T item)
+        {
+            while (Count >= Limit)
+            {
+                Dequeue();
+            }
+            base.Enqueue(item);
+        }
+    }
+}

Common/Util/FuncTextWriter.cs

@@ -0,0 +1,65 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.IO;
+using System.Text;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="TextWriter"/> that redirects Write(string) and WriteLine(string)
+    /// </summary>
+    public class FuncTextWriter : TextWriter
+    {
+        private readonly Action<string> _writer;
+
+        /// <inheritdoc />
+        public override Encoding Encoding
+        {
+            get { return Encoding.Default; }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FuncTextWriter"/> that will direct
+        /// messages to the algorithm's Debug function.
+        /// </summary>
+        /// <param name="writer">The algorithm hosting the Debug function where messages will be directed</param>
+        public FuncTextWriter(Action<string> writer)
+        {
+            _writer = writer;
+        }
+
+        /// <summary>
+        /// Writes the string value using the delegate provided at construction
+        /// </summary>
+        /// <param name="value">The string value to be written</param>
+        public override void Write(string value)
+        {
+            _writer(value);
+        }
+
+        /// <summary>
+        /// Writes the string value using the delegate provided at construction
+        /// </summary>
+        /// <param name="value"></param>
+        public override void WriteLine(string value)
+        {
+            // these are grouped in a list so we don't need to add new line characters here
+            _writer(value);
+        }
+    }
+}

Common/Util/JsonRoundingConverter.cs

@@ -0,0 +1,83 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Globalization;
+using Newtonsoft.Json;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Helper <see cref="JsonConverter"/> that will round decimal and double types,
+    /// to <see cref="FractionalDigits"/> fractional digits
+    /// </summary>
+    public class JsonRoundingConverter : JsonConverter
+    {
+        /// <summary>
+        /// The number of fractional digits to round to
+        /// </summary>
+        public const int FractionalDigits = 4;
+
+        /// <summary>
+        /// Will always return false.
+        /// Gets a value indicating whether this <see cref="T:Newtonsoft.Json.JsonConverter" /> can read JSON.
+        /// </summary>
+        public override bool CanRead => false;
+
+        /// <summary>
+        /// Determines whether this instance can convert the specified object type.
+        /// </summary>
+        /// <param name="objectType">Type of the object.</param>
+        /// <returns>True if this instance can convert the specified object type</returns>
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(decimal)
+                || objectType == typeof(double);
+        }
+
+        /// <summary>
+        /// Not implemented, will throw <see cref="NotImplementedException"/>
+        /// </summary>
+        /// <param name="reader">The <see cref="T:Newtonsoft.Json.JsonReader"/> to read from.</param>
+        /// <param name="objectType">Type of the object.</param>
+        /// <param name="existingValue">The existing value of object being read.</param>
+        /// <param name="serializer">The calling serializer.</param>
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            throw new NotImplementedException();
+        }
+
+        /// <summary>
+        /// Writes the JSON representation of the object.
+        /// </summary>
+        /// <param name="writer">The <see cref="T:Newtonsoft.Json.JsonWriter"/> to write to.</param>
+        /// <param name="value">The value.</param>
+        /// <param name="serializer">The calling serializer.</param>
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            if (value is double)
+            {
+                var rounded = Math.Round((double)value, FractionalDigits);
+                writer.WriteValue(rounded.ToString(CultureInfo.InvariantCulture));
+            }
+            else
+            {
+                // we serialize decimal as string so that json doesn't use exponential notation which actually will lose precision
+                var rounded = Math.Round(Convert.ToDecimal(value, CultureInfo.InvariantCulture), FractionalDigits);
+                writer.WriteValue(rounded.ToString(CultureInfo.InvariantCulture));
+            }
+        }
+    }
+}

Common/Util/KeyStringSynchronizer.cs

@@ -0,0 +1,116 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Threading;
+using System.Collections.Generic;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Helper class to synchronize execution based on a string key
+    /// </summary>
+    public class KeyStringSynchronizer
+    {
+        private readonly Dictionary<string, string> _currentStrings = new ();
+
+        /// <summary>
+        /// Execute the given action synchronously with any other thread using the same key
+        /// </summary>
+        /// <param name="key">The synchronization key</param>
+        /// <param name="singleExecution">True if execution should happen only once at the same time for multiple threads</param>
+        /// <param name="action">The action to execute</param>
+        public void Execute(string key, bool singleExecution, Action action)
+        {
+            ExecuteImplementation(key, singleExecution, action);
+        }
+
+        /// <summary>
+        /// Execute the given function synchronously with any other thread using the same key
+        /// </summary>
+        /// <param name="key">The synchronization key</param>
+        /// <param name="action">The function to execute</param>
+        public T Execute<T>(string key, Func<T> action)
+        {
+            T result = default;
+            ExecuteImplementation(key, singleExecution: false, () =>
+            {
+                result = action();
+            });
+            return result;
+        }
+
+        private void ExecuteImplementation(string key, bool singleExecution, Action action)
+        {
+            lock (key)
+            {
+                while (true)
+                {
+                    bool lockTaken = false;
+                    string existingKey;
+                    lock (_currentStrings)
+                    {
+                        if(!_currentStrings.TryGetValue(key, out existingKey))
+                        {
+                            _currentStrings[key] = existingKey = key;
+                        }
+                    }
+
+                    try
+                    {
+                        lockTaken = Monitor.TryEnter(existingKey);
+                        // this way we can handle reentry with no issues
+                        if (lockTaken)
+                        {
+                            try
+                            {
+                                // happy case
+                                action();
+                                return;
+                            }
+                            finally
+                            {
+                                lock (_currentStrings)
+                                {
+                                    // even if we fail we need to release it
+                                    _currentStrings.Remove(key);
+                                }
+                            }
+                        }
+                    }
+                    finally
+                    {
+                        if (lockTaken)
+                        {
+                            Monitor.Exit(existingKey);
+                        }
+                    }
+
+                    lock (existingKey)
+                    {
+                        // if we are here the thread that had the lock finished
+                        if (!singleExecution)
+                        {
+                            // time to try again!
+                            continue;
+                        }
+                        return;
+                    }
+                }
+            }
+        }
+    }
+}

Common/Util/LeanDataPathComponents.cs

@@ -0,0 +1,200 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using QuantConnect.Securities.Option;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Type representing the various pieces of information emebedded into a lean data file path
+    /// </summary>
+    public class LeanDataPathComponents
+    {
+        /// <summary>
+        /// Gets the date component from the file name
+        /// </summary>
+        public DateTime Date
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the security type from the path
+        /// </summary>
+        public SecurityType SecurityType
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the market from the path
+        /// </summary>
+        public string Market
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the resolution from the path
+        /// </summary>
+        public Resolution Resolution
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the file name, not inluding directory information
+        /// </summary>
+        public string Filename
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the symbol object implied by the path. For options, or any
+        /// multi-entry zip file, this should be the canonical symbol
+        /// </summary>
+        public Symbol Symbol
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Gets the tick type from the file name
+        /// </summary>
+        public TickType TickType
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LeanDataPathComponents"/> class
+        /// </summary>
+        public LeanDataPathComponents(SecurityType securityType, string market, Resolution resolution, Symbol symbol, string filename, DateTime date, TickType tickType)
+        {
+            Date = date;
+            SecurityType = securityType;
+            Market = market;
+            Resolution = resolution;
+            Filename = filename;
+            Symbol = symbol;
+            TickType = tickType;
+        }
+
+        /// <summary>
+        /// Parses the specified path into a new instance of the <see cref="LeanDataPathComponents"/> class
+        /// </summary>
+        /// <param name="path">The path to be parsed</param>
+        /// <returns>A new instance of the <see cref="LeanDataPathComponents"/> class representing the specified path</returns>
+        public static LeanDataPathComponents Parse(string path)
+        {
+            //"../Data/equity/usa/hour/spy.zip"
+            //"../Data/equity/usa/hour/spy/20160218_trade.zip"
+            var fileinfo = new FileInfo(path);
+            var filename = fileinfo.Name;
+            var parts = path.Split('/', '\\');
+
+            // defines the offsets of the security relative to the end of the path
+            const int LowResSecurityTypeOffset = 4;
+            const int HighResSecurityTypeOffset = 5;
+
+            // defines other offsets relative to the beginning of the substring produce by the above offsets
+            const int MarketOffset = 1;
+            const int ResolutionOffset = 2;
+            const int TickerOffset = 3;
+
+
+            if (parts.Length < LowResSecurityTypeOffset)
+            {
+                throw new FormatException($"Unexpected path format: {path}");
+            }
+
+            var securityTypeOffset = LowResSecurityTypeOffset;
+            SecurityType securityType;
+            var rawValue = parts[parts.Length - securityTypeOffset];
+            if (!Enum.TryParse(rawValue, true, out securityType))
+            {
+                securityTypeOffset = HighResSecurityTypeOffset;
+                rawValue = parts[parts.Length - securityTypeOffset];
+                if (!Enum.TryParse(rawValue, true, out securityType))
+                {
+                    throw new FormatException($"Unexpected path format: {path}");
+                }
+            }
+
+            var market = parts[parts.Length - securityTypeOffset + MarketOffset];
+            var resolution = (Resolution) Enum.Parse(typeof (Resolution), parts[parts.Length - securityTypeOffset + ResolutionOffset], true);
+            string ticker;
+            if (securityTypeOffset == LowResSecurityTypeOffset)
+            {
+                ticker = Path.GetFileNameWithoutExtension(path);
+                if (securityType.IsOption())
+                {
+                    // ticker_year_trade_american
+                    ticker = ticker.Substring(0, ticker.IndexOf("_", StringComparison.InvariantCulture));
+                }
+                if (securityType == SecurityType.Future || securityType == SecurityType.CryptoFuture)
+                {
+                    // ticker_trade
+                    ticker = ticker.Substring(0, ticker.LastIndexOfInvariant("_"));
+                }
+                if (securityType == SecurityType.Crypto &&
+                    (resolution == Resolution.Daily || resolution == Resolution.Hour))
+                {
+                    // ticker_trade or ticker_quote
+                    ticker = ticker.Substring(0, ticker.LastIndexOfInvariant("_"));
+                }
+            }
+            else
+            {
+                ticker = parts[parts.Length - securityTypeOffset + TickerOffset];
+            }
+
+            var date = securityTypeOffset == LowResSecurityTypeOffset ? DateTime.MinValue : DateTime.ParseExact(filename.Substring(0, filename.IndexOf("_", StringComparison.Ordinal)), DateFormat.EightCharacter, null);
+
+            Symbol symbol;
+            if (securityType == SecurityType.Option)
+            {
+                var withoutExtension = Path.GetFileNameWithoutExtension(filename);
+                rawValue = withoutExtension.Substring(withoutExtension.LastIndexOf("_", StringComparison.Ordinal) + 1);
+                var style = (OptionStyle) Enum.Parse(typeof (OptionStyle), rawValue, true);
+                symbol = Symbol.CreateOption(ticker, market, style, default, 0, SecurityIdentifier.DefaultDate);
+            }
+            else if (securityType == SecurityType.FutureOption || securityType == SecurityType.IndexOption)
+            {
+                var withoutExtension = Path.GetFileNameWithoutExtension(filename);
+                rawValue = withoutExtension.Substring(withoutExtension.LastIndexOf("_", StringComparison.Ordinal) + 1);
+                var style = (OptionStyle) Enum.Parse(typeof (OptionStyle), rawValue, true);
+                var underlyingSecurityType = Symbol.GetUnderlyingFromOptionType(securityType);
+                var underlyingSymbol = Symbol.Create(OptionSymbol.MapToUnderlying(ticker, securityType), underlyingSecurityType, market);
+                symbol = Symbol.CreateOption(underlyingSymbol, ticker, market, style, default, 0, SecurityIdentifier.DefaultDate);
+            }
+            else if (securityType == SecurityType.Future)
+            {
+                symbol = Symbol.CreateFuture(ticker, market, SecurityIdentifier.DefaultDate);
+            }
+            else
+            {
+                symbol = Symbol.Create(ticker, securityType, market);
+            }
+
+            var tickType = filename.Contains("_quote") ? TickType.Quote : (filename.Contains("_openinterest") ? TickType.OpenInterest : TickType.Trade);
+
+            return new LeanDataPathComponents(securityType, market, resolution, symbol, filename, date, tickType);
+        }
+    }
+}

Common/Util/LinqExtensions.cs

@@ -0,0 +1,378 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Collections.ObjectModel;
+using System.Linq;
+using Common.Util;
+using QuantConnect.Data.Market;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides more extension methods for the enumerable types
+    /// </summary>
+    public static class LinqExtensions
+    {
+        /// <summary>
+        /// Creates a new read-only dictionary from the key value pairs
+        /// </summary>
+        /// <typeparam name="K">The key type</typeparam>
+        /// <typeparam name="V">The value type</typeparam>
+        /// <param name="enumerable">The IEnumerable of KeyValuePair instances to convert to a dictionary</param>
+        /// <returns>A read-only dictionary holding the same data as the enumerable</returns>
+        public static IReadOnlyDictionary<K, V> ToReadOnlyDictionary<K, V>(this IEnumerable<KeyValuePair<K, V>> enumerable)
+        {
+            return new ReadOnlyDictionary<K, V>(enumerable.ToDictionary());
+        }
+
+        /// <summary>
+        /// Creates a new <see cref="HashSet{T}"/> from the elements in the specified enumerable
+        /// </summary>
+        /// <typeparam name="T">The item type of the source enumerable</typeparam>
+        /// <typeparam name="TResult">The type of the items in the output <see cref="HashSet{T}"/></typeparam>
+        /// <param name="enumerable">The items to be placed into the enumerable</param>
+        /// <param name="selector">Selects items from the enumerable to be placed into the <see cref="HashSet{T}"/></param>
+        /// <returns>A new <see cref="HashSet{T}"/> containing the items in the enumerable</returns>
+        public static HashSet<TResult> ToHashSet<T, TResult>(this IEnumerable<T> enumerable, Func<T, TResult> selector)
+        {
+            return new HashSet<TResult>(enumerable.Select(selector));
+        }
+
+        /// <summary>
+        /// Creates a new <see cref="IList{T}"/> from the projected elements in the specified enumerable
+        /// </summary>
+        /// <typeparam name="T">The item type of the source enumerable</typeparam>
+        /// <typeparam name="TResult">The type of the items in the output <see cref="List{T}"/></typeparam>
+        /// <param name="enumerable">The items to be placed into the list</param>
+        /// <param name="selector">Selects items from the enumerable to be placed into the <see cref="List{T}"/></param>
+        /// <returns>A new <see cref="List{T}"/> containing the items in the enumerable</returns>
+        public static List<TResult> ToList<T, TResult>(this IEnumerable<T> enumerable, Func<T, TResult> selector)
+        {
+            return enumerable.Select(selector).ToList();
+        }
+
+        /// <summary>
+        /// Creates a new array from the projected elements in the specified enumerable
+        /// </summary>
+        /// <typeparam name="T">The item type of the source enumerable</typeparam>
+        /// <typeparam name="TResult">The type of the items in the output array</typeparam>
+        /// <param name="enumerable">The items to be placed into the array</param>
+        /// <param name="selector">Selects items from the enumerable to be placed into the array</param>
+        /// <returns>A new array containing the items in the enumerable</returns>
+        public static TResult[] ToArray<T, TResult>(this IEnumerable<T> enumerable, Func<T, TResult> selector)
+        {
+            return enumerable.Select(selector).ToArray();
+        }
+
+        /// <summary>
+        /// Creates a new immutable array from the projected elements in the specified enumerable
+        /// </summary>
+        /// <typeparam name="T">The item type of the source enumerable</typeparam>
+        /// <typeparam name="TResult">The type of the items in the output array</typeparam>
+        /// <param name="enumerable">The items to be placed into the array</param>
+        /// <param name="selector">Selects items from the enumerable to be placed into the array</param>
+        /// <returns>A new array containing the items in the enumerable</returns>
+        public static ImmutableArray<TResult> ToImmutableArray<T, TResult>(this IEnumerable<T> enumerable, Func<T, TResult> selector)
+        {
+            return enumerable.Select(selector).ToImmutableArray();
+        }
+
+        /// <summary>
+        /// Returns true if the specified enumerable is null or has no elements
+        /// </summary>
+        /// <typeparam name="T">The enumerable's item type</typeparam>
+        /// <param name="enumerable">The enumerable to check for a value</param>
+        /// <returns>True if the enumerable has elements, false otherwise</returns>
+        public static bool IsNullOrEmpty<T>(this IEnumerable<T> enumerable)
+        {
+            return enumerable == null || !enumerable.Any();
+        }
+
+        /// <summary>
+        /// Gets the median value in the collection
+        /// </summary>
+        /// <typeparam name="T">The item type in the collection</typeparam>
+        /// <param name="enumerable">The enumerable of items to search</param>
+        /// <returns>The median value, throws InvalidOperationException if no items are present</returns>
+        public static T Median<T>(this IEnumerable<T> enumerable)
+        {
+            var collection = enumerable.ToList();
+            return collection.OrderBy(x => x).Skip(collection.Count / 2).First();
+        }
+
+        /// <summary>
+        /// Gets the median value in the collection
+        /// </summary>
+        /// <typeparam name="T">The item type in the collection</typeparam>
+        /// <typeparam name="TProperty">The type of the value selected</typeparam>
+        /// <param name="collection">The collection of items to search</param>
+        /// <param name="selector">Function used to select a value from collection items</param>
+        /// <returns>The median value, throws InvalidOperationException if no items are present</returns>
+        public static TProperty Median<T, TProperty>(this IEnumerable<T> collection, Func<T, TProperty> selector)
+        {
+            return collection.Select(selector).Median();
+        }
+
+        /// <summary>
+        /// Performs a binary search on the specified collection.
+        /// </summary>
+        /// <typeparam name="TItem">The type of the item.</typeparam>
+        /// <typeparam name="TSearch">The type of the searched item.</typeparam>
+        /// <param name="list">The list to be searched.</param>
+        /// <param name="value">The value to search for.</param>
+        /// <param name="comparer">The comparer that is used to compare the value with the list items.</param>
+        /// <returns>The index of the item if found, otherwise the bitwise complement where the value should be per MSDN specs</returns>
+        public static int BinarySearch<TItem, TSearch>(this IList<TItem> list, TSearch value, Func<TSearch, TItem, int> comparer)
+        {
+            if (list == null)
+            {
+                throw new ArgumentNullException(nameof(list));
+            }
+            if (comparer == null)
+            {
+                throw new ArgumentNullException(nameof(comparer));
+            }
+
+            var lower = 0;
+            var upper = list.Count - 1;
+
+            while (lower <= upper)
+            {
+                var middle = lower + (upper - lower) / 2;
+                var comparisonResult = comparer(value, list[middle]);
+                if (comparisonResult < 0)
+                {
+                    upper = middle - 1;
+                }
+                else if (comparisonResult > 0)
+                {
+                    lower = middle + 1;
+                }
+                else
+                {
+                    return middle;
+                }
+            }
+
+            return ~lower;
+        }
+
+        /// <summary>
+        /// Performs a binary search on the specified collection.
+        /// </summary>
+        /// <typeparam name="TItem">The type of the item.</typeparam>
+        /// <param name="list">The list to be searched.</param>
+        /// <param name="value">The value to search for.</param>
+        /// <returns>The index of the item if found, otherwise the bitwise complement where the value should be per MSDN specs</returns>
+        public static int BinarySearch<TItem>(this IList<TItem> list, TItem value)
+        {
+            return BinarySearch(list, value, Comparer<TItem>.Default);
+        }
+
+        /// <summary>
+        /// Performs a binary search on the specified collection.
+        /// </summary>
+        /// <typeparam name="TItem">The type of the item.</typeparam>
+        /// <param name="list">The list to be searched.</param>
+        /// <param name="value">The value to search for.</param>
+        /// <param name="comparer">The comparer that is used to compare the value with the list items.</param>
+        /// <returns>The index of the item if found, otherwise the bitwise complement where the value should be per MSDN specs</returns>
+        public static int BinarySearch<TItem>(this IList<TItem> list, TItem value, IComparer<TItem> comparer)
+        {
+            return list.BinarySearch(value, comparer.Compare);
+        }
+
+        /// <summary>
+        /// Wraps the specified enumerable such that it will only be enumerated once
+        /// </summary>
+        /// <typeparam name="T">The enumerable's element type</typeparam>
+        /// <param name="enumerable">The source enumerable to be wrapped</param>
+        /// <returns>A new enumerable that can be enumerated multiple times without re-enumerating the source enumerable</returns>
+        public static IEnumerable<T> Memoize<T>(this IEnumerable<T> enumerable)
+        {
+            if (enumerable is MemoizingEnumerable<T>) return enumerable;
+            return new MemoizingEnumerable<T>(enumerable);
+        }
+
+        /// <summary>
+        /// Produces the an enumerable of the range of values between start and end using the specified
+        /// incrementing function
+        /// </summary>
+        /// <typeparam name="T">The enumerable item type</typeparam>
+        /// <param name="start">The start of the range</param>
+        /// <param name="end">The end of the range, non-inclusive by default</param>
+        /// <param name="incrementer">The incrementing function, with argument of the current item</param>
+        /// <param name="includeEndPoint">True to emit the end point, false otherwise</param>
+        /// <returns>An enumerable of the range of items between start and end</returns>
+        public static IEnumerable<T> Range<T>(T start, T end, Func<T, T> incrementer, bool includeEndPoint = false)
+            where T : IComparable
+        {
+            var current = start;
+            if (includeEndPoint)
+            {
+                while (current.CompareTo(end) <= 0)
+                {
+                    yield return current;
+                    current = incrementer(current);
+                }
+            }
+            else
+            {
+                while (current.CompareTo(end) < 0)
+                {
+                    yield return current;
+                    current = incrementer(current);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Groups adjacent elements of the enumerale using the specified grouper function
+        /// </summary>
+        /// <typeparam name="T">The enumerable item type</typeparam>
+        /// <param name="enumerable">The source enumerable to be grouped</param>
+        /// <param name="grouper">A function that accepts the previous value and the next value and returns
+        /// true if the next value belongs in the same group as the previous value, otherwise returns false</param>
+        /// <returns>A new enumerable of the groups defined by grouper. These groups don't have a key
+        /// and are only grouped by being emitted separately from this enumerable</returns>
+        public static IEnumerable<IEnumerable<T>> GroupAdjacentBy<T>(this IEnumerable<T> enumerable, Func<T, T, bool> grouper)
+        {
+            using (var e = enumerable.GetEnumerator())
+            {
+                if (e.MoveNext())
+                {
+                    var list = new List<T> { e.Current };
+                    var pred = e.Current;
+                    while (e.MoveNext())
+                    {
+                        if (grouper(pred, e.Current))
+                        {
+                            list.Add(e.Current);
+                        }
+                        else
+                        {
+                            yield return list;
+                            list = new List<T> { e.Current };
+                        }
+                        pred = e.Current;
+                    }
+                    yield return list;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Determines if there are any differences between the left and right collections.
+        /// This method uses sets to improve performance and also uses lazy evaluation so if a
+        /// difference is found, true is immediately returned and evaluation is halted.
+        /// </summary>
+        /// <typeparam name="T">The item type</typeparam>
+        /// <param name="left">The left set</param>
+        /// <param name="right">The right set</param>
+        /// <returns>True if there are any differences between the two sets, false otherwise</returns>
+        public static bool AreDifferent<T>(this ISet<T> left, ISet<T> right)
+        {
+            if (ReferenceEquals(left, right))
+            {
+                return false;
+            }
+            return !left.SetEquals(right);
+        }
+
+        /// <summary>
+        /// Converts an <see cref="IEnumerator{T}"/> to an <see cref="IEnumerable{T}"/>
+        /// </summary>
+        /// <typeparam name="T">Collection element type</typeparam>
+        /// <param name="enumerator">The enumerator to convert to an enumerable</param>
+        /// <returns>An enumerable wrapping the specified enumerator</returns>
+        public static IEnumerable<T> AsEnumerable<T>(this IEnumerator<T> enumerator)
+        {
+            using (enumerator)
+            {
+                while (enumerator.MoveNext())
+                {
+                    yield return enumerator.Current;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets the value associated with the specified key or provided default value if key is not found.
+        /// </summary>
+        /// <typeparam name="K">The key type</typeparam>
+        /// <typeparam name="V">The value type</typeparam>
+        /// <param name="dictionary">The dictionary instance</param>
+        /// <param name="key">Lookup key</param>
+        /// <param name="defaultValue">Default value</param>
+        /// <returns>Value associated with the specified key or  default value</returns>
+        public static V GetValueOrDefault<K, V>(this IDictionary<K, V> dictionary, K key, V defaultValue = default(V))
+        {
+            V obj;
+            return dictionary.TryGetValue(key, out obj) ? obj : defaultValue;
+        }
+
+        /// <summary>
+        /// Performs an action for each element in collection source
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="source">Collection source</param>
+        /// <param name="action">An action to perform</param>
+        public static void DoForEach<T>(this IEnumerable<T> source, Action<T> action)
+        {
+            foreach (var element in source)
+            {
+                action(element);
+            }
+        }
+
+        /// <summary>
+        /// Converts a dictionary to a ReadOnlyExtendedDictionary
+        /// </summary>
+        public static ReadOnlyExtendedDictionary<TKey, TValue> ToReadOnlyExtendedDictionary<TKey, TValue>(
+            this IDictionary<TKey, TValue> dictionary)
+        {
+            return new ReadOnlyExtendedDictionary<TKey, TValue>(dictionary);
+        }
+
+        /// <summary>
+        /// Creates a ReadOnlyExtendedDictionary from an IEnumerable according to specified key selector
+        /// </summary>
+        public static ReadOnlyExtendedDictionary<TKey, TValue> ToReadOnlyExtendedDictionary<TValue, TKey>(
+            this IEnumerable<TValue> source,
+            Func<TValue, TKey> keySelector)
+        {
+            return new ReadOnlyExtendedDictionary<TKey, TValue>(source, keySelector);
+        }
+
+        /// <summary>
+        /// Creates a DataDictionary from an IEnumerable according to specified key and value selectors
+        /// </summary>
+        public static DataDictionary<TValue> ToDataDictionary<TSource, TValue>(
+            this IEnumerable<TSource> source,
+            Func<TSource, Symbol> keySelector,
+            Func<TSource, TValue> valueSelector)
+        {
+            var result = new DataDictionary<TValue>();
+            foreach (var item in source)
+            {
+                result.Add(keySelector(item), valueSelector(item));
+            }
+            return result;
+        }
+    }
+}

Common/Util/ListComparer.cs

@@ -0,0 +1,48 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// An implementation of <see cref="IEqualityComparer{T}"/> for <see cref="List{T}"/>.
+    /// Useful when using a <see cref="List{T}"/> as the key of a collection.
+    /// </summary>
+    /// <typeparam name="T">The list type</typeparam>
+    public class ListComparer<T> : IEqualityComparer<IReadOnlyCollection<T>>
+    {
+        /// <summary>Determines whether the specified objects are equal.</summary>
+        /// <returns>true if the specified objects are equal; otherwise, false.</returns>
+        public bool Equals(IReadOnlyCollection<T> x, IReadOnlyCollection<T> y)
+        {
+            return x.SequenceEqual(y);
+        }
+
+        /// <summary>Returns a hash code for the specified object.</summary>
+        /// <returns>A hash code for the specified object created from combining the hash
+        /// code of all the elements in the collection.</returns>
+        public int GetHashCode(IReadOnlyCollection<T> obj)
+        {
+            var hashCode = 0;
+            foreach (var dateTime in obj)
+            {
+                hashCode = (hashCode * 397) ^ dateTime.GetHashCode();
+            }
+            return hashCode;
+        }
+    }
+}

Common/Util/MarketHoursDatabaseJsonConverter.cs

@@ -0,0 +1,375 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NodaTime;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+using QuantConnect.Securities.Option;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides json conversion for the <see cref="MarketHoursDatabase"/> class
+    /// </summary>
+    public class MarketHoursDatabaseJsonConverter : TypeChangeJsonConverter<MarketHoursDatabase, MarketHoursDatabaseJsonConverter.MarketHoursDatabaseJson>
+    {
+        /// <summary>
+        /// Convert the input value to a value to be serialzied
+        /// </summary>
+        /// <param name="value">The input value to be converted before serialziation</param>
+        /// <returns>A new instance of TResult that is to be serialzied</returns>
+        protected override MarketHoursDatabaseJson Convert(MarketHoursDatabase value)
+        {
+            return new MarketHoursDatabaseJson(value);
+        }
+
+        /// <summary>
+        /// Converts the input value to be deserialized
+        /// </summary>
+        /// <param name="value">The deserialized value that needs to be converted to T</param>
+        /// <returns>The converted value</returns>
+        protected override MarketHoursDatabase Convert(MarketHoursDatabaseJson value)
+        {
+            return value.Convert();
+        }
+
+        /// <summary>
+        /// Creates an instance of the un-projected type to be deserialized
+        /// </summary>
+        /// <param name="type">The input object type, this is the data held in the token</param>
+        /// <param name="token">The input data to be converted into a T</param>
+        /// <returns>A new instance of T that is to be serialized using default rules</returns>
+        protected override MarketHoursDatabase Create(Type type, JToken token)
+        {
+            var jobject = (JObject) token;
+            var instance = jobject.ToObject<MarketHoursDatabaseJson>();
+            return Convert(instance);
+        }
+
+        /// <summary>
+        /// Defines the json structure of the market-hours-database.json file
+        /// </summary>
+        [JsonObject(MemberSerialization.OptIn)]
+        public class MarketHoursDatabaseJson
+        {
+            /// <summary>
+            /// The entries in the market hours database, keyed by <see cref="SecurityDatabaseKey"/>
+            /// </summary>
+            [JsonProperty("entries")]
+            public Dictionary<string, MarketHoursDatabaseEntryJson> Entries { get; set; }
+
+            /// <summary>
+            /// Initializes a new instance of the <see cref="MarketHoursDatabaseJson"/> class
+            /// </summary>
+            /// <param name="database">The database instance to copy</param>
+            public MarketHoursDatabaseJson(MarketHoursDatabase database)
+            {
+                if (database == null) return;
+                Entries = new Dictionary<string, MarketHoursDatabaseEntryJson>();
+                foreach (var kvp in database.ExchangeHoursListing)
+                {
+                    var key = kvp.Key;
+                    var entry = kvp.Value;
+                    Entries[key.ToString()] = new MarketHoursDatabaseEntryJson(entry);
+                }
+            }
+
+            /// <summary>
+            /// Converts this json representation to the <see cref="MarketHoursDatabase"/> type
+            /// </summary>
+            /// <returns>A new instance of the <see cref="MarketHoursDatabase"/> class</returns>
+            public MarketHoursDatabase Convert()
+            {
+                // first we parse the entries keys so that later we can sort by security type
+                var entries = new Dictionary<SecurityDatabaseKey, MarketHoursDatabaseEntryJson>(Entries.Count);
+                foreach (var entry in Entries)
+                {
+                    try
+                    {
+                        var key = SecurityDatabaseKey.Parse(entry.Key);
+                        if (key != null)
+                        {
+                            entries[key] = entry.Value;
+                        }
+                    }
+                    catch (Exception err)
+                    {
+                        Log.Error(err);
+                    }
+                }
+
+                var result = new Dictionary<SecurityDatabaseKey, MarketHoursDatabase.Entry>(Entries.Count);
+                // we sort so we process generic entries and non options first
+                foreach (var entry in entries.OrderBy(kvp => kvp.Key.Symbol != null ? 1 : 0).ThenBy(kvp => kvp.Key.SecurityType.IsOption() ? 1 : 0))
+                {
+                    try
+                    {
+                        result.TryGetValue(entry.Key.CreateCommonKey(), out var marketEntry);
+                        var underlyingEntry = GetUnderlyingEntry(entry.Key, result);
+                        result[entry.Key] = entry.Value.Convert(underlyingEntry, marketEntry);
+                    }
+                    catch (Exception err)
+                    {
+                        Log.Error(err);
+                    }
+                }
+                return new MarketHoursDatabase(result);
+            }
+
+            /// <summary>
+            /// Helper method to get the already processed underlying entry for options
+            /// </summary>
+            private static MarketHoursDatabase.Entry GetUnderlyingEntry(SecurityDatabaseKey key, Dictionary<SecurityDatabaseKey, MarketHoursDatabase.Entry> result)
+            {
+                MarketHoursDatabase.Entry underlyingEntry = null;
+                if (key.SecurityType.IsOption())
+                {
+                    // if option, let's get the underlyings entry
+                    var underlyingSecurityType = Symbol.GetUnderlyingFromOptionType(key.SecurityType);
+                    var underlying = OptionSymbol.MapToUnderlying(key.Symbol, key.SecurityType);
+                    var underlyingKey = new SecurityDatabaseKey(key.Market, underlying, underlyingSecurityType);
+
+                    if (!result.TryGetValue(underlyingKey, out underlyingEntry)
+                        // let's retry with the wildcard
+                        && underlying != SecurityDatabaseKey.Wildcard)
+                    {
+                        var underlyingKeyWildCard = new SecurityDatabaseKey(key.Market, SecurityDatabaseKey.Wildcard, underlyingSecurityType);
+                        result.TryGetValue(underlyingKeyWildCard, out underlyingEntry);
+                    }
+                }
+                return underlyingEntry;
+            }
+        }
+
+        /// <summary>
+        /// Defines the json structure of a single entry in the market-hours-database.json file
+        /// </summary>
+        [JsonObject(MemberSerialization.OptIn)]
+        public class MarketHoursDatabaseEntryJson
+        {
+            /// <summary>
+            /// The data's raw time zone
+            /// </summary>
+            [JsonProperty("dataTimeZone")]
+            public string DataTimeZone { get; set; }
+
+            /// <summary>
+            /// The exchange's time zone id from the tzdb
+            /// </summary>
+            [JsonProperty("exchangeTimeZone")]
+            public string ExchangeTimeZone { get; set; }
+
+            /// <summary>
+            /// Sunday market hours segments
+            /// </summary>
+            [JsonProperty("sunday")]
+            public List<MarketHoursSegment> Sunday { get; set; }
+
+            /// <summary>
+            /// Monday market hours segments
+            /// </summary>
+            [JsonProperty("monday")]
+            public List<MarketHoursSegment> Monday { get; set; }
+
+            /// <summary>
+            /// Tuesday market hours segments
+            /// </summary>
+            [JsonProperty("tuesday")]
+            public List<MarketHoursSegment> Tuesday { get; set; }
+
+            /// <summary>
+            /// Wednesday market hours segments
+            /// </summary>
+            [JsonProperty("wednesday")]
+            public List<MarketHoursSegment> Wednesday { get; set; }
+
+            /// <summary>
+            /// Thursday market hours segments
+            /// </summary>
+            [JsonProperty("thursday")]
+            public List<MarketHoursSegment> Thursday { get; set; }
+
+            /// <summary>
+            /// Friday market hours segments
+            /// </summary>
+            [JsonProperty("friday")]
+            public List<MarketHoursSegment> Friday { get; set; }
+
+            /// <summary>
+            /// Saturday market hours segments
+            /// </summary>
+            [JsonProperty("saturday")]
+            public List<MarketHoursSegment> Saturday { get; set; }
+
+            /// <summary>
+            /// Holiday date strings
+            /// </summary>
+            [JsonProperty("holidays")]
+            public List<string> Holidays { get; set; } = new();
+
+            /// <summary>
+            /// Early closes by date
+            /// </summary>
+            [JsonProperty("earlyCloses")]
+            public Dictionary<string, TimeSpan> EarlyCloses { get; set; } = new Dictionary<string, TimeSpan>();
+
+            /// <summary>
+            /// Late opens by date
+            /// </summary>
+            [JsonProperty("lateOpens")]
+            public Dictionary<string, TimeSpan> LateOpens { get; set; } = new Dictionary<string, TimeSpan>();
+
+            /// <summary>
+            /// Bank holidays date strings
+            /// </summary>
+            [JsonProperty("bankHolidays")]
+            public List<string> BankHolidays { get; set; } = new();
+
+            /// <summary>
+            /// Initializes a new instance of the <see cref="MarketHoursDatabaseEntryJson"/> class
+            /// </summary>
+            /// <param name="entry">The entry instance to copy</param>
+            public MarketHoursDatabaseEntryJson(MarketHoursDatabase.Entry entry)
+            {
+                if (entry == null) return;
+                DataTimeZone = entry.DataTimeZone.Id;
+                var hours = entry.ExchangeHours;
+                ExchangeTimeZone = hours.TimeZone.Id;
+                SetSegmentsForDay(hours, DayOfWeek.Sunday, out var sunday);
+                Sunday = sunday;
+                SetSegmentsForDay(hours, DayOfWeek.Monday, out var monday);
+                Monday = monday;
+                SetSegmentsForDay(hours, DayOfWeek.Tuesday, out var tuesday);
+                Tuesday = tuesday;
+                SetSegmentsForDay(hours, DayOfWeek.Wednesday, out var wednesday);
+                Wednesday = wednesday;
+                SetSegmentsForDay(hours, DayOfWeek.Thursday, out var thursday);
+                Thursday = thursday;
+                SetSegmentsForDay(hours, DayOfWeek.Friday, out var friday);
+                Friday = friday;
+                SetSegmentsForDay(hours, DayOfWeek.Saturday, out var saturday);
+                Saturday = saturday;
+                Holidays = hours.Holidays.Select(x => x.ToString("M/d/yyyy", CultureInfo.InvariantCulture)).ToList();
+                EarlyCloses = entry.ExchangeHours.EarlyCloses.ToDictionary(pair => pair.Key.ToString("M/d/yyyy", CultureInfo.InvariantCulture), pair => pair.Value);
+                LateOpens = entry.ExchangeHours.LateOpens.ToDictionary(pair => pair.Key.ToString("M/d/yyyy", CultureInfo.InvariantCulture), pair => pair.Value);
+            }
+
+            /// <summary>
+            /// Converts this json representation to the <see cref="MarketHoursDatabase.Entry"/> type
+            /// </summary>
+            /// <returns>A new instance of the <see cref="MarketHoursDatabase.Entry"/> class</returns>
+            public MarketHoursDatabase.Entry Convert(MarketHoursDatabase.Entry underlyingEntry, MarketHoursDatabase.Entry marketEntry)
+            {
+                var hours = new Dictionary<DayOfWeek, LocalMarketHours>
+                {
+                    { DayOfWeek.Sunday, new LocalMarketHours(DayOfWeek.Sunday, Sunday) },
+                    { DayOfWeek.Monday, new LocalMarketHours(DayOfWeek.Monday, Monday) },
+                    { DayOfWeek.Tuesday, new LocalMarketHours(DayOfWeek.Tuesday, Tuesday) },
+                    { DayOfWeek.Wednesday, new LocalMarketHours(DayOfWeek.Wednesday, Wednesday) },
+                    { DayOfWeek.Thursday, new LocalMarketHours(DayOfWeek.Thursday, Thursday) },
+                    { DayOfWeek.Friday, new LocalMarketHours(DayOfWeek.Friday, Friday) },
+                    { DayOfWeek.Saturday, new LocalMarketHours(DayOfWeek.Saturday, Saturday) }
+                };
+                var holidayDates = Holidays.Select(x => DateTime.ParseExact(x, "M/d/yyyy", CultureInfo.InvariantCulture)).ToHashSet();
+                var bankHolidayDates = BankHolidays.Select(x => DateTime.ParseExact(x, "M/d/yyyy", CultureInfo.InvariantCulture)).ToHashSet();
+                IReadOnlyDictionary<DateTime, TimeSpan> earlyCloses = EarlyCloses.ToDictionary(x => DateTime.ParseExact(x.Key, "M/d/yyyy", CultureInfo.InvariantCulture), x => x.Value);
+                IReadOnlyDictionary<DateTime, TimeSpan> lateOpens = LateOpens.ToDictionary(x => DateTime.ParseExact(x.Key, "M/d/yyyy", CultureInfo.InvariantCulture), x => x.Value);
+
+                if(underlyingEntry != null)
+                {
+                    // If we have no entries but the underlying does, let's use the underlyings
+                    if (holidayDates.Count == 0)
+                    {
+                        holidayDates = underlyingEntry.ExchangeHours.Holidays;
+                    }
+                    if (bankHolidayDates.Count == 0)
+                    {
+                        bankHolidayDates = underlyingEntry.ExchangeHours.BankHolidays;
+                    }
+                    if (earlyCloses.Count == 0)
+                    {
+                        earlyCloses = underlyingEntry.ExchangeHours.EarlyCloses;
+                    }
+                    if (lateOpens.Count == 0)
+                    {
+                        lateOpens = underlyingEntry.ExchangeHours.LateOpens;
+                    }
+                }
+
+                if(marketEntry != null)
+                {
+                    if (marketEntry.ExchangeHours.Holidays.Count > 0)
+                    {
+                        holidayDates.UnionWith(marketEntry.ExchangeHours.Holidays);
+                    }
+
+                    if (marketEntry.ExchangeHours.BankHolidays.Count > 0)
+                    {
+                        bankHolidayDates.UnionWith(marketEntry.ExchangeHours.BankHolidays);
+                    }
+
+                    if (marketEntry.ExchangeHours.EarlyCloses.Count > 0 )
+                    {
+                        earlyCloses = MergeLateOpensAndEarlyCloses(marketEntry.ExchangeHours.EarlyCloses, earlyCloses);
+                    }
+
+                    if (marketEntry.ExchangeHours.LateOpens.Count > 0)
+                    {
+                        lateOpens = MergeLateOpensAndEarlyCloses(marketEntry.ExchangeHours.LateOpens, lateOpens);
+                    }
+                }
+
+                var exchangeHours = new SecurityExchangeHours(DateTimeZoneProviders.Tzdb[ExchangeTimeZone], holidayDates, hours, earlyCloses, lateOpens, bankHolidayDates);
+                return new MarketHoursDatabase.Entry(DateTimeZoneProviders.Tzdb[DataTimeZone], exchangeHours);
+            }
+
+            private void SetSegmentsForDay(SecurityExchangeHours hours, DayOfWeek day, out List<MarketHoursSegment> segments)
+            {
+                LocalMarketHours local;
+                if (hours.MarketHours.TryGetValue(day, out local))
+                {
+                    segments = local.Segments.ToList();
+                }
+                else
+                {
+                    segments = new List<MarketHoursSegment>();
+                }
+            }
+
+            /// <summary>
+            /// Merges the late opens or early closes from the common entry (with wildcards) with the specific entry
+            /// (e.g. Indices-usa-[*] with Indices-usa-VIX).
+            /// The specific entry takes precedence.
+            /// </summary>
+            private static Dictionary<DateTime, TimeSpan> MergeLateOpensAndEarlyCloses(IReadOnlyDictionary<DateTime, TimeSpan> common,
+                IReadOnlyDictionary<DateTime, TimeSpan> specific)
+            {
+                var result = common.ToDictionary();
+                foreach (var (key, value) in specific)
+                {
+                    result[key] = value;
+                }
+
+                return result;
+            }
+        }
+    }
+}

Common/Util/MemoizingEnumerable.cs

@@ -0,0 +1,169 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Defines an enumerable that can be enumerated many times while
+    /// only performing a single enumeration of the root enumerable
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public class MemoizingEnumerable<T> : IEnumerable<T>
+    {
+        private List<T> _buffer;
+        private IEnumerator<T> _enumerator;
+
+        /// <summary>
+        /// Allow disableing the buffering
+        /// </summary>
+        /// <remarks>Should be called before the enumeration starts</remarks>
+        public bool Enabled { get; set; }
+
+        /// <summary>
+        /// Gets the count of items in the enumerable. This will force enumeration of the entire collection if it has not already been enumerated.
+        /// </summary>
+        public int Count
+        {
+            get
+            {
+                if (!Enabled)
+                {
+                    throw new InvalidOperationException("Count is not supported when memoization is disabled");
+                }
+                if (_buffer == null)
+                {
+                    foreach (var item in this) {}
+                }
+                return _buffer.Count;
+            }
+        }
+
+        /// <summary>
+        /// Gets whether the enumerable is empty. This will force enumeration of the first item if it has not already been enumerated.
+        /// </summary>
+        public bool Empty
+        {
+            get
+            {
+                if (!Enabled)
+                {
+                    throw new InvalidOperationException("Empty is not supported when memoization is disabled");
+                }
+                if (_buffer == null || _buffer.Count == 0)
+                {
+                    using var enumerator = GetEnumerator();
+                    return !enumerator.MoveNext();
+                }
+                return _buffer.Count == 0;
+            }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="MemoizingEnumerable{T}"/> class
+        /// </summary>
+        /// <param name="enumerable">The source enumerable to be memoized</param>
+        public MemoizingEnumerable(IEnumerable<T> enumerable)
+        {
+            Enabled = true;
+            _enumerator = enumerable.GetEnumerator();
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through the collection.
+        /// </summary>
+        /// <returns>
+        /// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection.
+        /// </returns>
+        /// <filterpriority>1</filterpriority>
+        public IEnumerator<T> GetEnumerator()
+        {
+            if (!Enabled)
+            {
+                if (_enumerator != null)
+                {
+                    while (_enumerator.MoveNext())
+                    {
+                        yield return _enumerator.Current;
+                    }
+
+                    // important to avoid leak!
+                    _enumerator.Dispose();
+                    _enumerator = null;
+                }
+            }
+            else
+            {
+                if (_buffer == null)
+                {
+                    // lazy create our buffer
+                    _buffer = new List<T>();
+                }
+
+                int i = 0;
+                while (i <= _buffer.Count)
+                {
+                    // sync for multiple threads access to _enumerator and _buffer
+                    lock (_buffer)
+                    {
+                        // check to see if we need to move next
+                        if (_enumerator != null && i >= _buffer.Count)
+                        {
+                            if (_enumerator.MoveNext())
+                            {
+                                var value = _enumerator.Current;
+                                _buffer.Add(value);
+                                yield return value;
+                            }
+                            else
+                            {
+                                // important to avoid leak!
+                                _enumerator.Dispose();
+                                _enumerator = null;
+                            }
+                        }
+                        else
+                        {
+                            // we have a value if it's in the buffer
+                            if (_buffer.Count > i)
+                            {
+                                yield return _buffer[i];
+                            }
+                        }
+                    }
+
+                    // increment for next time
+                    i++;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through a collection.
+        /// </summary>
+        /// <returns>
+        /// An <see cref="T:System.Collections.IEnumerator"/> object that can be used to iterate through the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+    }
+}

Common/Util/NullStringValueConverter.cs

@@ -0,0 +1,71 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using Newtonsoft.Json;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Converts the string "null" into a new instance of T.
+    /// This converter only handles deserialization concerns.
+    /// </summary>
+    /// <typeparam name="T">The output type of the converter</typeparam>
+    public class NullStringValueConverter<T> : JsonConverter
+        where T : new()
+    {
+        /// <summary>
+        /// Writes the JSON representation of the object.
+        /// </summary>
+        /// <param name="writer">The <see cref="T:Newtonsoft.Json.JsonWriter"/> to write to.</param>
+        /// <param name="value">The value.</param>
+        /// <param name="serializer">The calling serializer.</param>
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            throw new NotImplementedException();
+        }
+
+        /// <summary>
+        /// Reads the JSON representation of the object.
+        /// </summary>
+        /// <param name="reader">The <see cref="T:Newtonsoft.Json.JsonReader"/> to read from.</param>
+        /// <param name="objectType">Type of the object.</param>
+        /// <param name="existingValue">The existing value of object being read.</param>
+        /// <param name="serializer">The calling serializer.</param>
+        /// <returns>
+        /// The object value.
+        /// </returns>
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            if (reader.TokenType == JsonToken.Null || (reader.TokenType == JsonToken.String && (string)reader.Value == "null"))
+            {
+                return new T();
+            }
+            return serializer.Deserialize<T>(reader);
+        }
+
+        /// <summary>
+        /// Determines whether this instance can convert the specified object type.
+        /// </summary>
+        /// <param name="objectType">Type of the object.</param>
+        /// <returns>
+        /// <c>true</c> if this instance can convert the specified object type; otherwise, <c>false</c>.
+        /// </returns>
+        public override bool CanConvert(Type objectType)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

Common/Util/ObjectActivator.cs

@@ -0,0 +1,151 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using CloneExtensions;
+using Fasterflect;
+using QuantConnect.Logging;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides methods for creating new instances of objects
+    /// </summary>
+    public static class ObjectActivator
+    {
+        private static readonly object _lock = new object();
+        private static readonly Dictionary<Type, MethodInvoker> _cloneMethodsByType = new Dictionary<Type, MethodInvoker>();
+        private static readonly Dictionary<Type, Func<object[], object>> _activatorsByType = new Dictionary<Type, Func<object[], object>>();
+
+        static ObjectActivator()
+        {
+            // we can reuse the symbol instance in the clone since it's immutable
+            ((HashSet<Type>) CloneFactory.KnownImmutableTypes).Add(typeof (Symbol));
+            ((HashSet<Type>) CloneFactory.KnownImmutableTypes).Add(typeof (SecurityIdentifier));
+        }
+
+        /// <summary>
+        /// Fast Object Creator from Generic Type:
+        /// Modified from http://rogeralsing.com/2008/02/28/linq-expressions-creating-objects/
+        /// </summary>
+        /// <remarks>This assumes that the type has a parameterless, default constructor</remarks>
+        /// <param name="dataType">Type of the object we wish to create</param>
+        /// <returns>Method to return an instance of object</returns>
+        public static Func<object[], object> GetActivator(Type dataType)
+        {
+            lock (_lock)
+            {
+                // if we already have it, just use it
+                Func<object[], object> factory;
+                if (_activatorsByType.TryGetValue(dataType, out factory))
+                {
+                    return factory;
+                }
+
+                var ctor = dataType.GetConstructor(new Type[] {});
+
+                //User has forgotten to include a parameterless constructor:
+                if (ctor == null) return null;
+
+                var paramsInfo = ctor.GetParameters();
+
+                //create a single param of type object[]
+                var param = Expression.Parameter(typeof (object[]), "args");
+                var argsExp = new Expression[paramsInfo.Length];
+
+                for (var i = 0; i < paramsInfo.Length; i++)
+                {
+                    var index = Expression.Constant(i);
+                    var paramType = paramsInfo[i].ParameterType;
+                    var paramAccessorExp = Expression.ArrayIndex(param, index);
+                    var paramCastExp = Expression.Convert(paramAccessorExp, paramType);
+                    argsExp[i] = paramCastExp;
+                }
+
+                var newExp = Expression.New(ctor, argsExp);
+                var lambda = Expression.Lambda(typeof (Func<object[], object>), newExp, param);
+                factory = (Func<object[], object>) lambda.Compile();
+
+                // save it for later
+                _activatorsByType.Add(dataType, factory);
+
+                return factory;
+            }
+        }
+
+        /// <summary>
+        /// Clones the specified instance using reflection
+        /// </summary>
+        /// <param name="instanceToClone">The instance to be cloned</param>
+        /// <returns>A field/property wise, non-recursive clone of the instance</returns>
+        public static object Clone(object instanceToClone)
+        {
+            var type = instanceToClone.GetType();
+            MethodInvoker func;
+            if (_cloneMethodsByType.TryGetValue(type, out func))
+            {
+                return func(null, instanceToClone);
+            }
+
+            // public static T GetClone<T>(this T source, CloningFlags flags)
+            var method = typeof (CloneFactory).GetMethods().FirstOrDefault(x => x.Name == "GetClone" && x.GetParameters().Length == 1);
+            method = method.MakeGenericMethod(type);
+            func = method.DelegateForCallMethod();
+            _cloneMethodsByType[type] = func;
+            return func(null, instanceToClone);
+        }
+
+        /// <summary>
+        /// Clones the specified instance and then casts it to T before returning
+        /// </summary>
+        public static T Clone<T>(T instanceToClone) where T : class
+        {
+            var clone = Clone((object)instanceToClone) as T;
+            if (clone == null)
+            {
+                throw new ArgumentException($"Unable to clone instance of type {instanceToClone.GetType().Name} to {typeof(T).Name}");
+            }
+            return clone;
+        }
+
+        /// <summary>
+        /// Adds method to return an instance of object
+        /// </summary>
+        /// <param name="key">The key of the method to add</param>
+        /// <param name="value">The value of the method to add</param>
+        public static void AddActivator(Type key, Func<object[], object> value)
+        {
+            if (!_activatorsByType.ContainsKey(key))
+            {
+                _activatorsByType.Add(key, value);
+            }
+            else
+            {
+                throw new ArgumentException($"ObjectActivator.AddActivator(): a method to return an instance of {key.Name} has already been added");
+            }
+        }
+
+        /// <summary>
+        /// Reset the object activators
+        /// </summary>
+        public static void ResetActivators()
+        {
+            _activatorsByType.Clear();
+        }
+    }
+}

Common/Util/OptionPayoff.cs

@@ -0,0 +1,78 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Runtime.CompilerServices;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Static class containing useful methods related with options payoff
+    /// </summary>
+    public static class OptionPayoff
+    {
+        /// <summary>
+        /// Intrinsic value function of the option
+        /// </summary>
+        /// <param name="underlyingPrice">The price of the underlying</param>
+        /// <param name="strike">The strike price of the option</param>
+        /// <param name="right">The option right of the option, call or put</param>
+        /// <returns>The intrinsic value remains for the option at expiry</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static decimal GetIntrinsicValue(decimal underlyingPrice, decimal strike, OptionRight right)
+        {
+            return Math.Max(0.0m, GetPayOff(underlyingPrice, strike, right));
+        }
+
+        /// <summary>
+        /// Intrinsic value function of the option
+        /// </summary>
+        /// <param name="underlyingPrice">The price of the underlying</param>
+        /// <param name="strike">The strike price of the option</param>
+        /// <param name="right">The option right of the option, call or put</param>
+        /// <returns>The intrinsic value remains for the option at expiry</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static double GetIntrinsicValue(double underlyingPrice, double strike, OptionRight right)
+        {
+            return Math.Max(0.0, GetPayOff(underlyingPrice, strike, right));
+        }
+
+        /// <summary>
+        /// Option payoff function at expiration time
+        /// </summary>
+        /// <param name="underlyingPrice">The price of the underlying</param>
+        /// <param name="strike">The strike price of the option</param>
+        /// <param name="right">The option right of the option, call or put</param>
+        /// <returns></returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static decimal GetPayOff(decimal underlyingPrice, decimal strike, OptionRight right)
+        {
+            return right == OptionRight.Call ? underlyingPrice - strike : strike - underlyingPrice;
+        }
+
+        /// <summary>
+        /// Option payoff function at expiration time
+        /// </summary>
+        /// <param name="underlyingPrice">The price of the underlying</param>
+        /// <param name="strike">The strike price of the option</param>
+        /// <param name="right">The option right of the option, call or put</param>
+        /// <returns></returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static double GetPayOff(double underlyingPrice, double strike, OptionRight right)
+        {
+            return right == OptionRight.Call ? underlyingPrice - strike : strike - underlyingPrice;
+        }
+    }
+}

Common/Util/PerformanceTimer.cs

@@ -0,0 +1,60 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Helper class to keep track of wall time, an efficient stop watch implementation
+    /// </summary>
+    public class PerformanceTimer
+    {
+        private static readonly double _frequency = Stopwatch.Frequency;
+        private long _start;
+        private long _currentTicks;
+        private double _totalSeconds;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Start()
+        {
+            _start = Stopwatch.GetTimestamp();
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Stop()
+        {
+            _currentTicks += Stopwatch.GetTimestamp() - _start;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public decimal GetAndReset()
+        {
+            var currentSeconds = _currentTicks / _frequency;
+            _currentTicks = 0;
+            _totalSeconds += currentSeconds;
+            return (decimal)Math.Round(currentSeconds, 2);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public decimal GetTotalTime()
+        {
+            return (decimal)Math.Round(_totalSeconds, 1);
+        }
+    }
+}

Common/Util/PerformanceTrackingTool.cs

@@ -0,0 +1,262 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Interfaces;
+using System.Runtime.CompilerServices;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Helper class to track algorithm performance
+    /// </summary>
+    public class PerformanceTrackingTool
+    {
+        private Series _onDataSeries;
+        private Series _dataSubscriptionSeries;
+        private Series _scheduleSeries;
+        private Series _selectionSeries;
+        private Series _sliceCreationSeries;
+        private Series _wallTimeSeries;
+        private Series _securityUpdatesSeries;
+        private Series _consolidatorsSeries;
+        private Series _transactionSeries;
+        private Series _splitsDividendsDelistingSeries;
+
+        private Series _activeSecuritiesCount;
+        private Series _consumedDataPointsCount;
+        private Series _consumedHistoryDataPointsCount;
+
+        private Series _cpuUsage;
+        private Series _managedRamUsage;
+        private Series _totalRamUsage;
+
+        private PerformanceTimer _onData;
+        private PerformanceTimer _dataSubscription;
+        private PerformanceTimer _schedule;
+        private PerformanceTimer _selection;
+        private PerformanceTimer _securityUpdates;
+        private PerformanceTimer _sliceCreation;
+        private PerformanceTimer _consolidators;
+        private PerformanceTimer _transactions;
+        private PerformanceTimer _splitsDividendsDelisting;
+        private bool _sampleEnabled;
+
+        private IAlgorithm _algorithm;
+        private long _previousDps;
+        private long _previousHistoryDps;
+        private DateTime _startWallTime;
+        private DateTime _nextSampleAlgoTime;
+        private DateTime _previousSampleAlgoTime;
+
+        private DateTime _previousSampleWallTime;
+
+        /// <summary>
+        /// Gets the number of data points processed per second
+        /// </summary>
+        public long DataPoints { get; private set; }
+
+        /// <summary>
+        /// Gets the number of data points of algorithm history provider
+        /// </summary>
+        public int HistoryDataPoints => _algorithm?.HistoryProvider?.DataPointCount ?? 0;
+
+        public void Initialize(IAlgorithm algorithm)
+        {
+            _algorithm = algorithm;
+            _sampleEnabled = algorithm.Settings.PerformanceSamplePeriod > TimeSpan.Zero;
+            if (_sampleEnabled)
+            {
+                _onData = new();
+                _dataSubscription = new();
+                _schedule = new();
+                _selection = new();
+                _sliceCreation = new();
+                _consolidators = new();
+                _securityUpdates = new();
+                _transactions = new();
+                _splitsDividendsDelisting = new();
+
+                var chart = new Chart("Performance");
+
+                _onDataSeries = new Series(PerformanceTarget.OnData.ToString(), unit: "Δ");
+                _dataSubscriptionSeries = new Series(PerformanceTarget.Subscriptions.ToString(), unit: "Δ");
+                _scheduleSeries = new Series(PerformanceTarget.Schedule.ToString(), unit: "Δ");
+                _selectionSeries = new Series(PerformanceTarget.Selection.ToString(), unit: "Δ");
+                _sliceCreationSeries = new Series(PerformanceTarget.Slice.ToString(), unit: "Δ");
+                _consolidatorsSeries = new Series(PerformanceTarget.Consolidators.ToString(), unit: "Δ");
+                _securityUpdatesSeries = new Series(PerformanceTarget.Securities.ToString(), unit: "Δ");
+                _transactionSeries = new Series(PerformanceTarget.Transactions.ToString(), unit: "Δ");
+                _splitsDividendsDelistingSeries = new Series(PerformanceTarget.SplitsDividendsDelisting.ToString(), unit: "Δ");
+                _wallTimeSeries = new Series("WallTime", unit: "Δ");
+                _activeSecuritiesCount = new Series("ActiveSecurities", unit: "#");
+                _consumedDataPointsCount = new Series("DataPoints", SeriesType.Bar, 1, unit: "#");
+                _consumedHistoryDataPointsCount = new Series("HistoryDataPoints", SeriesType.Bar, 1, unit: "#");
+                _cpuUsage = new Series("CPU", unit: "%");
+                _managedRamUsage = new Series("ManagedRAM", unit: string.Empty);
+                _totalRamUsage = new Series("TotalRAM", unit: string.Empty);
+
+                chart.AddSeries(_cpuUsage);
+                chart.AddSeries(_managedRamUsage);
+                chart.AddSeries(_totalRamUsage);
+                chart.AddSeries(_onDataSeries);
+                chart.AddSeries(_consolidatorsSeries);
+                chart.AddSeries(_dataSubscriptionSeries);
+                chart.AddSeries(_scheduleSeries);
+                chart.AddSeries(_wallTimeSeries);
+                chart.AddSeries(_securityUpdatesSeries);
+                chart.AddSeries(_selectionSeries);
+                chart.AddSeries(_sliceCreationSeries);
+                chart.AddSeries(_activeSecuritiesCount);
+                chart.AddSeries(_consumedDataPointsCount);
+                chart.AddSeries(_consumedHistoryDataPointsCount);
+                chart.AddSeries(_transactionSeries);
+                chart.AddSeries(_splitsDividendsDelistingSeries);
+                algorithm.AddChart(chart);
+
+                _previousSampleWallTime = _startWallTime = DateTime.UtcNow;
+                _previousSampleAlgoTime = algorithm.UtcTime;
+                _nextSampleAlgoTime = _previousSampleAlgoTime + _algorithm.Settings.PerformanceSamplePeriod;
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Sample(int dataPointCount, DateTime utcAlgoTime)
+        {
+            DataPoints += dataPointCount;
+            if (!_sampleEnabled)
+            {
+                return;
+            }
+
+            if (utcAlgoTime >= _nextSampleAlgoTime)
+            {
+                var nowUtc = DateTime.UtcNow;
+
+                // these share the same unit, real wall time
+                _onDataSeries.AddPoint(utcAlgoTime, _onData.GetAndReset());
+                _dataSubscriptionSeries.AddPoint(utcAlgoTime, _dataSubscription.GetAndReset());
+                _scheduleSeries.AddPoint(utcAlgoTime, _schedule.GetAndReset());
+                _selectionSeries.AddPoint(utcAlgoTime, _selection.GetAndReset());
+                _sliceCreationSeries.AddPoint(utcAlgoTime, _sliceCreation.GetAndReset());
+                _consolidatorsSeries.AddPoint(utcAlgoTime, _consolidators.GetAndReset());
+                _securityUpdatesSeries.AddPoint(utcAlgoTime, _securityUpdates.GetAndReset());
+                _transactionSeries.AddPoint(utcAlgoTime, _transactions.GetAndReset());
+                _splitsDividendsDelistingSeries.AddPoint(utcAlgoTime, _splitsDividendsDelisting.GetAndReset());
+                _wallTimeSeries.AddPoint(utcAlgoTime, (decimal)Math.Round((nowUtc - _previousSampleWallTime).TotalSeconds, 2));
+
+                _activeSecuritiesCount.AddPoint(utcAlgoTime, _algorithm.UniverseManager.ActiveSecurities.Count);
+                _consumedDataPointsCount.AddPoint(utcAlgoTime, DataPoints - _previousDps);
+                _consumedHistoryDataPointsCount.AddPoint(utcAlgoTime, HistoryDataPoints - _previousHistoryDps);
+
+                _cpuUsage.AddPoint(utcAlgoTime, (int)OS.CpuUsage);
+                _managedRamUsage.AddPoint(utcAlgoTime, OS.TotalPhysicalMemoryUsed);
+                _totalRamUsage.AddPoint(utcAlgoTime, OS.ApplicationMemoryUsed);
+
+                _previousHistoryDps = HistoryDataPoints;
+                _previousDps = DataPoints;
+                _previousSampleWallTime = nowUtc;
+                _nextSampleAlgoTime = utcAlgoTime + _algorithm.Settings.PerformanceSamplePeriod;
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Start(PerformanceTarget target)
+        {
+            if (!_sampleEnabled)
+            {
+                return;
+            }
+            Get(target).Start();
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Stop(PerformanceTarget target)
+        {
+            if (!_sampleEnabled)
+            {
+                return;
+            }
+            Get(target).Stop();
+        }
+
+        public void Shutdown()
+        {
+            if (!_sampleEnabled)
+            {
+                return;
+            }
+
+            var endTime = DateTime.UtcNow;
+            var message = $"Dps {DataPoints}. HistoryDps {HistoryDataPoints}." +
+                $" TotalRuntime: {(endTime - _startWallTime):hh\\:mm\\:ss}." +
+                $" OnData: {_onData.GetTotalTime()}s." +
+                $" DataSubscription: {_dataSubscription.GetTotalTime()}s." +
+                $" SliceCreation: {_sliceCreation.GetTotalTime()}s." +
+                $" Selection: {_selection.GetTotalTime()}s." +
+                $" Schedule: {_schedule.GetTotalTime()}s." +
+                $" Consolidators: {_consolidators.GetTotalTime()}s." +
+                $" Securities: {_securityUpdates.GetTotalTime()}s." +
+                $" Transactions: {_transactions.GetTotalTime()}s." +
+                $" SplitsDividendsDelisting: {_splitsDividendsDelisting.GetTotalTime()}s." +
+                $" ActiveSecurities: {_algorithm.UniverseManager.ActiveSecurities.Count}";
+
+            Logging.Log.Trace($"PerformanceTrackingTool.Summary(): {message}");
+            _algorithm.Debug(message);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private PerformanceTimer Get(PerformanceTarget target)
+        {
+            switch (target)
+            {
+                case PerformanceTarget.Subscriptions:
+                    return _dataSubscription;
+                case PerformanceTarget.Slice:
+                    return _sliceCreation;
+                case PerformanceTarget.Selection:
+                    return _selection;
+                case PerformanceTarget.Schedule:
+                    return _schedule;
+                case PerformanceTarget.OnData:
+                    return _onData;
+                case PerformanceTarget.Consolidators:
+                    return _consolidators;
+                case PerformanceTarget.Securities:
+                    return _securityUpdates;
+                case PerformanceTarget.Transactions:
+                    return _transactions;
+                case PerformanceTarget.SplitsDividendsDelisting:
+                    return _splitsDividendsDelisting;
+                default:
+                    throw new ArgumentException(nameof(target));
+            }
+        }
+    }
+
+    public enum PerformanceTarget
+    {
+        Selection,
+        Subscriptions,
+        Slice,
+        OnData,
+        Schedule,
+        Consolidators,
+        Securities,
+        Transactions,
+        SplitsDividendsDelisting,
+    }
+}

Common/Util/PythonUtil.cs

@@ -0,0 +1,386 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using Python.Runtime;
+using System.Collections.Generic;
+using QuantConnect.Data.Fundamental;
+using System.Text.RegularExpressions;
+using QuantConnect.Data.UniverseSelection;
+using System.Globalization;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Collection of utils for python objects processing
+    /// </summary>
+    public class PythonUtil
+    {
+        private static Regex LineRegex = new Regex("line (\\d+)", RegexOptions.Compiled);
+        private static Regex StackTraceFileLineRegex = new Regex("\"(.+)\", line (\\d+), in (.+)", RegexOptions.Compiled | RegexOptions.Singleline);
+        private static readonly Lazy<dynamic> lazyInspect = new Lazy<dynamic>(() => Py.Import("inspect"));
+
+        /// <summary>
+        /// The python exception stack trace line shift to use
+        /// </summary>
+        public static int ExceptionLineShift { get; set; } = 0;
+
+        /// <summary>
+        /// Encapsulates a python method with a <see cref="System.Action{T1}"/>
+        /// </summary>
+        /// <typeparam name="T1">The input type</typeparam>
+        /// <param name="pyObject">The python method</param>
+        /// <returns>A <see cref="System.Action{T1}"/> that encapsulates the python method</returns>
+        public static Action<T1> ToAction<T1>(PyObject pyObject)
+        {
+            using (Py.GIL())
+            {
+                long count = 0;
+                if (!TryGetArgLength(pyObject, out count) || count != 1)
+                {
+                    return null;
+                }
+                dynamic method = GetModule().GetAttr("to_action1");
+                return method(pyObject, typeof(T1)).AsManagedObject(typeof(Action<T1>));
+            }
+        }
+
+        /// <summary>
+        /// Encapsulates a python method with a <see cref="System.Action{T1, T2}"/>
+        /// </summary>
+        /// <typeparam name="T1">The first input type</typeparam>
+        /// <typeparam name="T2">The second input type type</typeparam>
+        /// <param name="pyObject">The python method</param>
+        /// <returns>A <see cref="System.Action{T1, T2}"/> that encapsulates the python method</returns>
+        public static Action<T1, T2> ToAction<T1, T2>(PyObject pyObject)
+        {
+            using (Py.GIL())
+            {
+                long count = 0;
+                if (!TryGetArgLength(pyObject, out count) || count != 2)
+                {
+                    return null;
+                }
+                dynamic method = GetModule().GetAttr("to_action2");
+                return method(pyObject, typeof(T1), typeof(T2)).AsManagedObject(typeof(Action<T1, T2>));
+            }
+        }
+
+        /// <summary>
+        /// Encapsulates a python method with a <see cref="System.Func{T1, T2}"/>
+        /// </summary>
+        /// <typeparam name="T1">The data type</typeparam>
+        /// <typeparam name="T2">The output type</typeparam>
+        /// <param name="pyObject">The python method</param>
+        /// <returns>A <see cref="System.Func{T1, T2}"/> that encapsulates the python method</returns>
+        public static Func<T1, T2> ToFunc<T1, T2>(PyObject pyObject)
+        {
+            using (Py.GIL())
+            {
+                long count = 0;
+                if (!TryGetArgLength(pyObject, out count) || count != 1)
+                {
+                    return null;
+                }
+                dynamic method = GetModule().GetAttr("to_func1");
+                return method(pyObject, typeof(T1), typeof(T2)).AsManagedObject(typeof(Func<T1, T2>));
+            }
+        }
+
+        /// <summary>
+        /// Encapsulates a python method with a <see cref="System.Func{T1, T2, T3}"/>
+        /// </summary>
+        /// <typeparam name="T1">The first argument's type</typeparam>
+        /// <typeparam name="T2">The first argument's type</typeparam>
+        /// <typeparam name="T3">The output type</typeparam>
+        /// <param name="pyObject">The python method</param>
+        /// <returns>A <see cref="System.Func{T1, T2, T3}"/> that encapsulates the python method</returns>
+        public static Func<T1, T2, T3> ToFunc<T1, T2, T3>(PyObject pyObject)
+        {
+            using (Py.GIL())
+            {
+                long count = 0;
+                if (!TryGetArgLength(pyObject, out count) || count != 2)
+                {
+                    return null;
+                }
+                dynamic method = GetModule().GetAttr("to_func2");
+                return method(pyObject, typeof(T1), typeof(T2), typeof(T3)).AsManagedObject(typeof(Func<T1, T2, T3>));
+            }
+        }
+
+        /// <summary>
+        /// Encapsulates a python method in coarse fundamental universe selector.
+        /// </summary>
+        /// <param name="pyObject">The python method</param>
+        /// <returns>A <see cref="Func{T, TResult}"/> (parameter is <see cref="IEnumerable{CoarseFundamental}"/>, return value is <see cref="IEnumerable{Symbol}"/>) that encapsulates the python method</returns>
+        public static Func<IEnumerable<CoarseFundamental>, IEnumerable<Symbol>> ToCoarseFundamentalSelector(PyObject pyObject)
+        {
+            var selector = ToFunc<IEnumerable<CoarseFundamental>, Symbol[]>(pyObject);
+            if (selector == null)
+            {
+                using (Py.GIL())
+                {
+                    throw new ArgumentException($"{pyObject.Repr()} is not a valid coarse fundamental universe selector method.");
+                }
+            }
+            return selector;
+        }
+
+        /// <summary>
+        /// Encapsulates a python method in fine fundamental universe selector.
+        /// </summary>
+        /// <param name="pyObject">The python method</param>
+        /// <returns>A <see cref="Func{T, TResult}"/> (parameter is <see cref="IEnumerable{FineFundamental}"/>, return value is <see cref="IEnumerable{Symbol}"/>) that encapsulates the python method</returns>
+        public static Func<IEnumerable<FineFundamental>, IEnumerable<Symbol>> ToFineFundamentalSelector(PyObject pyObject)
+        {
+            var selector = ToFunc<IEnumerable<FineFundamental>, Symbol[]>(pyObject);
+            if (selector == null)
+            {
+                using (Py.GIL())
+                {
+                    throw new ArgumentException($"{pyObject.Repr()} is not a valid fine fundamental universe selector method.");
+                }
+            }
+            return selector;
+        }
+
+        /// <summary>
+        /// Parsers <see cref="PythonException"/> into a readable message
+        /// </summary>
+        /// <param name="pythonException">The exception to parse</param>
+        /// <returns>String with relevant part of the stacktrace</returns>
+        public static string PythonExceptionParser(PythonException pythonException)
+        {
+            return PythonExceptionMessageParser(pythonException.Message) + PythonExceptionStackParser(pythonException.StackTrace);
+        }
+
+        /// <summary>
+        /// Parsers <see cref="Exception.Message"/> into a readable message
+        /// </summary>
+        /// <param name="message">The python exception message</param>
+        /// <returns>String with relevant part of the stacktrace</returns>
+        public static string PythonExceptionMessageParser(string message)
+        {
+            var match = LineRegex.Match(message);
+            if (match.Success)
+            {
+                foreach (Match lineCapture in match.Captures)
+                {
+                    var newLineNumber = int.Parse(lineCapture.Groups[1].Value) + ExceptionLineShift;
+                    message = Regex.Replace(message, lineCapture.ToString(), $"line {newLineNumber}");
+                }
+            }
+            else if (message.Contains(" value cannot be converted to ", StringComparison.InvariantCulture))
+            {
+                message += ": This error is often encountered when assigning to a member defined in the base QCAlgorithm class. For example, self.universe conflicts with 'QCAlgorithm.Universe' but can be fixed by prefixing private variables with an underscore, self._universe.";
+            }
+
+            return message;
+        }
+
+        /// <summary>
+        /// Parsers <see cref="PythonException.StackTrace"/> into a readable message
+        /// </summary>
+        /// <param name="value">String with the stacktrace information</param>
+        /// <returns>String with relevant part of the stacktrace</returns>
+        public static string PythonExceptionStackParser(string value)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                return string.Empty;
+            }
+
+            // The stack trace info before "at Python.Runtime." is the trace we want,
+            // which is for user Python code.
+            var endIndex = value.IndexOf("at Python.Runtime.", StringComparison.InvariantCulture);
+            var neededStackTrace = endIndex > 0 ? value.Substring(0, endIndex) : value;
+
+            // The stack trace is separated in blocks by file
+            var blocks = neededStackTrace.Split("  File ", StringSplitOptions.RemoveEmptyEntries)
+                .Select(fileTrace =>
+                {
+                    var trimedTrace = fileTrace.Trim();
+                    if (string.IsNullOrWhiteSpace(trimedTrace))
+                    {
+                        return string.Empty;
+                    }
+
+                    var match = StackTraceFileLineRegex.Match(trimedTrace);
+                    if (!match.Success)
+                    {
+                        return string.Empty;
+                    }
+
+                    var capture = match.Captures[0] as Match;
+
+                    var filePath = capture.Groups[1].Value;
+                    var lastFileSeparatorIndex = Math.Max(filePath.LastIndexOf('/'), filePath.LastIndexOf('\\'));
+                    if (lastFileSeparatorIndex < 0)
+                    {
+                        return string.Empty;
+                    }
+
+                    var fileName = filePath.Substring(lastFileSeparatorIndex + 1);
+                    var lineNumber = int.Parse(capture.Groups[2].Value, CultureInfo.InvariantCulture) + ExceptionLineShift;
+                    var locationAndInfo = capture.Groups[3].Value.Trim();
+
+                    return $"  at {locationAndInfo}{Environment.NewLine} in {fileName}: line {lineNumber}";
+                })
+                .Where(x => !string.IsNullOrWhiteSpace(x));
+
+            var result = string.Join(Environment.NewLine, blocks);
+            result = Logging.Log.ClearLeanPaths(result);
+
+            return string.IsNullOrWhiteSpace(result)
+                ? string.Empty
+                : $"{Environment.NewLine}{result}{Environment.NewLine}";
+        }
+
+        /// <summary>
+        /// Try to get the length of arguments of a method
+        /// </summary>
+        /// <param name="pyObject">Object representing a method</param>
+        /// <param name="length">Lenght of arguments</param>
+        /// <returns>True if pyObject is a method</returns>
+        private static bool TryGetArgLength(PyObject pyObject, out long length)
+        {
+            using (Py.GIL())
+            {
+                var inspect = lazyInspect.Value;
+                if (inspect.isfunction(pyObject))
+                {
+                    var args = inspect.getfullargspec(pyObject).args as PyObject;
+                    var pyList = new PyList(args);
+                    length = pyList.Length();
+                    pyList.Dispose();
+                    args.Dispose();
+                    return true;
+                }
+
+                if (inspect.ismethod(pyObject))
+                {
+                    var args = inspect.getfullargspec(pyObject).args as PyObject;
+                    var pyList = new PyList(args);
+                    length = pyList.Length() - 1;
+                    pyList.Dispose();
+                    args.Dispose();
+                    return true;
+                }
+            }
+            length = 0;
+            return false;
+        }
+
+        /// <summary>
+        /// Creates a python module with utils methods
+        /// </summary>
+        /// <returns>PyObject with a python module</returns>
+        private static PyObject GetModule()
+        {
+            return PyModule.FromString("x",
+                "from clr import AddReference\n" +
+                "AddReference(\"System\")\n" +
+                "from System import Action, Func\n" +
+                "def to_action1(pyobject, t1):\n" +
+                "    return Action[t1](pyobject)\n" +
+                "def to_action2(pyobject, t1, t2):\n" +
+                "    return Action[t1, t2](pyobject)\n" +
+                "def to_func1(pyobject, t1, t2):\n" +
+                "    return Func[t1, t2](pyobject)\n" +
+                "def to_func2(pyobject, t1, t2, t3):\n" +
+                "    return Func[t1, t2, t3](pyobject)");
+        }
+
+        /// <summary>
+        /// Convert Python input to a list of Symbols
+        /// </summary>
+        /// <param name="input">Object with the desired property</param>
+        /// <returns>List of Symbols</returns>
+        public static IEnumerable<Symbol> ConvertToSymbols(PyObject input)
+        {
+            List<Symbol> symbolsList;
+            Symbol symbol;
+
+            using (Py.GIL())
+            {
+                // Handle the possible types of conversions
+                if (PyList.IsListType(input))
+                {
+                    List<string> symbolsStringList;
+
+                    //Check if an entry in the list is a string type, if so then try and convert the whole list
+                    if (PyString.IsStringType(input[0]) && input.TryConvert(out symbolsStringList))
+                    {
+                        symbolsList = new List<Symbol>();
+                        foreach (var stringSymbol in symbolsStringList)
+                        {
+                            symbol = QuantConnect.Symbol.Create(stringSymbol, SecurityType.Equity, Market.USA);
+                            symbolsList.Add(symbol);
+                        }
+                    }
+                    //Try converting it to list of symbols, if it fails throw exception
+                    else if (!input.TryConvert(out symbolsList))
+                    {
+                        throw new ArgumentException($"Cannot convert list {input.Repr()} to symbols");
+                    }
+                }
+                else
+                {
+                    //Check if its a single string, and try and convert it
+                    string symbolString;
+                    if (PyString.IsStringType(input) && input.TryConvert(out symbolString))
+                    {
+                        symbol = QuantConnect.Symbol.Create(symbolString, SecurityType.Equity, Market.USA);
+                        symbolsList = new List<Symbol> { symbol };
+                    }
+                    else if (input.TryConvert(out symbol))
+                    {
+                        symbolsList = new List<Symbol> { symbol };
+                    }
+                    else
+                    {
+                        throw new ArgumentException($"Cannot convert object {input.Repr()} to symbol");
+                    }
+                }
+            }
+            return symbolsList;
+        }
+
+        /// <summary>
+        /// Attempts to convert a PyObject into a pure C# instance of <typeparamref name="T"/>.
+        /// If conversion fails, a wrapper instance is created/>.
+        /// </summary>
+        /// <typeparam name="T">The C# type expected, which may be an interface or a concrete class</typeparam>
+        /// <param name="pyObject">The Python object to convert.</param>
+        /// <param name="createWrapper">Factory function used to create a wrapper around the Python object</param>
+        /// <returns>
+        /// A pure C# instance if conversion is possible, otherwise a wrapper instance.
+        /// </returns>
+        public static T CreateInstanceOrWrapper<T>(PyObject pyObject, Func<PyObject, T> createWrapper)
+        {
+            if (pyObject.TryConvert<T>(out var instance))
+            {
+                // Successfully converted to pure C#
+                return instance;
+            }
+
+            // Fallback to wrapper
+            return createWrapper(pyObject);
+        }
+    }
+}

Common/Util/RateGate.cs

@@ -0,0 +1,246 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Threading;
+using System.Collections.Generic;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Used to control the rate of some occurrence per unit of time.
+    /// </summary>
+    /// <see href="http://www.jackleitch.net/2010/10/better-rate-limiting-with-dot-net/"/>
+    /// <remarks>
+    ///     <para>
+    ///     To control the rate of an action using a <see cref="RateGate"/>,
+    ///     code should simply call <see cref="WaitToProceed()"/> prior to
+    ///     performing the action. <see cref="WaitToProceed()"/> will block
+    ///     the current thread until the action is allowed based on the rate
+    ///     limit.
+    ///     </para>
+    ///     <para>
+    ///     This class is thread safe. A single <see cref="RateGate"/> instance
+    ///     may be used to control the rate of an occurrence across multiple
+    ///     threads.
+    ///     </para>
+    /// </remarks>
+    public class RateGate : IDisposable
+    {
+        // Semaphore used to count and limit the number of occurrences per
+        // unit time.
+        private readonly SemaphoreSlim _semaphore;
+
+        // Times (in millisecond ticks) at which the semaphore should be exited.
+        private readonly Queue<int> _exitTimes;
+
+        // Timer used to trigger exiting the semaphore.
+        private readonly Timer _exitTimer;
+
+        // Whether this instance is disposed.
+        private bool _isDisposed;
+
+        /// <summary>
+        /// Number of occurrences allowed per unit of time.
+        /// </summary>
+        public int Occurrences
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// The length of the time unit, in milliseconds.
+        /// </summary>
+        public int TimeUnitMilliseconds
+        {
+            get; private set;
+        }
+
+        /// <summary>
+        /// Flag indicating we are currently being rate limited
+        /// </summary>
+        public bool IsRateLimited
+        {
+            get { return !WaitToProceed(0); }
+        }
+
+        /// <summary>
+        /// Initializes a <see cref="RateGate"/> with a rate of <paramref name="occurrences"/>
+        /// per <paramref name="timeUnit"/>.
+        /// </summary>
+        /// <param name="occurrences">Number of occurrences allowed per unit of time.</param>
+        /// <param name="timeUnit">Length of the time unit.</param>
+        /// <exception cref="ArgumentOutOfRangeException">
+        /// If <paramref name="occurrences"/> or <paramref name="timeUnit"/> is negative.
+        /// </exception>
+        public RateGate(int occurrences, TimeSpan timeUnit)
+        {
+            // Check the arguments.
+            if (occurrences <= 0)
+                throw new ArgumentOutOfRangeException(nameof(occurrences), "Number of occurrences must be a positive integer");
+            if (timeUnit != timeUnit.Duration())
+                throw new ArgumentOutOfRangeException(nameof(timeUnit), "Time unit must be a positive span of time");
+            if (timeUnit >= TimeSpan.FromMilliseconds(UInt32.MaxValue))
+                throw new ArgumentOutOfRangeException(nameof(timeUnit), "Time unit must be less than 2^32 milliseconds");
+
+            Occurrences = occurrences;
+            TimeUnitMilliseconds = (int)timeUnit.TotalMilliseconds;
+
+            // Create the semaphore, with the number of occurrences as the maximum count.
+            _semaphore = new SemaphoreSlim(Occurrences, Occurrences);
+
+            // Create a queue to hold the semaphore exit times.
+            _exitTimes = new ();
+
+            // Create a timer to exit the semaphore. Use the time unit as the original
+            // interval length because that's the earliest we will need to exit the semaphore.
+            _exitTimer = new Timer(ExitTimerCallback, null, Timeout.Infinite, Timeout.Infinite);
+        }
+
+        // Callback for the exit timer that exits the semaphore based on exit times
+        // in the queue and then sets the timer for the nextexit time.
+        // Credit to Jim: http://www.jackleitch.net/2010/10/better-rate-limiting-with-dot-net/#comment-3620
+        // for providing the code below, fixing issue #3499 - https://github.com/QuantConnect/Lean/issues/3499
+        private void ExitTimerCallback(object state)
+        {
+            try
+            {
+                // While there are exit times that are passed due still in the queue,
+                // exit the semaphore and dequeue the exit time.
+                var exitTime = 0;
+                var exitTimeValid = false;
+                var tickCount = Environment.TickCount;
+                lock (_exitTimes)
+                {
+                    exitTimeValid = _exitTimes.TryPeek(out exitTime);
+                    while (exitTimeValid)
+                    {
+                        if (unchecked(exitTime - tickCount) > 0)
+                        {
+                            break;
+                        }
+                        _semaphore.Release();
+                        _exitTimes.Dequeue();
+                        exitTimeValid = _exitTimes.TryPeek(out exitTime);
+                    }
+
+                    // only schedule if there's someone waiting
+                    if (exitTimeValid)
+                    {
+                        // we are already holding the next item from the queue, do not peek again
+                        // although this exit time may have already pass by this stmt.
+                        var timeUntilNextCheck = Math.Min(TimeUnitMilliseconds, Math.Max(0, exitTime - tickCount));
+                        _exitTimer.Change(timeUntilNextCheck, Timeout.Infinite);
+                    }
+                }
+            }
+            catch (Exception)
+            {
+                // can throw if called when disposing
+            }
+        }
+
+        /// <summary>
+        /// Blocks the current thread until allowed to proceed or until the
+        /// specified timeout elapses.
+        /// </summary>
+        /// <param name="millisecondsTimeout">Number of milliseconds to wait, or -1 to wait indefinitely.</param>
+        /// <returns>true if the thread is allowed to proceed, or false if timed out</returns>
+        public bool WaitToProceed(int millisecondsTimeout)
+        {
+            // Check the arguments.
+            if (millisecondsTimeout < -1)
+                throw new ArgumentOutOfRangeException(nameof(millisecondsTimeout));
+
+            CheckDisposed();
+
+            // Block until we can enter the semaphore or until the timeout expires.
+            var entered = _semaphore.Wait(millisecondsTimeout);
+
+            // If we entered the semaphore, compute the corresponding exit time
+            // and add it to the queue.
+            if (entered)
+            {
+                var timeToExit = unchecked(Environment.TickCount + TimeUnitMilliseconds);
+                lock(_exitTimes)
+                {
+                    if (_exitTimes.Count == 0)
+                    {
+                        // schedule, there was no one so it's not scheduled
+                        _exitTimer.Change(TimeUnitMilliseconds, Timeout.Infinite);
+                    }
+                    _exitTimes.Enqueue(timeToExit);
+                }
+            }
+
+            return entered;
+        }
+
+        /// <summary>
+        /// Blocks the current thread until allowed to proceed or until the
+        /// specified timeout elapses.
+        /// </summary>
+        /// <param name="timeout"></param>
+        /// <returns>true if the thread is allowed to proceed, or false if timed out</returns>
+        public bool WaitToProceed(TimeSpan timeout)
+        {
+            return WaitToProceed((int)timeout.TotalMilliseconds);
+        }
+
+        /// <summary>
+        /// Blocks the current thread indefinitely until allowed to proceed.
+        /// </summary>
+        public void WaitToProceed()
+        {
+            WaitToProceed(Timeout.Infinite);
+        }
+
+        // Throws an ObjectDisposedException if this object is disposed.
+        private void CheckDisposed()
+        {
+            if (_isDisposed)
+                throw new ObjectDisposedException("RateGate is already disposed");
+        }
+
+        /// <summary>
+        /// Releases unmanaged resources held by an instance of this class.
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Releases unmanaged resources held by an instance of this class.
+        /// </summary>
+        /// <param name="isDisposing">Whether this object is being disposed.</param>
+        protected virtual void Dispose(bool isDisposing)
+        {
+            if (!_isDisposed)
+            {
+                if (isDisposing)
+                {
+                    // The semaphore and timer both implement IDisposable and
+                    // therefore must be disposed.
+                    _semaphore.Dispose();
+                    _exitTimer.Dispose();
+
+                    _isDisposed = true;
+                }
+            }
+        }
+    }
+}

Common/Util/RateLimit/BusyWaitSleepStrategy.cs

@@ -0,0 +1,36 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Threading;
+
+namespace QuantConnect.Util.RateLimit
+{
+    /// <summary>
+    /// Provides a CPU intensive means of waiting for more tokens to be available in <see cref="ITokenBucket"/>.
+    /// This strategy is only viable when the requested number of tokens is expected to become available in an
+    /// extremely short period of time. This implementation aims to keep the current thread executing to prevent
+    /// potential content switches arising from a thread yielding or sleeping strategy.
+    /// </summary>
+    public class BusyWaitSleepStrategy : ISleepStrategy
+    {
+        /// <summary>
+        /// Provides a CPU intensive sleep by executing <see cref="Thread.SpinWait"/> for a single spin.
+        /// </summary>
+        public void Sleep()
+        {
+            Thread.SpinWait(1);
+        }
+    }
+}

Common/Util/RateLimit/FixedIntervalRefillStrategy.cs

@@ -0,0 +1,79 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+
+namespace QuantConnect.Util.RateLimit
+{
+    /// <summary>
+    /// Provides a refill strategy that has a constant, quantized refill rate.
+    /// For example, after 1 minute passes add 5 units. If 59 seconds has passed, it will add zero unit,
+    /// but if 2 minutes have passed, then 10 units would be added.
+    /// </summary>
+    public class FixedIntervalRefillStrategy : IRefillStrategy
+    {
+        private readonly object _sync = new object();
+
+        private long _nextRefillTimeTicks;
+
+        private readonly long _refillAmount;
+        private readonly long _refillIntervalTicks;
+        private readonly ITimeProvider _timeProvider;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FixedIntervalRefillStrategy"/> class.
+        /// </summary>
+        /// <param name="timeProvider">Provides the current time used for determining how much time has elapsed
+        /// between invocations of the refill method</param>
+        /// <param name="refillAmount">Defines the constant number of tokens to be made available for consumption
+        /// each time the provided <paramref name="refillInterval"/> has passed</param>
+        /// <param name="refillInterval">The amount of time that must pass before adding the specified <paramref name="refillAmount"/>
+        /// back to the bucket</param>
+        public FixedIntervalRefillStrategy(ITimeProvider timeProvider, long refillAmount, TimeSpan refillInterval)
+        {
+            _timeProvider = timeProvider;
+            _refillAmount = refillAmount;
+            _refillIntervalTicks = refillInterval.Ticks;
+            _nextRefillTimeTicks = _timeProvider.GetUtcNow().Ticks + _refillIntervalTicks;
+        }
+
+        /// <summary>
+        /// Computes the number of new tokens made available to the bucket for consumption by determining the
+        /// number of time intervals that have passed and multiplying by the number of tokens to refill for
+        /// each time interval.
+        /// </summary>
+        public long Refill()
+        {
+            lock (_sync)
+            {
+                var currentTimeTicks = _timeProvider.GetUtcNow().Ticks;
+                if (currentTimeTicks < _nextRefillTimeTicks)
+                {
+                    return 0L;
+                }
+
+                // determine number of time increments that have passed
+                var deltaTimeTicks = currentTimeTicks - _nextRefillTimeTicks;
+                var intervalsElapsed = 1 + Math.Max(deltaTimeTicks / _refillIntervalTicks, 0);
+
+                // update next refill time as quantized via the number of passed intervals
+                _nextRefillTimeTicks += _refillIntervalTicks * intervalsElapsed;
+
+                // refill by the tokens per interval times the number of intervals elapsed
+                return _refillAmount * intervalsElapsed;
+            }
+        }
+    }
+}

Common/Util/RateLimit/IRefillStrategy.cs

@@ -0,0 +1,28 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+namespace QuantConnect.Util.RateLimit
+{
+    /// <summary>
+    /// Provides a strategy for making tokens available for consumption in the <see cref="ITokenBucket"/>
+    /// </summary>
+    public interface IRefillStrategy
+    {
+        /// <summary>
+        /// Computes the number of new tokens made available, typically via the passing of time.
+        /// </summary>
+        long Refill();
+    }
+}

Common/Util/RateLimit/ISleepStrategy.cs

@@ -0,0 +1,30 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+namespace QuantConnect.Util.RateLimit
+{
+    /// <summary>
+    /// Defines a strategy for sleeping the current thread of execution. This is currently used via the
+    /// <see cref="ITokenBucket.Consume"/> in order to wait for new tokens to become available for consumption.
+    /// </summary>
+    public interface ISleepStrategy
+    {
+        /// <summary>
+        /// Sleeps the current thread in an implementation specific way
+        /// and for an implementation specific amount of time
+        /// </summary>
+        void Sleep();
+    }
+}

Common/Util/RateLimit/ITokenBucket.cs

@@ -0,0 +1,59 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Threading;
+
+namespace QuantConnect.Util.RateLimit
+{
+    /// <summary>
+    /// Defines a token bucket for rate limiting
+    /// See: https://en.wikipedia.org/wiki/Token_bucket
+    /// </summary>
+    /// <remarks>
+    /// This code is ported from https://github.com/mxplusb/TokenBucket - since it's a dotnet core
+    /// project, there were issued importing the nuget package directly. The referenced repository
+    /// is provided under the Apache V2 license.
+    /// </remarks>
+    public interface ITokenBucket
+    {
+        /// <summary>
+        /// Gets the maximum capacity of tokens this bucket can hold.
+        /// </summary>
+        long Capacity { get; }
+
+        /// <summary>
+        /// Gets the total number of currently available tokens for consumption
+        /// </summary>
+        long AvailableTokens { get; }
+
+        /// <summary>
+        /// Blocks until the specified number of tokens are available for consumption
+        /// and then consumes that number of tokens.
+        /// </summary>
+        /// <param name="tokens">The number of tokens to consume</param>
+        /// <param name="timeout">The maximum amount of time, in milliseconds, to block. A <see cref="TimeoutException"/>
+        /// is throw in the event it takes longer than the stated timeout to consume the requested number of tokens.
+        /// The default timeout is set to infinite, which will block forever.</param>
+        void Consume(long tokens, long timeout = Timeout.Infinite);
+
+        /// <summary>
+        /// Attempts to consume the specified number of tokens from the bucket. If the
+        /// requested number of tokens are not immediately available, then this method
+        /// will return false to indicate that zero tokens have been consumed.
+        /// </summary>
+        bool TryConsume(long tokens);
+    }
+}

Common/Util/RateLimit/LeakyBucket.cs

@@ -0,0 +1,172 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Threading;
+
+namespace QuantConnect.Util.RateLimit
+{
+    /// <summary>
+    /// Provides an implementation of <see cref="ITokenBucket"/> that implements the leaky bucket algorithm
+    /// See: https://en.wikipedia.org/wiki/Leaky_bucket
+    /// </summary>
+    public class LeakyBucket : ITokenBucket
+    {
+        private readonly object _sync = new object();
+
+        private long _available;
+        private readonly ISleepStrategy _sleep;
+        private readonly IRefillStrategy _refill;
+        private readonly ITimeProvider _timeProvider;
+
+        /// <summary>
+        /// Gets the maximum capacity of tokens this bucket can hold.
+        /// </summary>
+        public long Capacity { get; }
+
+        /// <summary>
+        /// Gets the total number of currently available tokens for consumption
+        /// </summary>
+        public long AvailableTokens
+        {
+            // synchronized read w/ the modification of available tokens in TryConsume
+            get { lock (_sync) return _available; }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LeakyBucket"/> class.
+        /// This constructor initializes the bucket using the <see cref="ThreadSleepStrategy.Sleep"/> with a 1 millisecond
+        /// sleep to prevent being CPU intensive and uses the <see cref="FixedIntervalRefillStrategy"/> to refill bucket
+        /// tokens according to the <paramref name="refillAmount"/> and <paramref name="refillInterval"/> parameters.
+        /// </summary>
+        /// <param name="capacity">The maximum number of tokens this bucket can hold</param>
+        /// <param name="refillAmount">The number of tokens to add to the bucket each <paramref name="refillInterval"/></param>
+        /// <param name="refillInterval">The interval which after passing more tokens are added to the bucket</param>
+        public LeakyBucket(long capacity, long refillAmount, TimeSpan refillInterval)
+            : this(capacity, ThreadSleepStrategy.Sleeping(1),
+                new FixedIntervalRefillStrategy(RealTimeProvider.Instance, refillAmount, refillInterval)
+            )
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="LeakyBucket"/> class
+        /// </summary>
+        /// <param name="capacity">The maximum number of tokens this bucket can hold</param>
+        /// <param name="sleep">Defines the <see cref="ISleepStrategy"/> used when <see cref="Consume"/> is invoked
+        /// but the bucket does not have enough tokens yet</param>
+        /// <param name="refill">Defines the <see cref="IRefillStrategy"/> that computes how many tokens to add
+        /// back to the bucket each time consumption is attempted</param>
+        /// <param name="timeProvider">Defines the <see cref="ITimeProvider"/> used to enforce timeouts when
+        /// invoking <see cref="Consume"/></param>
+        public LeakyBucket(long capacity, ISleepStrategy sleep, IRefillStrategy refill, ITimeProvider timeProvider = null)
+        {
+            _sleep = sleep;
+            _refill = refill;
+            Capacity = capacity;
+            _available = capacity;
+            _timeProvider = timeProvider ?? RealTimeProvider.Instance;
+        }
+
+        /// <summary>
+        /// Blocks until the specified number of tokens are available for consumption
+        /// and then consumes that number of tokens.
+        /// </summary>
+        /// <param name="tokens">The number of tokens to consume</param>
+        /// <param name="timeout">The maximum amount of time, in milliseconds, to block. An exception is
+        /// throw in the event it takes longer than the stated timeout to consume the requested number
+        /// of tokens</param>
+        public void Consume(long tokens, long timeout = Timeout.Infinite)
+        {
+            if (timeout < Timeout.Infinite)
+            {
+                throw new ArgumentOutOfRangeException(nameof(timeout),
+                    "Invalid timeout. Use -1 for no timeout, 0 for immediate timeout and a positive number " +
+                    "of milliseconds to indicate a timeout. All other values are out of range."
+                );
+            }
+
+            var startTime = _timeProvider.GetUtcNow();
+
+            while (true)
+            {
+                if (TryConsume(tokens))
+                {
+                    break;
+                }
+
+                if (timeout != Timeout.Infinite)
+                {
+                    // determine if the requested timeout has elapsed
+                    var currentTime = _timeProvider.GetUtcNow();
+                    var elapsedMilliseconds = (currentTime - startTime).TotalMilliseconds;
+                    if (elapsedMilliseconds > timeout)
+                    {
+                        throw new TimeoutException("The operation timed out while waiting for the rate limit to be lifted.");
+                    }
+                }
+
+                _sleep.Sleep();
+            }
+        }
+
+        /// <summary>
+        /// Attempts to consume the specified number of tokens from the bucket. If the
+        /// requested number of tokens are not immediately available, then this method
+        /// will return false to indicate that zero tokens have been consumed.
+        /// </summary>
+        public bool TryConsume(long tokens)
+        {
+            if (tokens <= 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(tokens),
+                    "Number of tokens to consume must be positive"
+                );
+            }
+
+            if (tokens > Capacity)
+            {
+                throw new ArgumentOutOfRangeException(nameof(tokens),
+                    "Number of tokens to consume must be less than or equal to the capacity"
+                );
+            }
+
+            lock (_sync)
+            {
+                // determine how many units have become available since last invocation
+                var refilled = Math.Max(0, _refill.Refill());
+
+                // the number of tokens to add, the max of which is the difference between capacity and currently available
+                var deltaTokens = Math.Min(Capacity - _available, refilled);
+
+                // update the available number of units with the new tokens
+                _available += deltaTokens;
+
+                if (tokens > _available)
+                {
+                    // we don't have enough tokens yet
+                    Logging.Log.Trace($"LeakyBucket.TryConsume({tokens}): Failed to consumed tokens. Available: {_available}");
+                    return false;
+                }
+
+                // subtract the number of tokens consumed
+                _available = _available - tokens;
+                Logging.Log.Trace($"LeakyBucket.TryConsume({tokens}): Successfully consumed tokens. Available: {_available}");
+                return true;
+            }
+        }
+
+    }
+}

Common/Util/RateLimit/ThreadSleepStrategy.cs

@@ -0,0 +1,60 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Threading;
+
+namespace QuantConnect.Util.RateLimit
+{
+    /// <summary>
+    /// Provides a CPU non-intensive means of waiting for more tokens to be available in <see cref="ITokenBucket"/>.
+    /// This strategy should be the most commonly used as it either sleeps or yields the currently executing thread,
+    /// allowing for other threads to execute while the current thread is blocked and waiting for new tokens to become
+    /// available in the bucket for consumption.
+    /// </summary>
+    public class ThreadSleepStrategy : ISleepStrategy
+    {
+        /// <summary>
+        /// Gets an instance of <see cref="ISleepStrategy"/> that yields the current thread
+        /// </summary>
+        public static readonly ISleepStrategy Yielding = new ThreadSleepStrategy(0);
+
+        /// <summary>
+        /// Gets an instance of <see cref="ISleepStrategy"/> that sleeps the current thread for
+        /// the specified number of milliseconds
+        /// </summary>
+        /// <param name="milliseconds">The duration of time to sleep, in milliseconds</param>
+        public static ISleepStrategy Sleeping(int milliseconds) => new ThreadSleepStrategy(milliseconds);
+
+        private readonly int _milliseconds;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ThreadSleepStrategy"/> using the specified
+        /// number of <paramref name="milliseconds"/> for each <see cref="Sleep"/> invocation.
+        /// </summary>
+        /// <param name="milliseconds">The duration of time to sleep, in milliseconds</param>
+        public ThreadSleepStrategy(int milliseconds)
+        {
+            _milliseconds = milliseconds;
+        }
+
+        /// <summary>
+        /// Sleeps the current thread using the initialized number of milliseconds
+        /// </summary>
+        public void Sleep()
+        {
+            Thread.Sleep(_milliseconds);
+        }
+    }
+}

Common/Util/RateLimit/TokenBucket.cs

@@ -0,0 +1,51 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Threading;
+
+namespace QuantConnect.Util.RateLimit
+{
+    /// <summary>
+    /// Provides extension methods for interacting with <see cref="ITokenBucket"/> instances as well
+    /// as access to the <see cref="NullTokenBucket"/> via <see cref="TokenBucket.Null"/>
+    /// </summary>
+    public static class TokenBucket
+    {
+        /// <summary>
+        /// Gets an <see cref="ITokenBucket"/> that always permits consumption
+        /// </summary>
+        public static ITokenBucket Null = new NullTokenBucket();
+
+        /// <summary>
+        /// Provides an overload of <see cref="ITokenBucket.Consume"/> that accepts a <see cref="TimeSpan"/> timeout
+        /// </summary>
+        public static void Consume(this ITokenBucket bucket, long tokens, TimeSpan timeout)
+        {
+            bucket.Consume(tokens, (long) timeout.TotalMilliseconds);
+        }
+
+        /// <summary>
+        /// Provides an implementation of <see cref="ITokenBucket"/> that does not enforce rate limiting
+        /// </summary>
+        private class NullTokenBucket : ITokenBucket
+        {
+            public long Capacity => long.MaxValue;
+            public long AvailableTokens => long.MaxValue;
+            public bool TryConsume(long tokens) { return true; }
+            public void Consume(long tokens, long timeout = Timeout.Infinite) { }
+        }
+    }
+}

Common/Util/ReadOnlyExtendedDictionary.cs

@@ -0,0 +1,126 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using QuantConnect.Python;
+
+namespace Common.Util
+{
+    /// <summary>
+    /// Provides a read-only implementation of ExtendedDictionary
+    /// </summary>
+    [PandasNonExpandable]
+    public class ReadOnlyExtendedDictionary<TKey, TValue> : BaseExtendedDictionary<TKey, TValue>, IReadOnlyDictionary<TKey, TValue>
+    {
+        /// <summary>
+        /// Initializes a new instance of the ReadOnlyExtendedDictionary class that is empty
+        /// </summary>
+        public ReadOnlyExtendedDictionary()
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the ReadOnlyExtendedDictionary class that contains elements copied from the specified dictionary
+        /// </summary>
+        /// <param name="dictionary">The dictionary whose elements are copied to the new dictionary</param>
+        public ReadOnlyExtendedDictionary(IDictionary<TKey, TValue> dictionary) : base(dictionary)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the ReadOnlyExtendedDictionary class
+        /// using the specified <paramref name="data"/> as a data source
+        /// </summary>
+        /// <param name="data">The data source for this dictionary</param>
+        /// <param name="keySelector">Delegate used to select a key from the value</param>
+        public ReadOnlyExtendedDictionary(IEnumerable<TValue> data, Func<TValue, TKey> keySelector) : base(data, keySelector)
+        {
+        }
+
+        /// <summary>
+        /// Gets a value indicating whether the dictionary is read-only
+        /// </summary>
+        public override bool IsReadOnly => true;
+
+        /// <summary>
+        /// Gets an enumerable collection containing the keys of the read-only dictionary
+        /// </summary>
+        IEnumerable<TKey> IReadOnlyDictionary<TKey, TValue>.Keys => base.Keys;
+
+        /// <summary>
+        /// Gets an enumerable collection containing the values of the read-only dictionary
+        /// </summary>
+        IEnumerable<TValue> IReadOnlyDictionary<TKey, TValue>.Values => base.Values;
+
+        /// <summary>
+        /// Gets or sets the value associated with the specified key
+        /// </summary>
+        /// <param name="key">The key of the value to get or set</param>
+        /// <returns>The value associated with the specified key</returns>
+        public override TValue this[TKey key]
+        {
+            get => base[key];
+            set => throw new InvalidOperationException("Dictionary is read-only");
+        }
+
+        /// <summary>
+        /// Removes all items from the dictionary
+        /// </summary>
+        public override void Clear()
+        {
+            throw new InvalidOperationException("Dictionary is read-only");
+        }
+
+        /// <summary>
+        /// Removes the value with the specified key
+        /// </summary>
+        /// <param name="key">The key of the element to remove</param>
+        /// <returns>true if the element was successfully found and removed; otherwise, false</returns>
+        public override bool Remove(TKey key)
+        {
+            throw new InvalidOperationException("Dictionary is read-only");
+        }
+
+        /// <summary>
+        /// Adds an element with the provided key and value to the dictionary
+        /// </summary>
+        /// <param name="key">The key of the element to add</param>
+        /// <param name="value">The value of the element to add</param>
+        public new void Add(TKey key, TValue value)
+        {
+            throw new InvalidOperationException("Dictionary is read-only");
+        }
+
+        /// <summary>
+        /// Adds an element with the provided key-value pair to the dictionary
+        /// </summary>
+        /// <param name="item">The key-value pair to add</param>
+        public new void Add(KeyValuePair<TKey, TValue> item)
+        {
+            throw new InvalidOperationException("Dictionary is read-only");
+        }
+
+        /// <summary>
+        /// Removes the first occurrence of a specific object from the dictionary
+        /// </summary>
+        /// <param name="item">The key-value pair to remove</param>
+        /// <returns>true if the key-value pair was successfully removed; otherwise, false</returns>
+        public new bool Remove(KeyValuePair<TKey, TValue> item)
+        {
+            throw new InvalidOperationException("Dictionary is read-only");
+        }
+    }
+}

Common/Util/ReaderWriterLockSlimExtensions.cs

@@ -0,0 +1,107 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Threading;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides extension methods to make working with the <see cref="ReaderWriterLockSlim"/> class easier
+    /// </summary>
+    public static class ReaderWriterLockSlimExtensions
+    {
+        /// <summary>
+        /// Opens the read lock
+        /// </summary>
+        /// <param name="readerWriterLockSlim">The lock to open for read</param>
+        /// <returns>A disposable reference which will release the lock upon disposal</returns>
+        public static IDisposable Read(this ReaderWriterLockSlim readerWriterLockSlim)
+        {
+            return new ReaderLockToken(readerWriterLockSlim);
+        }
+
+        /// <summary>
+        /// Opens the write lock
+        /// </summary>
+        /// <param name="readerWriterLockSlim">The lock to open for write</param>
+        /// <returns>A disposale reference which will release thelock upon disposal</returns>
+        public static IDisposable Write(this ReaderWriterLockSlim readerWriterLockSlim)
+        {
+            return new WriteLockToken(readerWriterLockSlim);
+        }
+
+        private sealed class ReaderLockToken : ReaderWriterLockSlimToken
+        {
+            public ReaderLockToken(ReaderWriterLockSlim readerWriterLockSlim)
+                : base(readerWriterLockSlim)
+            {
+            }
+
+            protected override void EnterLock(ReaderWriterLockSlim readerWriterLockSlim)
+            {
+                readerWriterLockSlim.EnterReadLock();
+            }
+
+            protected override void ExitLock(ReaderWriterLockSlim readerWriterLockSlim)
+            {
+                readerWriterLockSlim.ExitReadLock();
+            }
+        }
+
+        private sealed class WriteLockToken : ReaderWriterLockSlimToken
+        {
+            public WriteLockToken(ReaderWriterLockSlim readerWriterLockSlim)
+                : base(readerWriterLockSlim)
+            {
+            }
+
+            protected override void EnterLock(ReaderWriterLockSlim readerWriterLockSlim)
+            {
+                readerWriterLockSlim.EnterWriteLock();
+            }
+
+            protected override void ExitLock(ReaderWriterLockSlim readerWriterLockSlim)
+            {
+                readerWriterLockSlim.ExitWriteLock();
+            }
+        }
+
+        private abstract class ReaderWriterLockSlimToken : IDisposable
+        {
+            private ReaderWriterLockSlim _readerWriterLockSlim;
+
+            public ReaderWriterLockSlimToken(ReaderWriterLockSlim readerWriterLockSlim)
+            {
+                _readerWriterLockSlim = readerWriterLockSlim;
+                // ReSharper disable once DoNotCallOverridableMethodsInConstructor -- we control the subclasses, this is fine
+                EnterLock(_readerWriterLockSlim);
+            }
+
+            protected abstract void EnterLock(ReaderWriterLockSlim readerWriterLockSlim);
+            protected abstract void ExitLock(ReaderWriterLockSlim readerWriterLockSlim);
+
+            public void Dispose()
+            {
+                if (_readerWriterLockSlim != null)
+                {
+                    ExitLock(_readerWriterLockSlim);
+                    _readerWriterLockSlim = null;
+                }
+            }
+        }
+    }
+}

Common/Util/Ref.cs

@@ -0,0 +1,109 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Represents a read-only reference to any value, T
+    /// </summary>
+    /// <typeparam name="T">The data type the reference points to</typeparam>
+    public interface IReadOnlyRef<out T>
+    {
+        /// <summary>
+        /// Gets the current value this reference points to
+        /// </summary>
+        T Value { get; }
+    }
+
+    /// <summary>
+    /// Represents a reference to any value, T
+    /// </summary>
+    /// <typeparam name="T">The data type the reference points to</typeparam>
+    public sealed class Ref<T> : IReadOnlyRef<T>
+    {
+        private readonly Func<T> _getter;
+        private readonly Action<T> _setter;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Ref{T}"/> class
+        /// </summary>
+        /// <param name="getter">A function delegate to get the current value</param>
+        /// <param name="setter">A function delegate to set the current value</param>
+        public Ref(Func<T> getter, Action<T> setter)
+        {
+            _getter = getter;
+            _setter = setter;
+        }
+
+        /// <summary>
+        /// Gets or sets the value of this reference
+        /// </summary>
+        public T Value
+        {
+            get { return _getter(); }
+            set { _setter(value); }
+        }
+
+        /// <summary>
+        /// Returns a read-only version of this instance
+        /// </summary>
+        /// <returns>A new instance with read-only semantics/gaurantees</returns>
+        public IReadOnlyRef<T> AsReadOnly()
+        {
+            return new Ref<T>(_getter, value =>
+            {
+                throw new InvalidOperationException("This instance is read-only.");
+            });
+        }
+    }
+
+    /// <summary>
+    /// Provides some helper methods that leverage C# type inference
+    /// </summary>
+    public static class Ref
+    {
+        /// <summary>
+        /// Creates a new <see cref="Ref{T}"/> instance
+        /// </summary>
+        public static Ref<T> Create<T>(Func<T> getter, Action<T> setter)
+        {
+            return new Ref<T>(getter, setter);
+        }
+        /// <summary>
+        /// Creates a new <see cref="IReadOnlyRef{T}"/> instance
+        /// </summary>
+        public static IReadOnlyRef<T> CreateReadOnly<T>(Func<T> getter)
+        {
+            return new Ref<T>(getter, value =>
+            {
+                throw new InvalidOperationException("This instance is read-only.");
+            });
+        }
+        /// <summary>
+        /// Creates a new <see cref="Ref{T}"/> instance by closing over
+        /// the specified <paramref name="initialValue"/> variable.
+        /// NOTE: This won't close over the variable input to the function,
+        /// but rather a copy of the variable. This reference will use it's
+        /// own storage.
+        /// </summary>
+        public static Ref<T> Create<T>(T initialValue)
+        {
+            return new Ref<T>(() => initialValue, value => { initialValue = value; });
+        }
+    }
+}

Common/Util/ReferenceWrapper.cs

@@ -0,0 +1,41 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// We wrap a T instance, a value type, with a class, a reference type, to achieve thread safety when assigning new values
+    /// and reading from multiple threads. This is possible because assignments are atomic operations in C# for reference types (among others).
+    /// </summary>
+    /// <remarks>This is a simpler, performance oriented version of <see cref="Ref"/></remarks>
+    public class ReferenceWrapper<T> 
+        where T : struct
+    {
+        /// <summary>
+        /// The current value
+        /// </summary>
+        public T Value { get; init; }
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="value">The value to use</param>
+        public ReferenceWrapper(T value)
+        {
+            Value = value;
+        }
+    }
+}

Common/Util/SecurityExtensions.cs

@@ -0,0 +1,36 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides useful infrastructure methods to the <see cref="Security"/> class.
+    /// These are added in this way to avoid mudding the class's public API
+    /// </summary>
+    public static class SecurityExtensions
+    {
+        /// <summary>
+        /// Determines if all subscriptions for the security are internal feeds
+        /// </summary>
+        public static bool IsInternalFeed(this Security security)
+        {
+            return security.Subscriptions.All(x => x.IsInternalFeed);
+        }
+    }
+}

Common/Util/SecurityIdentifierJsonConverter.cs

@@ -0,0 +1,47 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using Newtonsoft.Json;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// A <see cref="JsonConverter"/> implementation that serializes a <see cref="SecurityIdentifier"/> as a string
+    /// </summary>
+    public class SecurityIdentifierJsonConverter : TypeChangeJsonConverter<SecurityIdentifier, string>
+    {
+        /// <summary>
+        /// Converts as security identifier to a string
+        /// </summary>
+        /// <param name="value">The input value to be converted before serialziation</param>
+        /// <returns>A new instance of TResult that is to be serialzied</returns>
+        protected override string Convert(SecurityIdentifier value)
+        {
+            return value.ToString();
+        }
+
+        /// <summary>
+        /// Converts the input string to a security identifier
+        /// </summary>
+        /// <param name="value">The deserialized value that needs to be converted to T</param>
+        /// <returns>The converted value</returns>
+        protected override SecurityIdentifier Convert(string value)
+        {
+            return SecurityIdentifier.Parse(value);
+        }
+    }
+}

Common/Util/SeriesJsonConverter.cs

@@ -0,0 +1,175 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using System.Drawing;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System.Collections.Generic;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Json Converter for Series which handles special Pie Series serialization case
+    /// </summary>
+    public class SeriesJsonConverter : JsonConverter
+    {
+        private ColorJsonConverter _colorJsonConverter = new ();
+
+        /// <summary>
+        /// Write Series to Json
+        /// </summary>
+        /// <param name="writer">The Json Writer to use</param>
+        /// <param name="value">The value to written to Json</param>
+        /// <param name="serializer">The Json Serializer to use</param>
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            var baseSeries = value as BaseSeries;
+            if (baseSeries == null)
+            {
+                return;
+            }
+
+            writer.WriteStartObject();
+
+            writer.WritePropertyName("name");
+            writer.WriteValue(baseSeries.Name);
+            writer.WritePropertyName("unit");
+            writer.WriteValue(baseSeries.Unit);
+            writer.WritePropertyName("index");
+            writer.WriteValue(baseSeries.Index);
+            writer.WritePropertyName("seriesType");
+            writer.WriteValue(baseSeries.SeriesType);
+
+            if (baseSeries.ZIndex.HasValue)
+            {
+                writer.WritePropertyName("zIndex");
+                writer.WriteValue(baseSeries.ZIndex.Value);
+            }
+
+            if (baseSeries.IndexName != null)
+            {
+                writer.WritePropertyName("indexName");
+                writer.WriteValue(baseSeries.IndexName);
+            }
+
+            if (baseSeries.Tooltip != null)
+            {
+                writer.WritePropertyName("tooltip");
+                writer.WriteValue(baseSeries.Tooltip);
+            }
+
+            switch (value)
+            {
+                case Series series:
+                    var values = series.Values;
+                    if (series.SeriesType == SeriesType.Pie)
+                    {
+                        values = new List<ISeriesPoint>();
+                        var dataPoint = series.ConsolidateChartPoints();
+                        if (dataPoint != null)
+                        {
+                            values.Add(dataPoint);
+                        }
+                    }
+
+                    // have to add the converter we want to use, else will use default
+                    serializer.Converters.Add(_colorJsonConverter);
+
+                    writer.WritePropertyName("values");
+                    serializer.Serialize(writer, values);
+                    writer.WritePropertyName("color");
+                    serializer.Serialize(writer, series.Color);
+                    writer.WritePropertyName("scatterMarkerSymbol");
+                    serializer.Serialize(writer, series.ScatterMarkerSymbol);
+                    break;
+
+                default:
+                    writer.WritePropertyName("values");
+                    serializer.Serialize(writer, baseSeries.Values);
+                    break;
+            }
+
+            writer.WriteEndObject();
+        }
+
+        /// <summary>
+        /// Reads series from Json
+        /// </summary>
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            var jObject = JObject.Load(reader);
+
+            var name = (jObject["Name"] ?? jObject["name"]).Value<string>();
+            var unit = (jObject["Unit"] ?? jObject["unit"]).Value<string>();
+            var index = (jObject["Index"] ?? jObject["index"]).Value<int>();
+            var seriesType = (SeriesType)(jObject["SeriesType"] ?? jObject["seriesType"]).Value<int>();
+            var values = (JArray)(jObject["Values"] ?? jObject["values"]);
+
+            var zindex = jObject.TryGetPropertyValue<int?>("ZIndex") ?? jObject.TryGetPropertyValue<int?>("zIndex");
+            var indexName = jObject.TryGetPropertyValue<string>("IndexName") ?? jObject.TryGetPropertyValue<string>("indexName");
+            var tooltip = jObject.TryGetPropertyValue<string>("Tooltip") ?? jObject.TryGetPropertyValue<string>("tooltip");
+
+            if (seriesType == SeriesType.Candle)
+            {
+                return new CandlestickSeries()
+                {
+                    Name = name,
+                    Unit = unit,
+                    Index = index,
+                    ZIndex = zindex,
+                    Tooltip = tooltip,
+                    IndexName = indexName,
+                    SeriesType = seriesType,
+                    Values = values.ToObject<List<Candlestick>>(serializer).Where(x => x != null).Cast<ISeriesPoint>().ToList()
+                };
+            }
+
+            var result = new Series()
+            {
+                Name = name,
+                Unit = unit,
+                Index = index,
+                ZIndex = zindex,
+                Tooltip = tooltip,
+                IndexName = indexName,
+                SeriesType = seriesType,
+                Color = (jObject["Color"] ?? jObject["color"])?.ToObject<Color>(serializer) ?? Color.Empty,
+                ScatterMarkerSymbol = (jObject["ScatterMarkerSymbol"] ?? jObject["scatterMarkerSymbol"])?.ToObject<ScatterMarkerSymbol>(serializer) ?? ScatterMarkerSymbol.None
+            };
+
+            if (seriesType == SeriesType.Scatter)
+            {
+                result.Values = values.ToObject<List<ScatterChartPoint>>(serializer).Where(x => x != null).Cast<ISeriesPoint>().ToList();
+            }
+            else
+            {
+                result.Values = values.ToObject<List<ChartPoint>>(serializer).Where(x => x != null).Cast<ISeriesPoint>().ToList();
+            }
+            return result;
+        }
+
+        /// <summary>
+        /// Determine if this Converter can convert this type
+        /// </summary>
+        /// <param name="objectType">Type that we would like to convert</param>
+        /// <returns>True if <see cref="Series"/></returns>
+        public override bool CanConvert(Type objectType)
+        {
+            return typeof(BaseSeries).IsAssignableFrom(objectType);
+        }
+    }
+}

Common/Util/SingleValueListConverter.cs

@@ -0,0 +1,76 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Reads json and always produces a List, even if the input has just an object
+    /// </summary>
+    public class SingleValueListConverter<T> : JsonConverter
+    {
+        /// <summary>
+        /// Writes the JSON representation of the object. If the instance is not a list then it will
+        /// be wrapped in a list
+        /// </summary>
+        /// <param name="writer">The <see cref="T:Newtonsoft.Json.JsonWriter"/> to write to.</param>
+        /// <param name="value">The value.</param>
+        /// <param name="serializer">The calling serializer.</param>
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            if (value is T)
+            {
+                value = new List<T> {(T)value};
+            }
+            serializer.Serialize(writer, value);
+        }
+
+        /// <summary>
+        /// Reads the JSON representation of the object. If the JSON represents a singular instance, it will be returned
+        /// in a list.
+        /// </summary>
+        /// <param name="reader">The <see cref="T:Newtonsoft.Json.JsonReader"/> to read from.</param>
+        /// <param name="objectType">Type of the object.</param>
+        /// <param name="existingValue">The existing value of object being read.</param>
+        /// <param name="serializer">The calling serializer.</param>
+        /// <returns>The object value</returns>
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            switch (reader.TokenType)
+            {
+                case JsonToken.String:
+                case JsonToken.StartObject:
+                    return new List<T> {serializer.Deserialize<T>(reader)};
+                case JsonToken.StartArray:
+                    return serializer.Deserialize<List<T>>(reader);
+                default:
+                    throw new ArgumentException("The JsonReader is expected to point at a JsonToken.StartObject or JsonToken.StartArray.");
+            }
+        }
+
+        /// <summary>
+        /// Determines whether this instance can convert the specified object type.
+        /// </summary>
+        /// <param name="objectType">Type of the object.</param>
+        /// <returns><c>true</c> if this instance can convert the specified object type; otherwise, <c>false</c>.</returns>
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(T) || objectType == typeof(List<T>);
+        }
+    }
+}

Common/Util/StreamReaderEnumerable.cs

@@ -0,0 +1,155 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Converts a <see cref="StreamReader"/> into an enumerable of string
+    /// </summary>
+    public class StreamReaderEnumerable : IEnumerable<string>, IDisposable
+    {
+        private int _disposed;
+        private int _createdEnumerator;
+        private readonly StreamReader _reader;
+        private readonly IDisposable[] _disposables;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="StreamReaderEnumerable"/> class
+        /// </summary>
+        /// <param name="stream">The stream to be read</param>
+        /// <param name="disposables">Allows specifying other resources that should be disposed when this instance is disposed</param>
+        public StreamReaderEnumerable(Stream stream, params IDisposable[] disposables)
+        {
+            _disposables = disposables;
+
+            // this StreamReader constructor gives ownership of the stream to the StreamReader
+            // which is mediated by the LeaveOpen property, so when _reader is disposed, stream
+            // will also be disposed
+            _reader = new StreamReader(stream);
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="StreamReaderEnumerable"/> class
+        /// </summary>
+        /// <param name="reader">The stream reader instance to convert to an enumerable of string</param>
+        /// <param name="disposables">Allows specifying other resources that should be disposed when this instance is disposed</param>
+        public StreamReaderEnumerable(StreamReader reader, params IDisposable[] disposables)
+        {
+            _reader = reader;
+            _disposables = disposables;
+        }
+
+        /// <summary>Returns an enumerator that iterates through the collection.</summary>
+        /// <returns>A <see cref="T:System.Collections.Generic.IEnumerator`1" /> that can be used to iterate through the collection.</returns>
+        /// <filterpriority>1</filterpriority>
+        public IEnumerator<string> GetEnumerator()
+        {
+            // can't share the underlying stream instance -- barf
+            if (Interlocked.CompareExchange(ref _createdEnumerator, 1, 0) == 1)
+            {
+                throw new InvalidOperationException("A StreamReaderEnumerable may only be enumerated once. Consider using memoization or materialization.");
+            }
+
+            return new Enumerator(this);
+        }
+
+        /// <summary>Returns an enumerator that iterates through a collection.</summary>
+        /// <returns>An <see cref="T:System.Collections.IEnumerator" /> object that can be used to iterate through the collection.</returns>
+        /// <filterpriority>2</filterpriority>
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+
+        /// <summary>Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.</summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            if (Interlocked.CompareExchange(ref _disposed, 1, 0) == 1)
+            {
+                return;
+            }
+
+            _reader.DisposeSafely();
+            if (_disposables != null)
+            {
+                foreach (var disposable in _disposables)
+                {
+                    disposable.DisposeSafely();
+                }
+            }
+        }
+
+        /// <summary>Allows an object to try to free resources and perform other cleanup operations before it is reclaimed by garbage collection.</summary>
+        ~StreamReaderEnumerable()
+        {
+            // be sure to clean up unmanaged resources via finalizer if
+            // dispose wasn't explicitly called by consuming code
+            Dispose();
+        }
+
+        private class Enumerator : IEnumerator<string>
+        {
+            private readonly StreamReaderEnumerable _enumerable;
+
+            public string Current { get; private set; }
+
+            object IEnumerator.Current => Current;
+
+            public Enumerator(StreamReaderEnumerable enumerable)
+            {
+                _enumerable = enumerable;
+            }
+
+            public bool MoveNext()
+            {
+                var line = _enumerable._reader.ReadLine();
+                if (line == null)
+                {
+                    return false;
+                }
+
+                Current = line;
+                return true;
+            }
+
+            public void Reset()
+            {
+                if (!_enumerable._reader.BaseStream.CanSeek)
+                {
+                    throw new InvalidOperationException("The underlying stream is unseekable");
+                }
+
+                _enumerable._reader.BaseStream.Seek(0, SeekOrigin.Begin);
+            }
+
+            public void Dispose()
+            {
+                _enumerable.Dispose();
+            }
+
+            ~Enumerator()
+            {
+                _enumerable.Dispose();
+            }
+        }
+    }
+}

Common/Util/StreamReaderExtensions.cs

@@ -0,0 +1,250 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.Text;
+using System.Globalization;
+using System.Collections.Concurrent;
+using System.Runtime.CompilerServices;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Extension methods to fetch data from a <see cref="StreamReader"/> instance
+    /// </summary>
+    /// <remarks>The value of these methods is performance. The objective is to avoid  using
+    /// <see cref="StreamReader.ReadLine"/> and having to create intermediate substrings, parsing and splitting</remarks>
+    public static class StreamReaderExtensions
+    {
+        // we use '-1' value as a flag to determine whether we have decimal places or not, so we avoid having another variable required
+        private const int NoDecimalPlaces = -1;
+        private const char NoMoreData = unchecked((char)-1);
+        private const char DefaultDelimiter = ',';
+
+        /// <summary>
+        /// Gets a decimal from the provided stream reader
+        /// </summary>
+        /// <param name="stream">The data stream</param>
+        /// <param name="delimiter">The data delimiter character to use, default is ','</param>
+        /// <returns>The decimal read from the stream</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static decimal GetDecimal(this StreamReader stream, char delimiter = DefaultDelimiter)
+        {
+            return GetDecimal(stream, out _, delimiter);
+        }
+
+        /// <summary>
+        /// Gets a decimal from the provided stream reader
+        /// </summary>
+        /// <param name="stream">The data stream</param>
+        /// <param name="delimiter">The data delimiter character to use, default is ','</param>
+        /// <param name="pastEndLine">True if end line was past, useful for consumers to know a line ended</param>
+        /// <returns>The decimal read from the stream</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static decimal GetDecimal(this StreamReader stream, out bool pastEndLine, char delimiter = DefaultDelimiter)
+        {
+            long value = 0;
+            var decimalPlaces = NoDecimalPlaces;
+            var current = (char)stream.Read();
+
+            while (current == ' ')
+            {
+                current = (char)stream.Read();
+            }
+
+            var isNegative = current == '-';
+            if (isNegative)
+            {
+                current = (char)stream.Read();
+            }
+
+            pastEndLine = current == '\n' || current == '\r' && (stream.Peek() != '\n' || stream.Read() == '\n') || current == NoMoreData;
+            while (!(current == delimiter || pastEndLine || current == ' '))
+            {
+                if (current == '.')
+                {
+                    decimalPlaces = 0;
+                }
+                else
+                {
+                    value = value * 10 + (current - '0');
+                    if (decimalPlaces != NoDecimalPlaces)
+                    {
+                        decimalPlaces++;
+                    }
+                }
+                current = (char)stream.Read();
+                pastEndLine = current == '\n' || current == '\r' && (stream.Peek() != '\n' || stream.Read() == '\n') || current == NoMoreData;
+            }
+
+            var lo = (int)value;
+            var mid = (int)(value >> 32);
+            return new decimal(lo, mid, 0, isNegative, (byte)(decimalPlaces != NoDecimalPlaces ? decimalPlaces : 0));
+        }
+
+        /// <summary>
+        /// Gets a date time instance from a stream reader
+        /// </summary>
+        /// <param name="stream">The data stream</param>
+        /// <param name="format">The format in which the date time is</param>
+        /// <param name="delimiter">The data delimiter character to use, default is ','</param>
+        /// <returns>The date time instance read</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static DateTime GetDateTime(this StreamReader stream, string format = DateFormat.TwelveCharacter, char delimiter = DefaultDelimiter)
+        {
+            var current = (char)stream.Read();
+            while (current == ' ')
+            {
+                current = (char)stream.Read();
+            }
+
+            var index = 0;
+            // we know the exact format we want to parse so we can allocate the char array and not use an expensive string builder
+            var data = new char[format.Length];
+            while (!(current == delimiter || current == '\n' || current == '\r' && (stream.Peek() != '\n' || stream.Read() == '\n') || current == NoMoreData))
+            {
+                data[index++] = current;
+                current = (char)stream.Read();
+            }
+
+            return DateTime.ParseExact(data,
+                format,
+                CultureInfo.InvariantCulture);
+        }
+
+        /// <summary>
+        /// Gets an integer from a stream reader
+        /// </summary>
+        /// <param name="stream">The data stream</param>
+        /// <param name="delimiter">The data delimiter character to use, default is ','</param>
+        /// <returns>The integer instance read</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int GetInt32(this StreamReader stream, char delimiter = DefaultDelimiter)
+        {
+            var result = 0;
+            var current = (char)stream.Read();
+
+            while (current == ' ')
+            {
+                current = (char)stream.Read();
+            }
+
+            var isNegative = current == '-';
+            if (isNegative)
+            {
+                current = (char)stream.Read();
+            }
+
+            while (!(current == delimiter || current == '\n' || current == '\r' && (stream.Peek() != '\n' || stream.Read() == '\n') || current == NoMoreData || current == ' '))
+            {
+                result = (current - '0') + result * 10;
+                current = (char)stream.Read();
+            }
+            return isNegative ? result * -1 : result;
+        }
+
+        /// <summary>
+        /// Gets an integer from a stream reader
+        /// </summary>
+        /// <param name="stream">The data stream</param>
+        /// <param name="delimiter">The data delimiter character to use, default is ','</param>
+        /// <returns>The integer instance read</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static long GetInt64(this StreamReader stream, char delimiter = DefaultDelimiter)
+        {
+            var result = 0L;
+            var current = (char)stream.Read();
+
+            while (current == ' ')
+            {
+                current = (char)stream.Read();
+            }
+
+            var isNegative = current == '-';
+            if (isNegative)
+            {
+                current = (char)stream.Read();
+            }
+
+            while (!(current == delimiter || current == '\n' || current == '\r' && (stream.Peek() != '\n' || stream.Read() == '\n') || current == NoMoreData || current == ' '))
+            {
+                result = (current - '0') + result * 10L;
+                current = (char)stream.Read();
+            }
+            return isNegative ? result * -1L : result;
+        }
+
+        private readonly static ConcurrentBag<StringBuilder> StringBuilders = new();
+
+        /// <summary>
+        /// Gets a string from a stream reader
+        /// </summary>
+        /// <param name="stream">The data stream</param>
+        /// <param name="delimiter">The data delimiter character to use, default is ','</param>
+        /// <returns>The string instance read</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static string GetString(this StreamReader stream, char delimiter = DefaultDelimiter)
+        {
+            if (!StringBuilders.TryTake(out var builder))
+            {
+                builder = new();
+            }
+
+            try
+            {
+                var current = (char)stream.Read();
+
+                while (!(current == delimiter || current == '\n' || current == '\r' && (stream.Peek() != '\n' || stream.Read() == '\n') || current == NoMoreData))
+                {
+                    builder.Append(current);
+                    current = (char)stream.Read();
+                }
+                return builder.ToString();
+            }
+            finally
+            {
+                builder.Clear();
+                StringBuilders.Add(builder);
+            }
+        }
+
+        /// <summary>
+        /// Gets a character from a stream reader
+        /// </summary>
+        /// <param name="stream">The data stream</param>
+        /// <param name="delimiter">The data delimiter character to use, default is ','</param>
+        /// <returns>The string instance read</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static char GetChar(this StreamReader stream, char delimiter = DefaultDelimiter)
+        {
+            var current = (char)stream.Read();
+            var next = (char)stream.Peek();
+
+            if (current == delimiter || current == '\n' || current == '\r' && (next != '\n' || stream.Read() == '\n') || current == NoMoreData)
+            {
+                return '\0';
+            }
+
+            if (next == delimiter || next == '\n' || next == '\r' && stream.Read() == '\r' && stream.Peek() == '\n' || next == NoMoreData)
+            {
+                // Consume the delimiter
+                stream.Read();
+            }
+
+            return current;
+        }
+    }
+}

Common/Util/StringDecimalJsonConverter.cs

@@ -0,0 +1,70 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Globalization;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Allows for conversion of string numeric values from JSON to the <see cref="decimal"/> type
+    /// </summary>
+    public class StringDecimalJsonConverter : TypeChangeJsonConverter<decimal, string>
+    {
+        private readonly bool _defaultOnFailure;
+
+        /// <summary>
+        /// Creates an instance of the class, with an optional flag to default to decimal's default value on failure.
+        /// </summary>
+        /// <param name="defaultOnFailure">Default to decimal's default value on failure</param>
+        public StringDecimalJsonConverter(bool defaultOnFailure = false)
+        {
+            _defaultOnFailure = defaultOnFailure;
+        }
+
+        /// <summary>
+        /// Converts a decimal to a string
+        /// </summary>
+        /// <param name="value">The input value to be converted before serialization</param>
+        /// <returns>String representation of the decimal</returns>
+        protected override string Convert(decimal value)
+        {
+            return value.ToStringInvariant();
+        }
+
+        /// <summary>
+        /// Converts the input string to a decimal
+        /// </summary>
+        /// <param name="value">The deserialized value that needs to be converted to T</param>
+        /// <returns>The converted value</returns>
+        protected override decimal Convert(string value)
+        {
+            try
+            {
+                return decimal.Parse(value, NumberStyles.Any, CultureInfo.InvariantCulture);
+            }
+            catch (Exception)
+            {
+                if (_defaultOnFailure)
+                {
+                    return default(decimal);
+                }
+
+                throw;
+            }
+        }
+    }
+}

Common/Util/TypeChangeJsonConverter.cs

@@ -0,0 +1,113 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides a base class for a <see cref="JsonConverter"/> that serializes a
+    /// an input type as some other output type
+    /// </summary>
+    /// <typeparam name="T">The type to be serialized</typeparam>
+    /// <typeparam name="TResult">The output serialized type</typeparam>
+    public abstract class TypeChangeJsonConverter<T, TResult> : JsonConverter
+    {
+        // we use a json serializer which allows using non public default constructor
+        private readonly JsonSerializer _jsonSerializer = new JsonSerializer {ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor};
+
+        /// <summary>
+        /// True will populate TResult object returned by <see cref="Convert(TResult)"/> with json properties
+        /// </summary>
+        protected virtual bool PopulateProperties => true;
+
+        /// <summary>
+        /// Reads the JSON representation of the object.
+        /// </summary>
+        /// <param name="reader">The <see cref="T:Newtonsoft.Json.JsonReader"/> to read from.</param><param name="objectType">Type of the object.</param><param name="existingValue">The existing value of object being read.</param><param name="serializer">The calling serializer.</param>
+        /// <returns>
+        /// The object value.
+        /// </returns>
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            // Load token from stream
+            var token = JToken.Load(reader);
+
+            // Create target object based on token
+            var target = Create(objectType, token);
+
+            return target;
+        }
+
+        /// <summary>
+        /// Writes the JSON representation of the object.
+        /// </summary>
+        /// <param name="writer">The <see cref="T:Newtonsoft.Json.JsonWriter"/> to write to.</param><param name="value">The value.</param><param name="serializer">The calling serializer.</param>
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            // Convert the value into TResult to be serialized
+            var valueToSerialize = Convert((T)value);
+
+            serializer.Serialize(writer, valueToSerialize);
+        }
+
+        /// <summary>
+        /// Determines whether this instance can convert the specified object type.
+        /// </summary>
+        /// <param name="objectType">Type of the object.</param>
+        /// <returns>
+        /// <c>true</c> if this instance can convert the specified object type; otherwise, <c>false</c>.
+        /// </returns>
+        public override bool CanConvert(Type objectType)
+        {
+            return typeof(T) == objectType;
+        }
+
+        /// <summary>
+        /// Creates an instance of the un-projected type to be deserialized
+        /// </summary>
+        /// <param name="type">The input object type, this is the data held in the token</param>
+        /// <param name="token">The input data to be converted into a T</param>
+        /// <returns>A new instance of T that is to be serialized using default rules</returns>
+        protected virtual T Create(Type type, JToken token)
+        {
+            // reads the token as an object type
+            if (typeof(TResult).IsClass && typeof(T) != typeof(string))
+            {
+                return Convert(token.ToObject<TResult>(_jsonSerializer));
+            }
+
+            // reads the token as a value type
+            return Convert(token.Value<TResult>());
+        }
+
+        /// <summary>
+        /// Convert the input value to a value to be serialized
+        /// </summary>
+        /// <param name="value">The input value to be converted before serialziation</param>
+        /// <returns>A new instance of TResult that is to be serialzied</returns>
+        protected abstract TResult Convert(T value);
+
+        /// <summary>
+        /// Converts the input value to be deserialized
+        /// </summary>
+        /// <param name="value">The deserialized value that needs to be converted to T</param>
+        /// <returns>The converted value</returns>
+        protected abstract T Convert(TResult value);
+    }
+}

Common/Util/Validate.cs

@@ -0,0 +1,108 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Globalization;
+using System.Text.RegularExpressions;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides methods for validating strings following a certain format, such as an email address
+    /// </summary>
+    public static class Validate
+    {
+        /// <summary>
+        /// Validates the provided email address
+        /// </summary>
+        /// <remarks>
+        /// Implementation taken from msdn (with slight refactoring for readability and C#6 compliance):
+        /// https://docs.microsoft.com/en-us/dotnet/standard/base-types/how-to-verify-that-strings-are-in-valid-email-format
+        /// </remarks>
+        /// <param name="emailAddress">The email address to be validated</param>
+        /// <returns>True if the provided email address is valid</returns>
+        public static bool EmailAddress(string emailAddress)
+        {
+            if (emailAddress == null)
+            {
+                return true;
+            }
+
+            if (string.IsNullOrWhiteSpace(emailAddress))
+            {
+                return false;
+            }
+
+            emailAddress = NormalizeEmailAddressDomainName(emailAddress);
+            if (emailAddress == null)
+            {
+                // an error occurred during domain name normalization
+                return false;
+            }
+
+            try
+            {
+                return RegularExpression.Email.IsMatch(emailAddress);
+            }
+            catch
+            {
+                return false;
+            }
+        }
+
+        private static string NormalizeEmailAddressDomainName(string emailAddress)
+        {
+            try
+            {
+                // Normalize the domain
+                emailAddress = RegularExpression.EmailDomainName.Replace(emailAddress, match =>
+                {
+                    // Use IdnMapping class to convert Unicode domain names.
+                    var idn = new IdnMapping();
+
+                    // Pull out and process domain name (throws ArgumentException on invalid)
+                    var domainName = idn.GetAscii(match.Groups[2].Value);
+
+                    return match.Groups[1].Value + domainName;
+                });
+            }
+            catch
+            {
+                return null;
+            }
+
+            return emailAddress;
+        }
+
+        /// <summary>
+        /// Provides static storage of compiled regular expressions to preclude parsing on each invocation
+        /// </summary>
+        public static class RegularExpression
+        {
+            /// <summary>
+            /// Matches the domain name in an email address ignored@[domain.com]
+            /// Pattern sourced via msdn:
+            /// https://docs.microsoft.com/en-us/dotnet/standard/base-types/how-to-verify-that-strings-are-in-valid-email-format
+            /// </summary>
+            public static readonly Regex EmailDomainName = new Regex(@"(@)(.+)$", RegexOptions.Compiled);
+
+            /// <summary>
+            /// Matches a valid email address address@sub.domain.com
+            /// Pattern sourced via msdn:
+            /// https://docs.microsoft.com/en-us/dotnet/standard/base-types/how-to-verify-that-strings-are-in-valid-email-format
+            /// </summary>
+            public static readonly Regex Email = new Regex(@"^(?("")("".+?(?<!\\)""@)|(([0-9a-z]((\.(?!\.))|[-!#\$%&'\*\+/=\?\^`\{\}\|~\w])*)(?<=[0-9a-z])@))(?(\[)(\[(\d{1,3}\.){3}\d{1,3}\])|(([0-9a-z][-0-9a-z]*[0-9a-z]*\.)+[a-z0-9][\-a-z0-9]{0,22}[a-z0-9]))$", RegexOptions.IgnoreCase | RegexOptions.Compiled);
+        }
+    }
+}

Common/Util/WorkerThread.cs

@@ -0,0 +1,112 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+using QuantConnect.Logging;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// This worker tread is required to guarantee all python operations are
+    /// executed by the same thread, to enable complete debugging functionality.
+    /// We don't use the main thread, to avoid any chance of blocking the process
+    /// </summary>
+    public class WorkerThread : IDisposable
+    {
+        private readonly BlockingCollection<Action> _blockingCollection;
+        private readonly CancellationTokenSource _threadCancellationTokenSource;
+        private readonly Thread _workerThread;
+
+        /// <summary>
+        /// The worker thread instance
+        /// </summary>
+        public static WorkerThread Instance = new WorkerThread();
+
+        /// <summary>
+        /// Will be set when the worker thread finishes a work item
+        /// </summary>
+        public AutoResetEvent FinishedWorkItem { get; }
+
+        /// <summary>
+        /// Creates a new instance, which internally launches a new worker thread
+        /// </summary>
+        /// <remarks><see cref="Dispose"/></remarks>
+        protected WorkerThread()
+        {
+            _threadCancellationTokenSource = new CancellationTokenSource();
+            FinishedWorkItem = new AutoResetEvent(false);
+            _blockingCollection = new BlockingCollection<Action>();
+            _workerThread = new Thread(() =>
+            {
+                try
+                {
+                    foreach (var action in _blockingCollection.GetConsumingEnumerable(_threadCancellationTokenSource.Token))
+                    {
+                        FinishedWorkItem.Reset();
+                        try
+                        {
+                            action();
+                        }
+                        catch (Exception exception)
+                        {
+                            Log.Error(exception, "WorkerThread(): exception thrown when running task");
+                        }
+                        FinishedWorkItem.Set();
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    // pass, when the token gets cancelled
+                }
+            })
+            {
+                IsBackground = true,
+                Name = "Isolator Thread",
+                Priority = ThreadPriority.Highest
+            };
+            _workerThread.Start();
+        }
+
+        /// <summary>
+        /// Adds a new item of work
+        /// </summary>
+        /// <param name="action">The work item to add</param>
+        public void Add(Action action)
+        {
+            _blockingCollection.Add(action);
+        }
+
+        /// <summary>
+        /// Disposes the worker thread.
+        /// </summary>
+        /// <remarks>Note that the worker thread is a background thread,
+        /// so it won't block the process from terminating even if not disposed</remarks>
+        public virtual void Dispose()
+        {
+            try
+            {
+                _blockingCollection.CompleteAdding();
+                _workerThread.StopSafely(TimeSpan.FromMilliseconds(50), _threadCancellationTokenSource);
+                _threadCancellationTokenSource.DisposeSafely();
+            }
+            catch (Exception exception)
+            {
+                Log.Error(exception);
+            }
+        }
+    }
+}

Common/Util/XElementExtensions.cs

@@ -0,0 +1,42 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using System.Xml.Linq;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Provides extension methods for the XML to LINQ types
+    /// </summary>
+    public static class XElementExtensions
+    {
+        /// <summary>
+        /// Gets the value from the element and converts it to the specified type.
+        /// </summary>
+        /// <typeparam name="T">The output type</typeparam>
+        /// <param name="element">The element to access</param>
+        /// <param name="name">The attribute name to access on the element</param>
+        /// <returns>The converted value</returns>
+        public static T Get<T>(this XElement element, string name) 
+            where T : IConvertible
+        {
+            var xAttribute = element.Descendants(name).Single();
+            string value = xAttribute.Value;
+            return value.ConvertTo<T>();
+        }
+    }
+}