chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,476 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Newtonsoft.Json;
|
||||
using NUnit.Framework;
|
||||
using ProtoBuf;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Custom.IconicTypes;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Orders;
|
||||
using QuantConnect.Orders.Fees;
|
||||
using QuantConnect.Securities;
|
||||
|
||||
namespace QuantConnect.Tests.Common
|
||||
{
|
||||
[TestFixture, Parallelizable(ParallelScope.All)]
|
||||
public class ProtobufSerializationTests
|
||||
{
|
||||
private static readonly Dictionary<Type, BaseData> _iconicInstances = new Dictionary<Type, BaseData>
|
||||
{
|
||||
{ typeof(IndexedLinkedData), new IndexedLinkedData { Count = 1024 } },
|
||||
{ typeof(IndexedLinkedData2), new IndexedLinkedData2 { Count = 2048 } },
|
||||
{ typeof(LinkedData), new LinkedData { Count = 4096 } },
|
||||
{ typeof(UnlinkedData), new UnlinkedData { Ticker = "ABCDEF" } },
|
||||
{ typeof(UnlinkedDataTradeBar), new UnlinkedDataTradeBar
|
||||
{
|
||||
Open = 10m,
|
||||
High = 11m,
|
||||
Low = 9m,
|
||||
Close = 10.99m,
|
||||
Volume = 9999m
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
[TestCase(typeof(IndexedLinkedData), true)]
|
||||
[TestCase(typeof(IndexedLinkedData2), true)]
|
||||
[TestCase(typeof(LinkedData), true)]
|
||||
[TestCase(typeof(UnlinkedData), false)]
|
||||
[TestCase(typeof(UnlinkedDataTradeBar), false)]
|
||||
public void SerializeRoundTripIconicDataTypes(Type baseDataType, bool hasUnderlyingSymbol)
|
||||
{
|
||||
var item = CreateNewInstance(baseDataType, hasUnderlyingSymbol);
|
||||
var serialized = item.ProtobufSerialize(new Guid());
|
||||
|
||||
using (var stream = new MemoryStream(serialized))
|
||||
{
|
||||
var deserialized = Serializer.Deserialize<IEnumerable<BaseData>>(stream).Single();
|
||||
AssertAreEqual(item, deserialized);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SymbolRoundTrip()
|
||||
{
|
||||
var symbol = Symbols.AAPL;
|
||||
|
||||
using (var stream = new MemoryStream())
|
||||
{
|
||||
Serializer.Serialize(stream, symbol);
|
||||
|
||||
stream.Position = 0;
|
||||
|
||||
var result = Serializer.Deserialize<Symbol>(stream);
|
||||
|
||||
Assert.AreEqual(symbol, result);
|
||||
Assert.AreEqual(symbol.GetHashCode(), result.GetHashCode());
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase(10)]
|
||||
[TestCase(100)]
|
||||
[TestCase(1000)]
|
||||
public void TickListSerializationRoundTrip(int tickCount)
|
||||
{
|
||||
var time = DateTime.UtcNow;
|
||||
var ticks = new List<Tick>();
|
||||
for (int i = 0; i < tickCount; i++)
|
||||
{
|
||||
var tick = new Tick
|
||||
{
|
||||
Symbol = Symbols.AAPL,
|
||||
AskPrice = i,
|
||||
AskSize = i,
|
||||
Time = time + TimeSpan.FromMilliseconds(i),
|
||||
Quantity = i,
|
||||
DataType = MarketDataType.Tick,
|
||||
Exchange = "NASDAQ",
|
||||
SaleCondition = "VerySold",
|
||||
TickType = TickType.Quote,
|
||||
Value = i,
|
||||
BidPrice = i,
|
||||
BidSize = i
|
||||
};
|
||||
ticks.Add(tick);
|
||||
}
|
||||
|
||||
var stopwatch = new Stopwatch();
|
||||
stopwatch.Start();
|
||||
var serializedTick = ticks.ProtobufSerialize(new Guid());
|
||||
stopwatch.Stop();
|
||||
|
||||
Log.Trace($"Took {stopwatch.ElapsedMilliseconds}ms. TickCount : {tickCount}.");
|
||||
|
||||
// verify its correct
|
||||
using (var stream = new MemoryStream(serializedTick))
|
||||
{
|
||||
var results = Serializer.Deserialize<List<Tick>>(stream);
|
||||
|
||||
Assert.AreEqual(tickCount, results.Count);
|
||||
|
||||
for (int i = 0; i < tickCount; i++)
|
||||
{
|
||||
var result = results[i];
|
||||
Assert.AreEqual(i, result.AskPrice);
|
||||
Assert.AreEqual(i, result.AskSize);
|
||||
Assert.AreEqual(time + TimeSpan.FromMilliseconds(i), result.Time);
|
||||
Assert.AreEqual(i, result.Quantity);
|
||||
Assert.AreEqual(MarketDataType.Tick, result.DataType);
|
||||
Assert.AreEqual("NASDAQ", result.Exchange);
|
||||
Assert.AreEqual("VerySold", result.SaleCondition);
|
||||
Assert.AreEqual(TickType.Quote, result.TickType);
|
||||
Assert.AreEqual(time + TimeSpan.FromMilliseconds(i), result.EndTime);
|
||||
Assert.AreEqual(i, result.Value);
|
||||
Assert.AreEqual(i, result.BidPrice);
|
||||
Assert.AreEqual(i, result.BidSize);
|
||||
Assert.IsNull(result.Symbol);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void OpenInterestSerializationRoundTrip()
|
||||
{
|
||||
var openInterest = new OpenInterest(DateTime.UtcNow, Symbols.AAPL, 10);
|
||||
|
||||
var serializedTick = openInterest.ProtobufSerialize(new Guid());
|
||||
|
||||
// verify its correct
|
||||
using (var stream = new MemoryStream(serializedTick))
|
||||
{
|
||||
var result = (Tick)Serializer.Deserialize<IEnumerable<BaseData>>(stream).First();
|
||||
|
||||
Assert.IsNull(result.Symbol);
|
||||
Assert.AreEqual(openInterest.Time, result.Time);
|
||||
Assert.AreEqual(openInterest.EndTime, result.EndTime);
|
||||
Assert.AreEqual(openInterest.Value, result.Value);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TickSerializationRoundTrip()
|
||||
{
|
||||
var tick = new Tick
|
||||
{
|
||||
Symbol = Symbols.AAPL,
|
||||
AskPrice = 10,
|
||||
AskSize = 10,
|
||||
Time = DateTime.UtcNow,
|
||||
Quantity = 10,
|
||||
DataType = MarketDataType.Tick,
|
||||
Exchange = "NASDAQ",
|
||||
SaleCondition = "VerySold",
|
||||
TickType = TickType.Quote,
|
||||
EndTime = DateTime.UtcNow,
|
||||
Value = 10,
|
||||
BidPrice = 100,
|
||||
BidSize = 100
|
||||
};
|
||||
|
||||
var serializedTick = tick.ProtobufSerialize(new Guid());
|
||||
|
||||
// verify its correct
|
||||
using (var stream = new MemoryStream(serializedTick))
|
||||
{
|
||||
var result = (Tick) Serializer.Deserialize<IEnumerable<BaseData>>(stream).First();
|
||||
|
||||
Assert.AreEqual(tick.AskPrice, result.AskPrice);
|
||||
Assert.AreEqual(tick.AskSize, result.AskSize);
|
||||
Assert.AreEqual(tick.Time, result.Time);
|
||||
Assert.AreEqual(tick.Quantity, result.Quantity);
|
||||
Assert.AreEqual(tick.DataType, result.DataType);
|
||||
Assert.AreEqual("NASDAQ", result.Exchange);
|
||||
Assert.AreEqual(tick.SaleCondition, result.SaleCondition);
|
||||
Assert.AreEqual(tick.TickType, result.TickType);
|
||||
Assert.AreEqual(tick.EndTime, result.EndTime);
|
||||
Assert.AreEqual(tick.Value, result.Value);
|
||||
Assert.AreEqual(tick.BidPrice, result.BidPrice);
|
||||
Assert.AreEqual(tick.BidSize, result.BidSize);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TradeBarSerializationRoundTrip()
|
||||
{
|
||||
var tradeBar = new TradeBar
|
||||
{
|
||||
Symbol = Symbols.AAPL,
|
||||
Volume = 10,
|
||||
Time = DateTime.UtcNow,
|
||||
EndTime = DateTime.UtcNow,
|
||||
Value = 10,
|
||||
Close = 10,
|
||||
High = 100,
|
||||
Low = 100,
|
||||
Open = 100,
|
||||
Period = TimeSpan.FromMinutes(1)
|
||||
};
|
||||
|
||||
var serializedTradeBar = tradeBar.ProtobufSerialize(new Guid());
|
||||
using (var stream = new MemoryStream(serializedTradeBar))
|
||||
{
|
||||
// verify its correct
|
||||
var result = (TradeBar) Serializer.Deserialize<IEnumerable<BaseData>>(stream).First();
|
||||
|
||||
Assert.AreEqual(tradeBar.Time, result.Time);
|
||||
Assert.AreEqual(tradeBar.DataType, result.DataType);
|
||||
Assert.AreEqual(tradeBar.EndTime, result.EndTime);
|
||||
Assert.AreEqual(tradeBar.Value, result.Value);
|
||||
Assert.AreEqual(tradeBar.Volume, result.Volume);
|
||||
Assert.AreEqual(tradeBar.Close, result.Close);
|
||||
Assert.AreEqual(tradeBar.High, result.High);
|
||||
Assert.AreEqual(tradeBar.Low, result.Low);
|
||||
Assert.AreEqual(tradeBar.Open, result.Open);
|
||||
Assert.AreEqual(tradeBar.Period, result.Period);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void QuoteBarSerializationRoundTrip()
|
||||
{
|
||||
var quoteBar = new QuoteBar
|
||||
{
|
||||
Symbol = Symbols.AAPL,
|
||||
Time = DateTime.UtcNow,
|
||||
EndTime = DateTime.UtcNow,
|
||||
Value = 10,
|
||||
LastAskSize = 10,
|
||||
LastBidSize = 100,
|
||||
Ask = new Bar(1, 2, 3, 4),
|
||||
Bid = new Bar(11, 22, 33, 44),
|
||||
Period = TimeSpan.FromMinutes(1)
|
||||
};
|
||||
|
||||
var serializedQuoteBar = quoteBar.ProtobufSerialize(new Guid());
|
||||
using (var stream = new MemoryStream(serializedQuoteBar))
|
||||
{
|
||||
// verify its correct
|
||||
var result = (QuoteBar)Serializer.Deserialize<IEnumerable<BaseData>>(stream).First();
|
||||
|
||||
Assert.AreEqual(quoteBar.Time, result.Time);
|
||||
Assert.AreEqual(quoteBar.DataType, result.DataType);
|
||||
Assert.AreEqual(quoteBar.EndTime, result.EndTime);
|
||||
Assert.AreEqual(quoteBar.Value, result.Value);
|
||||
Assert.AreEqual(quoteBar.Close, result.Close);
|
||||
Assert.AreEqual(quoteBar.High, result.High);
|
||||
Assert.AreEqual(quoteBar.Low, result.Low);
|
||||
Assert.AreEqual(quoteBar.Open, result.Open);
|
||||
Assert.AreEqual(quoteBar.Period, result.Period);
|
||||
|
||||
Assert.AreEqual(quoteBar.Ask.Close, result.Ask.Close);
|
||||
Assert.AreEqual(quoteBar.Ask.High, result.Ask.High);
|
||||
Assert.AreEqual(quoteBar.Ask.Low, result.Ask.Low);
|
||||
Assert.AreEqual(quoteBar.Ask.Open, result.Ask.Open);
|
||||
|
||||
Assert.AreEqual(quoteBar.Bid.Close, result.Bid.Close);
|
||||
Assert.AreEqual(quoteBar.Bid.High, result.Bid.High);
|
||||
Assert.AreEqual(quoteBar.Bid.Low, result.Bid.Low);
|
||||
Assert.AreEqual(quoteBar.Bid.Open, result.Bid.Open);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DividendRoundTrip()
|
||||
{
|
||||
var dividend = new Dividend
|
||||
{
|
||||
DataType = MarketDataType.Auxiliary,
|
||||
Distribution = 0.5m,
|
||||
ReferencePrice = decimal.MaxValue - 10000m,
|
||||
|
||||
Symbol = Symbols.AAPL,
|
||||
Time = DateTime.UtcNow,
|
||||
Value = 0.5m
|
||||
};
|
||||
|
||||
var serializedDividend = dividend.ProtobufSerialize(new Guid());
|
||||
using (var stream = new MemoryStream(serializedDividend))
|
||||
{
|
||||
var result = (Dividend)Serializer.Deserialize<IEnumerable<BaseData>>(stream).First();
|
||||
|
||||
Assert.AreEqual(dividend.DataType, result.DataType);
|
||||
Assert.AreEqual(dividend.Distribution, result.Distribution);
|
||||
Assert.AreEqual(dividend.ReferencePrice, result.ReferencePrice);
|
||||
Assert.AreEqual(dividend.Time, result.Time);
|
||||
Assert.AreEqual(dividend.EndTime, result.EndTime);
|
||||
Assert.AreEqual(dividend.Value, result.Value);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SplitRoundTrip()
|
||||
{
|
||||
var split = new Split(Symbols.AAPL, DateTime.UtcNow, decimal.MaxValue, decimal.MinValue, SplitType.SplitOccurred);
|
||||
|
||||
var serializedSplit = split.ProtobufSerialize(new Guid());
|
||||
using (var stream = new MemoryStream(serializedSplit))
|
||||
{
|
||||
var result = (Split)Serializer.Deserialize<IEnumerable<BaseData>>(stream).First();
|
||||
|
||||
Assert.AreEqual(split.Type, result.Type);
|
||||
Assert.AreEqual(split.DataType, result.DataType);
|
||||
Assert.AreEqual(split.SplitFactor, result.SplitFactor);
|
||||
Assert.AreEqual(split.ReferencePrice, result.ReferencePrice);
|
||||
Assert.AreEqual(split.Time, result.Time);
|
||||
Assert.AreEqual(split.EndTime, result.EndTime);
|
||||
Assert.AreEqual(split.Value, result.Value);
|
||||
}
|
||||
}
|
||||
|
||||
[Test, Ignore("Performance test")]
|
||||
public void SpeedTest()
|
||||
{
|
||||
var symbols = new List<Symbol>
|
||||
{
|
||||
Symbol.Create("SPY", SecurityType.Equity, Market.USA),
|
||||
Symbol.Create("DE30EUR", SecurityType.Cfd, Market.Oanda),
|
||||
Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase),
|
||||
Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Bitfinex),
|
||||
Symbol.Create("EURUSD", SecurityType.Forex, Market.FXCM),
|
||||
Symbol.Create("EURUSD", SecurityType.Forex, Market.Oanda),
|
||||
Symbol.CreateOption("SPY", Market.USA, OptionStyle.American, OptionRight.Call, 1, DateTime.UtcNow),
|
||||
Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, DateTime.UtcNow)
|
||||
};
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var ticks = new List<Tick>();
|
||||
for (var i = 0; i < 10000; i++)
|
||||
{
|
||||
foreach (var symbol in symbols)
|
||||
{
|
||||
ticks.Add(new Tick
|
||||
{
|
||||
Symbol = symbol,
|
||||
AskPrice = i * 10,
|
||||
AskSize = i * 10,
|
||||
Time = now,
|
||||
Quantity = 10,
|
||||
DataType = MarketDataType.Tick,
|
||||
Exchange = "Pinocho",
|
||||
SaleCondition = "VerySold",
|
||||
TickType = TickType.Quote,
|
||||
EndTime = now,
|
||||
Value = i * 10,
|
||||
BidPrice = i * 100,
|
||||
BidSize = i * 100
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var guid = new Guid();
|
||||
{
|
||||
// warmup
|
||||
var serialized = ticks.ProtobufSerialize(guid);
|
||||
using (var stream = new MemoryStream(serialized))
|
||||
{
|
||||
// verify its correct
|
||||
var result = Serializer.Deserialize<List<Tick>>(stream);
|
||||
}
|
||||
|
||||
var start = DateTime.UtcNow;
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
serialized = ticks.ProtobufSerialize(guid);
|
||||
}
|
||||
var end = DateTime.UtcNow;
|
||||
Log.Trace($"PROTO BUF TOOK {end - start}");
|
||||
}
|
||||
|
||||
{
|
||||
// warmup
|
||||
var serialized = JsonConvert.SerializeObject(ticks);
|
||||
|
||||
var start = DateTime.UtcNow;
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
serialized = JsonConvert.SerializeObject(ticks);
|
||||
}
|
||||
var end = DateTime.UtcNow;
|
||||
|
||||
Log.Trace($"JSON TOOK {end - start}");
|
||||
}
|
||||
}
|
||||
|
||||
private static BaseData CreateNewInstance(Type baseDataType, bool hasUnderlyingSymbol)
|
||||
{
|
||||
var instance = _iconicInstances[baseDataType];
|
||||
|
||||
instance.Symbol = hasUnderlyingSymbol
|
||||
? Symbol.CreateBase(baseDataType, Symbols.AAPL, QuantConnect.Market.USA)
|
||||
: Symbol.Create("ABCDEF", SecurityType.Base, Market.USA, baseDataType: baseDataType);
|
||||
|
||||
instance.Time = new DateTime(2021, 6, 5);
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
private void AssertAreEqual(object expected, object result)
|
||||
{
|
||||
foreach (var propertyInfo in expected.GetType().GetProperties())
|
||||
{
|
||||
if (propertyInfo.CustomAttributes.Any(data => data.AttributeType == typeof(ProtoMemberAttribute)))
|
||||
{
|
||||
var expectedValue = propertyInfo.GetValue(expected);
|
||||
var resultValue = propertyInfo.GetValue(result);
|
||||
if (expectedValue is IList)
|
||||
{
|
||||
var expectedValueList = (IList) expectedValue;
|
||||
var resultValueList = (IList) resultValue;
|
||||
for (var i = 0; i < expectedValueList.Count; i++)
|
||||
{
|
||||
AssertAreEqual(expectedValueList[i], resultValueList[i]);
|
||||
}
|
||||
}
|
||||
else if (expectedValue is IDictionary)
|
||||
{
|
||||
var expectedValueDictionary = (IDictionary) expectedValue;
|
||||
var resultValueDictionary = (IDictionary) resultValue;
|
||||
foreach (dynamic kvp in expectedValueDictionary)
|
||||
{
|
||||
AssertAreEqual(kvp.Key, resultValueDictionary.Contains(kvp.Key));
|
||||
AssertAreEqual(kvp.Value, resultValueDictionary[kvp.Key]);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (expectedValue is OrderEvent || expectedValue is OrderFee)
|
||||
{
|
||||
AssertAreEqual(expectedValue, resultValue);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.AreEqual(expectedValue, resultValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach (var fieldInfo in expected.GetType().GetFields())
|
||||
{
|
||||
if (fieldInfo.CustomAttributes.Any(data => data.AttributeType == typeof(ProtoMemberAttribute)))
|
||||
{
|
||||
Assert.AreEqual(fieldInfo.GetValue(expected), fieldInfo.GetValue(result));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user