chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
*
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Logging;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using ZipEntry = Ionic.Zip.ZipEntry;
|
||||
|
||||
namespace QuantConnect.ToolBox.KaikoDataConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Decompress single entry from Kaiko crypto raw data.
|
||||
/// </summary>
|
||||
public class KaikoDataReader
|
||||
{
|
||||
private Symbol _symbol;
|
||||
private TickType _tickType;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="KaikoDataReader"/> class.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol.</param>
|
||||
/// <param name="tickType">Type of the tick.</param>
|
||||
public KaikoDataReader(Symbol symbol, TickType tickType)
|
||||
{
|
||||
_symbol = symbol;
|
||||
_tickType = tickType;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the ticks from Kaiko file zip entry.
|
||||
/// </summary>
|
||||
/// <param name="zipEntry">The zip entry.</param>
|
||||
/// <returns></returns>
|
||||
public IEnumerable<BaseData> GetTicksFromZipEntry(ZipEntry zipEntry)
|
||||
{
|
||||
var rawData = GetRawDataStreamFromEntry(zipEntry);
|
||||
return _tickType == TickType.Trade ? ParseKaikoTradeFile(rawData) : ParseKaikoQuoteFile(rawData);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the raw data from entry.
|
||||
/// </summary>
|
||||
/// <param name="zipEntry">The zip entry.</param>
|
||||
/// <returns>IEnumerable with the zip entry content.</returns>
|
||||
private IEnumerable<string> GetRawDataStreamFromEntry(ZipEntry zipEntry)
|
||||
{
|
||||
using (var outerStream = new StreamReader(zipEntry.OpenReader()))
|
||||
using (var innerStream = new GZipStream(outerStream.BaseStream, CompressionMode.Decompress))
|
||||
using (var outputStream = new StreamReader(innerStream))
|
||||
{
|
||||
string line;
|
||||
while ((line = outputStream.ReadLine()) != null)
|
||||
{
|
||||
yield return line;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse order book information for Kaiko data files
|
||||
/// </summary>
|
||||
/// <param name="rawDataLines">The raw data lines.</param>
|
||||
/// <returns>
|
||||
/// IEnumerable of ticks representing the Kaiko data
|
||||
/// </returns>
|
||||
private IEnumerable<Tick> ParseKaikoQuoteFile(IEnumerable<string> rawDataLines)
|
||||
{
|
||||
var headerLine = rawDataLines.First();
|
||||
var headerCsv = headerLine.ToCsv();
|
||||
var typeColumn = headerCsv.FindIndex(x => x == "type");
|
||||
var dateColumn = headerCsv.FindIndex(x => x == "date");
|
||||
var priceColumn = headerCsv.FindIndex(x => x == "price");
|
||||
var quantityColumn = headerCsv.FindIndex(x => x == "amount");
|
||||
|
||||
long currentEpoch = 0;
|
||||
var currentEpochTicks = new List<KaikoTick>();
|
||||
|
||||
foreach (var line in rawDataLines.Skip(1))
|
||||
{
|
||||
if (line == null || string.IsNullOrEmpty(line)) continue;
|
||||
|
||||
var lineParts = line.Split(',');
|
||||
|
||||
var tickEpoch = Parse.Long(lineParts[dateColumn]);
|
||||
|
||||
decimal quantity;
|
||||
decimal price;
|
||||
|
||||
try
|
||||
{
|
||||
quantity = ParseScientificNotationToDecimal(lineParts, quantityColumn);
|
||||
price = ParseScientificNotationToDecimal(lineParts, priceColumn);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error($"KaikoDataConverter.ParseKaikoQuoteFile(): Raw data corrupted. Line {string.Join(" ", lineParts)}, Exception {ex}");
|
||||
continue;
|
||||
}
|
||||
|
||||
var currentTick = new KaikoTick
|
||||
{
|
||||
TickType = TickType.Quote,
|
||||
Time = Time.UnixMillisecondTimeStampToDateTime(tickEpoch),
|
||||
Quantity = quantity,
|
||||
Value = price,
|
||||
OrderDirection = lineParts[typeColumn]
|
||||
};
|
||||
|
||||
if (currentEpoch != tickEpoch)
|
||||
{
|
||||
var quoteTick = CreateQuoteTick(Time.UnixMillisecondTimeStampToDateTime(currentEpoch), currentEpochTicks);
|
||||
|
||||
if (quoteTick != null) yield return quoteTick;
|
||||
|
||||
currentEpochTicks.Clear();
|
||||
currentEpoch = tickEpoch;
|
||||
}
|
||||
|
||||
currentEpochTicks.Add(currentTick);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Take a minute snapshot of order book information and make a single Lean quote tick
|
||||
/// </summary>
|
||||
/// <param name="date">The data being processed</param>
|
||||
/// <param name="currentEpcohTicks">The snapshot of bid/ask Kaiko data</param>
|
||||
/// <returns>A single Lean quote tick</returns>
|
||||
private Tick CreateQuoteTick(DateTime date, List<KaikoTick> currentEpcohTicks)
|
||||
{
|
||||
// lowest ask
|
||||
var bestAsk = currentEpcohTicks.Where(x => x.OrderDirection == "a")
|
||||
.OrderBy(x => x.Value)
|
||||
.FirstOrDefault();
|
||||
|
||||
// highest bid
|
||||
var bestBid = currentEpcohTicks.Where(x => x.OrderDirection == "b")
|
||||
.OrderByDescending(x => x.Value)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (bestAsk == null && bestBid == null)
|
||||
{
|
||||
// Did not have enough data to create a tick
|
||||
return null;
|
||||
}
|
||||
|
||||
var tick = new Tick()
|
||||
{
|
||||
Symbol = _symbol,
|
||||
Time = date,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
|
||||
if (bestBid != null)
|
||||
{
|
||||
tick.BidPrice = bestBid.Price;
|
||||
tick.BidSize = bestBid.Quantity;
|
||||
}
|
||||
|
||||
if (bestAsk != null)
|
||||
{
|
||||
tick.AskPrice = bestAsk.Price;
|
||||
tick.AskSize = bestAsk.Quantity;
|
||||
}
|
||||
|
||||
return tick;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse a kaiko trade file
|
||||
/// </summary>
|
||||
/// <param name="unzippedFile">The path to the unzipped file</param>
|
||||
/// <returns>Lean Ticks in the Kaiko file</returns>
|
||||
private IEnumerable<Tick> ParseKaikoTradeFile(IEnumerable<string> rawDataLines)
|
||||
{
|
||||
var headerLine = rawDataLines.First();
|
||||
var headerCsv = headerLine.ToCsv();
|
||||
var dateColumn = headerCsv.FindIndex(x => x == "date");
|
||||
var priceColumn = headerCsv.FindIndex(x => x == "price");
|
||||
var quantityColumn = headerCsv.FindIndex(x => x == "amount");
|
||||
|
||||
foreach (var line in rawDataLines.Skip(1))
|
||||
{
|
||||
if (line == null || string.IsNullOrEmpty(line)) continue;
|
||||
|
||||
var lineParts = line.Split(',');
|
||||
|
||||
decimal quantity;
|
||||
decimal price;
|
||||
|
||||
try
|
||||
{
|
||||
quantity = ParseScientificNotationToDecimal(lineParts, quantityColumn);
|
||||
price = ParseScientificNotationToDecimal(lineParts, priceColumn);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error($"KaikoDataConverter.ParseKaikoTradeFile(): Raw data corrupted. Line {string.Join(" ", lineParts)}, Exception {ex}");
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return new Tick
|
||||
{
|
||||
Symbol = _symbol,
|
||||
TickType = TickType.Trade,
|
||||
Time = Time.UnixMillisecondTimeStampToDateTime(Parse.Long(lineParts[dateColumn])),
|
||||
Quantity = quantity,
|
||||
Value = price
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse the quantity field of the kaiko ticks - can sometimes be expressed in scientific notation
|
||||
/// </summary>
|
||||
/// <param name="lineParts">The line from the Kaiko file</param>
|
||||
/// <param name="column">The index of the quantity column </param>
|
||||
/// <returns>The quantity as a decimal</returns>
|
||||
private static decimal ParseScientificNotationToDecimal(string[] lineParts, int column)
|
||||
{
|
||||
var value = lineParts[column];
|
||||
if (value.Contains('e', StringComparison.InvariantCulture))
|
||||
{
|
||||
return Parse.Decimal(value, NumberStyles.Float);
|
||||
}
|
||||
|
||||
return lineParts[column].ConvertInvariant<decimal>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Simple class to add order direction to Tick
|
||||
/// used for aggregating Kaiko order book snapshots
|
||||
/// </summary>
|
||||
private class KaikoTick : Tick
|
||||
{
|
||||
public string OrderDirection { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
*
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using QuantConnect.Data;
|
||||
using System.Diagnostics;
|
||||
using QuantConnect.Logging;
|
||||
using System.Collections.Generic;
|
||||
using ZipFile = Ionic.Zip.ZipFile;
|
||||
|
||||
namespace QuantConnect.ToolBox.KaikoDataConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Console application for converting a single day of Kaiko data into Lean data format for high resolutions (tick, second and minute)
|
||||
/// </summary>
|
||||
public static class KaikoDataConverterProgram
|
||||
{
|
||||
/// <summary>
|
||||
/// Kaiko data converter entry point.
|
||||
/// </summary>
|
||||
/// <param name="sourceDirectory">The source directory where all Kaiko zipped files are stored..</param>
|
||||
/// <param name="date">The date to process.</param>
|
||||
/// <param name="exchange">The exchange to process, if not defined, all exchanges will be processed.</param>
|
||||
/// <exception cref="ArgumentException">Source folder does not exists.</exception>
|
||||
/// <remarks>This converter will process automatically data for every exchange and for both tick types if the raw data files are available in the sourceDirectory</remarks>
|
||||
public static void KaikoDataConverter(string sourceDirectory, string date, string exchange = "")
|
||||
{
|
||||
var timer = new Stopwatch();
|
||||
timer.Start();
|
||||
var folderPath = new DirectoryInfo(sourceDirectory);
|
||||
if (!folderPath.Exists)
|
||||
{
|
||||
throw new ArgumentException($"Source folder {folderPath.FullName} not found");
|
||||
}
|
||||
|
||||
exchange = !string.IsNullOrEmpty(exchange) && exchange.ToLowerInvariant() == "gdax" ? "coinbase" : exchange;
|
||||
|
||||
var processingDate = Parse.DateTimeExact(date, DateFormat.EightCharacter);
|
||||
foreach (var filePath in folderPath.EnumerateFiles("*.zip"))
|
||||
{
|
||||
// Do not process exchanges other than the one defined.
|
||||
if (!string.IsNullOrEmpty(exchange) && !filePath.Name.ToLowerInvariant().Contains(exchange.ToLowerInvariant())) continue;
|
||||
|
||||
Log.Trace($"KaikoDataConverter(): Starting data conversion from source {filePath.Name} for date {processingDate:yyyy_MM_dd}... ");
|
||||
using (var zip = new ZipFile(filePath.FullName))
|
||||
{
|
||||
var targetDayEntries = zip.Entries.Where(e => e.FileName.Contains($"{processingDate.ToStringInvariant("yyyy_MM_dd")}")).ToList();
|
||||
|
||||
if (!targetDayEntries.Any())
|
||||
{
|
||||
Log.Error($"KaikoDataConverter(): Date {processingDate:yyyy_MM_dd} not found in source file {filePath.FullName}.");
|
||||
}
|
||||
|
||||
foreach (var zipEntry in targetDayEntries)
|
||||
{
|
||||
var nameParts = zipEntry.FileName.Split(new char[] { '/' }).Last().Split(new char[] { '_' });
|
||||
var market = nameParts[0] == "Coinbase" ? "GDAX" : nameParts[0];
|
||||
var ticker = nameParts[1];
|
||||
var tickType = nameParts[2] == "trades" ? TickType.Trade : TickType.Quote;
|
||||
var symbol = Symbol.Create(ticker, SecurityType.Crypto, market);
|
||||
|
||||
Log.Trace($"KaikoDataConverter(): Processing {symbol.Value} {tickType}");
|
||||
|
||||
// Generate ticks from raw data and write them to disk
|
||||
|
||||
var reader = new KaikoDataReader(symbol, tickType);
|
||||
var ticks = reader.GetTicksFromZipEntry(zipEntry);
|
||||
|
||||
var writer = new LeanDataWriter(Resolution.Tick, symbol, Globals.DataFolder, tickType);
|
||||
writer.Write(ticks);
|
||||
|
||||
try
|
||||
{
|
||||
Log.Trace($"KaikoDataConverter(): Starting consolidation for {symbol.Value} {tickType}");
|
||||
List<TickAggregator> consolidators = new List<TickAggregator>();
|
||||
|
||||
if (tickType == TickType.Trade)
|
||||
{
|
||||
consolidators.AddRange(new[]
|
||||
{
|
||||
new TradeTickAggregator(Resolution.Second),
|
||||
new TradeTickAggregator(Resolution.Minute),
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
consolidators.AddRange(new[]
|
||||
{
|
||||
new QuoteTickAggregator(Resolution.Second),
|
||||
new QuoteTickAggregator(Resolution.Minute),
|
||||
});
|
||||
}
|
||||
|
||||
foreach (var tick in ticks)
|
||||
{
|
||||
foreach (var consolidator in consolidators)
|
||||
{
|
||||
consolidator.Update(tick);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var consolidator in consolidators)
|
||||
{
|
||||
writer = new LeanDataWriter(consolidator.Resolution, symbol, Globals.DataFolder, tickType);
|
||||
writer.Write(consolidator.Flush());
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Error($"KaikoDataConverter(): Error processing entry {zipEntry.FileName}. Exception {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Log.Trace($"KaikoDataConverter(): Finished in {timer.Elapsed}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user