chore: import upstream snapshot with attribution

2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
@@ -0,0 +1,133 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ *
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.Linq;
+using QuantConnect.Data;
+using System.Diagnostics;
+using QuantConnect.Logging;
+using System.Collections.Generic;
+using ZipFile = Ionic.Zip.ZipFile;
+
+namespace QuantConnect.ToolBox.KaikoDataConverter
+{
+    /// <summary>
+    /// Console application for converting a single day of Kaiko data into Lean data format for high resolutions (tick, second and minute)
+    /// </summary>
+    public static class KaikoDataConverterProgram
+    {
+        /// <summary>
+        /// Kaiko data converter entry point.
+        /// </summary>
+        /// <param name="sourceDirectory">The source directory where all Kaiko zipped files are stored..</param>
+        /// <param name="date">The date to process.</param>
+        /// <param name="exchange">The exchange to process, if not defined, all exchanges will be processed.</param>
+        /// <exception cref="ArgumentException">Source folder does not exists.</exception>
+        /// <remarks>This converter will process automatically data for every exchange and for both tick types if the raw data files are available in the sourceDirectory</remarks>
+        public static void KaikoDataConverter(string sourceDirectory, string date, string exchange = "")
+        {
+            var timer = new Stopwatch();
+            timer.Start();
+            var folderPath = new DirectoryInfo(sourceDirectory);
+            if (!folderPath.Exists)
+            {
+                throw new ArgumentException($"Source folder {folderPath.FullName} not found");
+            }
+
+            exchange = !string.IsNullOrEmpty(exchange) && exchange.ToLowerInvariant() == "gdax" ? "coinbase" : exchange;
+
+            var processingDate = Parse.DateTimeExact(date, DateFormat.EightCharacter);
+            foreach (var filePath in folderPath.EnumerateFiles("*.zip"))
+            {
+                // Do not process exchanges other than the one defined.
+                if (!string.IsNullOrEmpty(exchange) && !filePath.Name.ToLowerInvariant().Contains(exchange.ToLowerInvariant())) continue;
+
+                Log.Trace($"KaikoDataConverter(): Starting data conversion from source {filePath.Name} for date {processingDate:yyyy_MM_dd}... ");
+                using (var zip = new ZipFile(filePath.FullName))
+                {
+                    var targetDayEntries = zip.Entries.Where(e => e.FileName.Contains($"{processingDate.ToStringInvariant("yyyy_MM_dd")}")).ToList();
+
+                    if (!targetDayEntries.Any())
+                    {
+                        Log.Error($"KaikoDataConverter(): Date {processingDate:yyyy_MM_dd} not found in source file {filePath.FullName}.");
+                    }
+
+                    foreach (var zipEntry in targetDayEntries)
+                    {
+                        var nameParts = zipEntry.FileName.Split(new char[] { '/' }).Last().Split(new char[] { '_' });
+                        var market = nameParts[0] == "Coinbase" ? "GDAX" : nameParts[0];
+                        var ticker = nameParts[1];
+                        var tickType = nameParts[2] == "trades" ? TickType.Trade : TickType.Quote;
+                        var symbol = Symbol.Create(ticker, SecurityType.Crypto, market);
+
+                        Log.Trace($"KaikoDataConverter(): Processing {symbol.Value} {tickType}");
+
+                        // Generate ticks from raw data and write them to disk
+
+                        var reader = new KaikoDataReader(symbol, tickType);
+                        var ticks = reader.GetTicksFromZipEntry(zipEntry);
+
+                        var writer = new LeanDataWriter(Resolution.Tick, symbol, Globals.DataFolder, tickType);
+                        writer.Write(ticks);
+
+                        try
+                        {
+                            Log.Trace($"KaikoDataConverter(): Starting consolidation for {symbol.Value} {tickType}");
+                            List<TickAggregator> consolidators = new List<TickAggregator>();
+
+                            if (tickType == TickType.Trade)
+                            {
+                                consolidators.AddRange(new[]
+                                {
+                                    new TradeTickAggregator(Resolution.Second),
+                                    new TradeTickAggregator(Resolution.Minute),
+                                });
+                            }
+                            else
+                            {
+                                consolidators.AddRange(new[]
+                                {
+                                    new QuoteTickAggregator(Resolution.Second),
+                                    new QuoteTickAggregator(Resolution.Minute),
+                                });
+                            }
+
+                            foreach (var tick in ticks)
+                            {
+                                foreach (var consolidator in consolidators)
+                                {
+                                    consolidator.Update(tick);
+                                }
+                            }
+
+                            foreach (var consolidator in consolidators)
+                            {
+                                writer = new LeanDataWriter(consolidator.Resolution, symbol, Globals.DataFolder, tickType);
+                                writer.Write(consolidator.Flush());
+                            }
+                        }
+                        catch (Exception e)
+                        {
+                            Log.Error($"KaikoDataConverter(): Error processing entry {zipEntry.FileName}. Exception {e}");
+                        }
+                    }
+                }
+            }
+            Log.Trace($"KaikoDataConverter(): Finished in {timer.Elapsed}");
+        }
+    }
+}