chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,264 @@
|
||||
Globex,Product Name,MultipleFactor,Info
|
||||
10Y,Micro 10-Year Yield Futures,0.1,
|
||||
1S,Propane Non-LDH Mont Belvieu (OPIS) BALMO Futures,1,
|
||||
22,Argus Propane Far East Index BALMO Futures,1,
|
||||
2YY,Micro 2-Year Yield Futures,0.1,
|
||||
30Y,Micro 30-Year Yield Futures,0.1,
|
||||
5YY,Micro 5-Year Yield Futures,0.1
|
||||
6A,Australian Dollar Futures,0.01,
|
||||
6B,British Pound Futures,0.01,
|
||||
6C,Canadian Dollar Futures,0.01,
|
||||
6E,Euro FX Futures,0.01,
|
||||
6J,Japanese Yen Futures,0.0001,
|
||||
6L,Brazilian Real Futures,0.01,
|
||||
6M,Mexican Peso Futures,0.0001,
|
||||
6N,New Zealand Dollar Futures,0.01,
|
||||
6R,Russian Ruble Futures,0.0001,
|
||||
6S,Swiss Franc Futures,0.01,
|
||||
6Z,South African Rand Futures,0.0001,
|
||||
A0D,Mini European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1,
|
||||
A0F,Mini Singapore Fuel Oil 180 cst (Platts) Futures,0.1,
|
||||
A1L,Gulf Coast ULSD (Platts) Up-Down BALMO Futures,1,
|
||||
A1M,Gulf Coast Jet (Platts) Up-Down BALMO Futures,1,
|
||||
A1R,Propane Non-LDH Mont Belvieu (OPIS) Futures,0.001,
|
||||
A32,European Propane CIF ARA (Argus) BALMO Futures,1,
|
||||
A3G,Premium Unleaded Gasoline 10 ppm FOB MED (Platts) Futures,0.1,
|
||||
A7E,Argus Propane Far East Index Futures,0.1,
|
||||
A7I,Gasoline Euro-bob Oxy NWE Barges (Argus) Crack Spread BALMO Futures,1,
|
||||
A7Q,Mont Belvieu Natural Gasoline (OPIS) Futures,0.001,
|
||||
A8J,Mont Belvieu Normal Butane (OPIS) BALMO Futures,1,
|
||||
A8K,Conway Propane (OPIS) Futures,0.001,
|
||||
A8O,Mont Belvieu LDH Propane (OPIS) BALMO Futures,1,
|
||||
A91,Argus Propane Far East Index vs. European Propane CIF ARA (Argus) Futures,0.1,
|
||||
A9N,Argus Propane (Saudi Aramco) Futures,0.1,
|
||||
AA6,Group Three ULSD (Platts) vs. NY Harbor ULSD Futures,0.01,
|
||||
AA8,Group Three Sub-octane Gasoline (Platts) vs. RBOB Futures,0.01,
|
||||
ABS,Singapore Fuel Oil 180 cst (Platts) BALMO Futures,1,
|
||||
ABT,Singapore Fuel Oil 380 cst (Platts) BALMO Futures,1,
|
||||
AC0,Mont Belvieu Ethane (OPIS) Futures,0.001,
|
||||
ACD,Australian Dollar/Canadian Dollar Futures,0.01,
|
||||
AD0,Mont Belvieu Normal Butane (OPIS) Futures,0.001,
|
||||
ADB,Brent Crude Oil vs. Dubai Crude Oil (Platts) Futures,0.1,
|
||||
AE5,Argus LLS vs. WTI (Argus) Trade Month Futures,1,
|
||||
AGA,Singapore Gasoil (Platts) vs. Low Sulphur Gasoil Futures,0.1,
|
||||
AJL,Los Angeles CARBOB Gasoline (OPIS) vs. RBOB Gasoline Futures,1,
|
||||
AJS,Los Angeles Jet (OPIS) vs. NY Harbor ULSD Futures,0.01,
|
||||
AJY,Australian Dollar/Japanese Yen Futures,1,
|
||||
AKL,Los Angeles CARB Diesel (OPIS) vs. NY Harbor ULSD Futures,0.01,
|
||||
AKZ,European Naphtha (Platts) BALMO Futures,1,
|
||||
ANE,Australian Dollar/New Zealand Dollar Futures,0.01,
|
||||
APS,European Propane CIF ARA (Argus) Futures,1,
|
||||
AR0,Mont Belvieu Natural Gasoline (OPIS) BALMO Futures,1,
|
||||
ARE,RBOB Gasoline Crack Spread Futures,1,
|
||||
AUP,Aluminum MW U.S. Transaction Premium Platts (25MT) Futures,0.001,
|
||||
AVZ,Gulf Coast HSFO (Platts) BALMO Futures,1,
|
||||
AW,Bloomberg Commodity Index Futures,10,
|
||||
AYV,Mars (Argus) vs. WTI Trade Month Futures,1,
|
||||
AYX,Mars (Argus) vs. WTI Financial Futures,1,
|
||||
AZ1,Ethanol T2 FOB Rdam Including Duty (Platts) Futures,0.1,
|
||||
B0,Mont Belvieu LDH Propane (OPIS) Futures,0.001,
|
||||
B7H,Gasoline Euro-bob Oxy NWE Barges (Argus) Futures,0.1,
|
||||
BCF,Black Sea Corn Financially Settled (Platts) Futures,1,
|
||||
BIO,E-mini Nasdaq-100 Biotechnology Index Futures,1,
|
||||
BK,WTI-Brent Financial Futures,1,
|
||||
BOO,3.5% Fuel Oil Barges FOB Rdam (Platts) Crack Spread (1000mt) Futures,0.1,
|
||||
BR7,Gasoline Euro-bob Oxy NWE Barges (Argus) BALMO Futures,1,
|
||||
BTC,Bitcoin Futures,100,
|
||||
BWF,Black Sea Wheat Financially Settled (Platts) Futures,1,
|
||||
BZ,Brent Last Day Financial Futures,1,
|
||||
CB,Cash-settled Butter Futures,0.001,
|
||||
CJY,Canadian Dollar/Japanese Yen Futures,1,
|
||||
CL,Crude Oil Futures,1,
|
||||
CNH,Standard-Size USD/Offshore RMB (CNH) Futures,0.01,
|
||||
CRB,Gulf Coast CBOB Gasoline A2 (Platts) vs. RBOB Gasoline Futures,0.01,
|
||||
CSC,Cash-Settled Cheese Futures,0.1,
|
||||
CSW,Clearbrook Bakken Sweet Crude Oil Monthly Index (Net Energy) Futures,0.1,
|
||||
CSX,WTI Financial Futures,1,
|
||||
CU,Chicago Ethanol (Platts) Futures,0.001,
|
||||
D1N,Singapore Mogas 92 Unleaded (Platts) Brent Crack Spread Futures,0.1,
|
||||
DC,Class III Milk Futures,1,
|
||||
DCB,Dubai Crude Oil (Platts) Financial Futures,0.1,
|
||||
DY,Dry Whey Futures,0.001,
|
||||
E6,Japan C&F Naphtha (Platts) BALMO Futures,1,
|
||||
E7,E-mini Euro FX Futures,0.001,
|
||||
EAD,Euro/Australian Dollar Futures,0.01,
|
||||
ECD,Euro/Canadian Dollar Futures,0.01,
|
||||
EDP,Aluminium European Premium Duty-Paid (Metal Bulletin) Futures,1,
|
||||
EH,Ethanol Futures,0.1,
|
||||
EI,E-mini FTSE Emerging Index Futures,1,
|
||||
EMD,E-mini S&P MidCap 400 Futures,1,
|
||||
EN,European Naphtha (Platts) Crack Spread Futures,0.001,
|
||||
EPN,European Propane CIF ARA (Argus) vs. Naphtha Cargoes CIF NWE (Platts) Futures,0.1,
|
||||
ES,E-mini S&P 500 Futures,1,
|
||||
ESK,Euro/Swedish Krona Futures,0.01,
|
||||
ETH,Ether Futures,1,
|
||||
EVC,Singapore Fuel Oil 380 cst (Platts) vs. European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,1,
|
||||
EWG,East-West Gasoline Spread (Platts-Argus) Futures,0.1,
|
||||
EWN,East-West Naphtha: Japan C&F vs. Cargoes CIF NWE Spread (Platts) Futures,0.1,
|
||||
EXR,"RBOB Gasoline vs. Euro-bob Oxy NWE Barges (Argus) (350,000 gallons) Futures",0.01,
|
||||
F1U,5-Year USD MAC Swap Futures,100,
|
||||
FO,3.5% Fuel Oil Barges FOB Rdam (Platts) Crack Spread Futures,0.1,
|
||||
FRC,Freight Route TC14 (Baltic) Futures,1,
|
||||
FSS,1% Fuel Oil Cargoes FOB NWE (Platts) vs. 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1,
|
||||
GC,Gold Futures,10,
|
||||
GCU,Gulf Coast HSFO (Platts) vs. European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1,
|
||||
GD,S&P-GSCI Commodity Index Futures,1,
|
||||
GDK,Class IV Milk Futures,1,
|
||||
GE,Eurodollar Futures,1,
|
||||
GF,Feeder Cattle Futures,0.001,
|
||||
GNF,Non-fat Dry Milk Futures,0.001,
|
||||
HCL,WTI Houston Crude Oil Futures,1,
|
||||
HE,Lean Hog Futures,0.001,
|
||||
HG,Copper Futures,0.01,
|
||||
HH,Natural Gas (Henry Hub) Last-day Financial Futures,0.1,
|
||||
HO,NY Harbor ULSD Futures,0.01,
|
||||
HP,Natural Gas (Henry Hub) Penultimate Financial Futures,0.1,
|
||||
HRC,U.S. Midwest Domestic Hot-Rolled Coil Steel (CRU) Index Futures,1,
|
||||
HTT,WTI Houston (Argus) vs. WTI Trade Month Futures,1,
|
||||
IBV,USD-Denominated Ibovespa Index Futures,100,
|
||||
J7,E-mini Japanese Yen Futures,0.00001,
|
||||
JA,Japan C&F Naphtha (Platts) Futures,0.1,
|
||||
JET,NY Buckeye Jet Fuel (Platts) vs. NY Harbor ULSD Futures,0.01,
|
||||
JTB,NY Buckeye Jet Fuel (Platts) vs. NY Harbor ULSD BALMO Futures,1,
|
||||
KE,KC HRW Wheat Futures,1,
|
||||
KRW,Korean Won Futures,0.00001,
|
||||
LBR,Lumber Futures,1,
|
||||
LBS,Random Length Lumber Futures,1,
|
||||
LE,Live Cattle Futures,0.001,
|
||||
LIB,7-year Eris Swap Futures,1,
|
||||
LID,4-year Eris Swap Futures,1,
|
||||
LIE,30-year Eris Swap Futures,1,
|
||||
LIT,2-Year Eris Swap Futures,1,
|
||||
LIW,5-year Eris Swap Futures,1,
|
||||
LIY,10-year Eris Swap Futures,1,
|
||||
LT,Gulf Coast ULSD (Platts) Up-Down Futures,0.01,
|
||||
M1B,Micro Gasoil 0.1% Barges FOB ARA (Platts) Futures,1,
|
||||
M2K,Micro E-mini Russell 2000 Index Futures,1,
|
||||
M35,Micro European 3.5% Fuel Oil Cargoes FOB Med (Platts) Futures,0.1,
|
||||
M5F,Micro Coal (API 5) fob Newcastle (Argus/McCloskey) Futures,1,
|
||||
M6A,E-micro Australian Dollar/American Dollar Futures,100,
|
||||
M6B,E-micro British Pound/American Dollar Futures,100,
|
||||
M6C,Micro USD/CAD Futures,100,
|
||||
M6E,E-micro Euro/American Dollar Futures,100,
|
||||
M6J,Micro USD/JPY Futures,100,
|
||||
M6S,Micro USD/CHF Futures,1000,
|
||||
MAE,Mini Argus Propane Far East Index Futures,0.1,
|
||||
MAF,Micro Singapore Fuel Oil 380CST (Platts) Futures,1,
|
||||
MBT,Micro Bitcoin Futures,100,
|
||||
MCD,E-micro Canadian Dollar/American Dollar Futures,100,
|
||||
MCL,Micro WTI Crude Oil Futures,1,
|
||||
ME,Gulf Coast Jet (Platts) Up-Down Futures,0.01,
|
||||
MEF,Micro European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1,
|
||||
MEE,Mini European Naphtha (Platts) BALMO Futures,1,
|
||||
MEO,Mini Gasoline Euro-bob Oxy NWE Barges (Argus) Futures,0.1,
|
||||
MES,Micro E-mini Standard and Poor's 500 Stock Price Index Futures,1,
|
||||
MET,Micro Ether Futures,1,
|
||||
MFB,Gulf Coast HSFO (Platts) Futures,1,
|
||||
MFF,Coal (API4) FOB Richards Bay (ARGUS-McCloskey) Futures,1,
|
||||
MGB,Mini Gasoil 0.1 Barges FOB Rdam (Platts) vs. Low Sulphur Gasoil Futures,0.1,
|
||||
MGC,E-micro Gold Futures,10,
|
||||
MGT,Micro Gold TAS,1,
|
||||
MIB,BTIC on Micro Bitcoin Futures,1,
|
||||
MIR,E-micro Indian Rupee/USD Futures,1,
|
||||
MJN,Mini Japan C&F Naphtha (Platts) Futures,0.1,
|
||||
MJY,E-micro Japanese Yen/American Dollar Futures,100,
|
||||
MM,New York Harbor Residual Fuel 1.0% (Platts) Futures,1,
|
||||
MMF,Mini 3.5% Fuel Oil Cargoes FOB MED (Platts) Financial Futures,0.1,
|
||||
MNC,Mini European Naphtha CIF NWE (Platts) Futures,0.1,
|
||||
MNH,Micro USD/CNH Futures,100,
|
||||
MNQ,Micro E-mini Nasdaq-100 Index Futures,1,
|
||||
MPS,Mini European Propane CIF ARA (Argus) Futures,2,
|
||||
MRB,BTIC on Micro Ether Futures,1,
|
||||
MSF,E-micro Swiss Franc/American Dollar Futures,100,
|
||||
MSG,Mini Singapore Gasoil (Platts) Futures,0.1,
|
||||
MTB,Mini Singapore Fuel Oil 380 cst (Platts) BALMO Futures,3,
|
||||
MTF,Coal (API2) CIF ARA (ARGUS-McCloskey) Futures,1,
|
||||
MTS,Mini Singapore Fuel Oil 380 cst (Platts) Futures,0.1,
|
||||
MYM,Micro E-mini Dow Jones Industrial Average Index Futures,100,
|
||||
N1B,Singapore Mogas 92 Unleaded (Platts) Futures,4,
|
||||
N1U,10-Year USD MAC Swap Futures,5,
|
||||
NBB,Naphtha Cargoes CIF NWE (Platts) Crack Spread (1000mt) BALMO Futures,6,
|
||||
NG,Henry Hub Natural Gas Futures,0.1,
|
||||
NIY,Nikkei/Yen Futures,100,
|
||||
NKD,Nikkei/USD Futures,100,
|
||||
NN,Henry Hub Natural Gas Last Day Financial Futures,0.1,
|
||||
NOK,Norwegian Krone Futures,0.001,
|
||||
NOO,Naphtha Cargoes CIF NWE (Platts) Crack Spread (1000mt) Futures,0.01,
|
||||
NQ,E-mini Nasdaq-100 Futures,1,
|
||||
PA,Palladium Futures,1,
|
||||
PAM,Micro Palladium Futures,1,
|
||||
PJY,British Pound/Japanese Yen Futures,1,
|
||||
PL,Platinum Futures,10,
|
||||
PLN,Polish Zloty Futures,0.001,
|
||||
PSF,British Pound/Swiss Franc Futures,0.01,
|
||||
QC,E-mini Copper Futures,0.01,
|
||||
QG,E-mini Natural Gas Futures,0.1,
|
||||
QI,E-mini Silver Futures,0.01,
|
||||
QM,E-mini Crude Oil Futures,0.1,
|
||||
QO,E-mini Gold Futures,1,
|
||||
R5O,Micro European FOB Rdam Marine Fuel 0.5% Barges (Platts) Futures,1,
|
||||
RB,RBOB Gasoline Futures,0.01,
|
||||
RBB,RBOB Gasoline Brent Crack Spread Futures,0.1,
|
||||
RF,Euro/Swiss Franc Futures,0.01,
|
||||
RP,Euro/British Pound Futures,0.001,
|
||||
RS1,E-mini Russell 1000 Index Futures,1,
|
||||
RSG,E-mini Russell 1000 Growth Index Futures,1,
|
||||
RSV,E-mini Russell 1000 Value Index Futures,1,
|
||||
RTY,E-mini Russell 2000 Index Futures,1,
|
||||
RVR,Gulf Coast Unl 87 Gasoline M2 (Platts) vs. RBOB Gasoline Futures,0.01,
|
||||
RX,Dow Jones Real Estate Futures,10,
|
||||
RY,Euro/Japanese Yen Futures,1,
|
||||
SDA,S&P 500 Annual Dividend Index Futures,0.1,
|
||||
SE,Singapore Fuel Oil 380 cst (Platts) Futures,0.1,
|
||||
SEK,Swedish Krona Futures,0.001,
|
||||
S5O,Micro Singapore FOB Marine Fuel 0.5% (Platts) Futures,1,
|
||||
SI,Silver Futures,0.1,
|
||||
SIL,1000-oz. Silver Futures,0.1,
|
||||
SIR,Indian Rupee/USD Futures,1,
|
||||
SON,Quarterly IMM SONIA Futures,8,
|
||||
SP,S&P 500 Futures,1,
|
||||
SR1,One-Month SOFR Futures,9,
|
||||
SR3,Three-Month SOFR Futures,10,
|
||||
T7K,Gasoline Euro-bob Oxy NWE Barges (Argus) Crack Spread Futures,0.1,
|
||||
TIO,Iron Ore 62% Fe CFR China (TSI) Futures,1,
|
||||
TL,Freight Route TD3C (Baltic) Futures,11,
|
||||
TM,Freight Route TC2 (Baltic) Futures,1,
|
||||
TN,Ultra 10-Year U.S. Treasury Note Futures,100,
|
||||
TPY,Yen Denominated TOPIX Futures,1,
|
||||
TRI,S&P 500 Total Return Index Futures,1,
|
||||
UA,Singapore Fuel Oil 180 cst (Platts) Futures,0.1,
|
||||
UB,Ultra U.S. Treasury Bond Futures,100,
|
||||
UME,Urea (Granular) FOB Middle East Futures,1,
|
||||
UN,European Naphtha Cargoes CIF NWE (Platts) Futures,0.1,
|
||||
UV,European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1,
|
||||
VX,Cboe Volatility Index (VIX),1,
|
||||
WCW,Western Canadian Select Oil (Net Energy) Monthly Index Futures,1,
|
||||
WTT,WTI Midland (Argus) vs. WTI Trade Month Futures,1,
|
||||
XAB,E-mini Materials Select Sector Futures,1,
|
||||
XAE,E-mini Energy Select Sector Futures,1,
|
||||
XAF,E-mini Financial Select Sector Futures,1,
|
||||
XAI,E-mini Industrial Select Sector Futures,1,
|
||||
XAK,E-mini Technology Select Sector Futures,1,
|
||||
XAP,E-mini Consumer Staples Select Sector Futures,1,
|
||||
XAR,E-mini Real Estate Select Sector Futures,1,
|
||||
XAU,E-mini Utilities Select Sector Futures,1,
|
||||
XAV,E-mini Health Care Select Sector Futures,1,
|
||||
XAY,E-mini Consumer Discretionary Select Sector Futures,1,
|
||||
XAZ,E-mini Communication Services Select Sector Futures,1,
|
||||
XC,Mini-Corn Futures,100,
|
||||
XK,Mini Soybean Futures,100,
|
||||
XW,Mini-sized Chicago SRW Wheat Futures,100,
|
||||
YM,E-mini Dow ($5) Futures,100,
|
||||
YO,Sugar # 11 CME Globex Futures,0.01,
|
||||
ZB,U.S. Treasury Bond Futures,100,
|
||||
ZC,Corn Futures,1,
|
||||
ZF,5-Year T-Note Futures,100,
|
||||
ZL,Soybean Oil Futures,0.01,
|
||||
ZM,Soybean Meal Futures,10,
|
||||
ZN,10-Year T-Note Futures,100,
|
||||
ZO,Oats Futures,1,
|
||||
ZQ,30 Day Federal Funds Futures,1,
|
||||
ZR,Rough Rice Futures,0.1,
|
||||
ZS,Soybean Futures,1,
|
||||
ZT,2-Year T-Note Futures,100,
|
||||
ZW,Chicago SRW Wheat Futures,1,
|
||||
|
@@ -0,0 +1,305 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Logging;
|
||||
|
||||
namespace QuantConnect.ToolBox.AlgoSeekFuturesConverter
|
||||
{
|
||||
using Processors = Dictionary<Symbol, List<List<AlgoSeekFuturesProcessor>>>;
|
||||
|
||||
/// <summary>
|
||||
/// Process a directory of algoseek futures files into separate resolutions.
|
||||
/// </summary>
|
||||
public class AlgoSeekFuturesConverter
|
||||
{
|
||||
private readonly DirectoryInfo _source;
|
||||
private readonly DirectoryInfo _remote;
|
||||
private readonly string _destination;
|
||||
private readonly List<Resolution> _resolutions;
|
||||
private readonly DateTime _referenceDate;
|
||||
private readonly HashSet<string> _symbolFilter;
|
||||
|
||||
/// <summary>
|
||||
/// Create a new instance of the AlgoSeekFutures Converter. Parse a single input directory into an output.
|
||||
/// </summary>
|
||||
/// <param name="resolutions">Convert this resolution</param>
|
||||
/// <param name="referenceDate">Datetime to be added to the milliseconds since midnight. Algoseek data is stored in channel files (XX.bz2) and in a source directory</param>
|
||||
/// <param name="remote">Remote directory of the .bz algoseek files</param>
|
||||
/// <param name="source">Source directory of the .csv algoseek files</param>
|
||||
/// <param name="destination">Destination directory of the processed future files</param>
|
||||
/// <param name="symbolFilter">Collection of underlying ticker to process.</param>
|
||||
public AlgoSeekFuturesConverter(List<Resolution> resolutions, DateTime referenceDate, string remote, string source, string destination, HashSet<string> symbolFilter = null)
|
||||
{
|
||||
_source = new DirectoryInfo(source);
|
||||
_remote = new DirectoryInfo(remote);
|
||||
_referenceDate = referenceDate;
|
||||
_destination = destination;
|
||||
_resolutions = resolutions;
|
||||
_symbolFilter = symbolFilter;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Give the reference date and source directory, convert the algoseek data into n-resolutions LEAN format.
|
||||
/// </summary>
|
||||
public void Convert()
|
||||
{
|
||||
Log.Trace("AlgoSeekFuturesConverter.Convert(): Copying remote raw data files locally.");
|
||||
//Get the list of available raw files, copy from its remote location to a local folder and then for each file open a separate streamer.
|
||||
|
||||
var files = GetFilesInRawFolder()
|
||||
.Where(f => (f.Extension == ".gz" || f.Extension == ".bz2") && !f.Name.Contains("option"))
|
||||
.Select(remote => remote.CopyTo(Path.Combine(Path.GetTempPath(), remote.Name), true))
|
||||
.ToList();
|
||||
|
||||
Log.Trace("AlgoSeekFuturesConverter.Convert(): Loading {0} AlgoSeekFuturesReader for {1} ", files.Count, _referenceDate);
|
||||
|
||||
//Initialize parameters
|
||||
var totalLinesProcessed = 0L;
|
||||
var totalFiles = files.Count;
|
||||
var totalFilesProcessed = 0;
|
||||
var start = DateTime.MinValue;
|
||||
|
||||
var symbolMultipliers = LoadSymbolMultipliers();
|
||||
|
||||
//Extract each file massively in parallel.
|
||||
Parallel.ForEach(files, file =>
|
||||
{
|
||||
try
|
||||
{
|
||||
Log.Trace("Remote File :" + file);
|
||||
|
||||
var csvFile = Path.Combine(_source.FullName, Path.GetFileNameWithoutExtension(file.Name));
|
||||
|
||||
Log.Trace("Source File :" + csvFile);
|
||||
|
||||
if (!File.Exists(csvFile))
|
||||
{
|
||||
// create the directory first or else 7z will fail
|
||||
var csvFileInfo = new FileInfo(csvFile);
|
||||
Directory.CreateDirectory(csvFileInfo.DirectoryName);
|
||||
|
||||
Log.Trace("AlgoSeekFuturesConverter.Convert(): Extracting " + file);
|
||||
|
||||
// Never time out extracting an archive; they can be pretty big
|
||||
// and take a while to extract depending on the computer running this application
|
||||
Compression.Extract7ZipArchive(file.FullName, _source.FullName, -1);
|
||||
}
|
||||
|
||||
// setting up local processors
|
||||
var processors = new Processors();
|
||||
|
||||
var reader = new AlgoSeekFuturesReader(csvFile, symbolMultipliers, _symbolFilter);
|
||||
if (start == DateTime.MinValue)
|
||||
{
|
||||
start = DateTime.Now;
|
||||
}
|
||||
|
||||
if (reader.Current != null) // reader contains the data
|
||||
{
|
||||
do
|
||||
{
|
||||
var tick = reader.Current as Tick;
|
||||
|
||||
if (tick.Symbol.ID.Symbol == "VX" && (
|
||||
tick.BidPrice >= 998m || tick.AskPrice >= 998m))
|
||||
{
|
||||
// Invalid value for VX futures. Invalid prices in raw data are 998/999
|
||||
continue;
|
||||
}
|
||||
//Add or create the consolidator-flush mechanism for symbol:
|
||||
List<List<AlgoSeekFuturesProcessor>> symbolProcessors;
|
||||
if (!processors.TryGetValue(tick.Symbol, out symbolProcessors))
|
||||
{
|
||||
symbolProcessors = new List<List<AlgoSeekFuturesProcessor>>(3)
|
||||
{
|
||||
{ _resolutions.Select(x => new AlgoSeekFuturesProcessor(tick.Symbol, _referenceDate, TickType.Trade, x, _destination)).ToList() },
|
||||
{ _resolutions.Select(x => new AlgoSeekFuturesProcessor(tick.Symbol, _referenceDate, TickType.Quote, x, _destination)).ToList() },
|
||||
{ _resolutions.Select(x => new AlgoSeekFuturesProcessor(tick.Symbol, _referenceDate, TickType.OpenInterest, x, _destination)).ToList() }
|
||||
};
|
||||
|
||||
processors[tick.Symbol] = symbolProcessors;
|
||||
}
|
||||
|
||||
// Pass current tick into processor: enum 0 = trade; 1 = quote, 2 = oi
|
||||
foreach (var processor in symbolProcessors[(int)tick.TickType])
|
||||
{
|
||||
processor.Process(tick);
|
||||
}
|
||||
|
||||
if (Interlocked.Increment(ref totalLinesProcessed) % 1000000m == 0)
|
||||
{
|
||||
var pro = (double)processors.Values.SelectMany(p => p.SelectMany(x => x)).Count();
|
||||
var symbols = (double)processors.Keys.Count;
|
||||
Log.Trace("AlgoSeekFuturesConverter.Convert(): Processed {0,3}M ticks( {1}k / sec); Memory in use: {2} MB; Total progress: {3}%, Processor per symbol {4}", Math.Round(totalLinesProcessed / 1000000m, 2), Math.Round(totalLinesProcessed / 1000L / (DateTime.Now - start).TotalSeconds), Process.GetCurrentProcess().WorkingSet64 / (1024 * 1024), 100 * totalFilesProcessed / totalFiles, pro / symbols);
|
||||
}
|
||||
|
||||
}
|
||||
while (reader.MoveNext());
|
||||
|
||||
Log.Trace("AlgoSeekFuturesConverter.Convert(): Performing final flush to disk... ");
|
||||
Flush(processors, DateTime.MaxValue, true);
|
||||
}
|
||||
|
||||
processors = null;
|
||||
GC.Collect();
|
||||
GC.WaitForPendingFinalizers();
|
||||
|
||||
Log.Trace("AlgoSeekFuturesConverter.Convert(): Finished processing file: " + file);
|
||||
Interlocked.Increment(ref totalFilesProcessed);
|
||||
}
|
||||
catch(Exception err)
|
||||
{
|
||||
Log.Error("Exception caught! File: {0} Err: {1} Source {2} Stack {3}", file, err.Message, err.Source, err.StackTrace);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the files in raw folder.
|
||||
/// </summary>
|
||||
/// <returns>List of files in source folder</returns>
|
||||
private IEnumerable<FileInfo> GetFilesInRawFolder()
|
||||
{
|
||||
var files = new List<FileInfo>();
|
||||
|
||||
var command = OS.IsLinux ? "ls" : "cmd.exe";
|
||||
var arguments = OS.IsWindows ? "/c dir /b /a-d" : string.Empty;
|
||||
|
||||
var processStartInfo = new ProcessStartInfo(command, arguments)
|
||||
{
|
||||
CreateNoWindow = true,
|
||||
WindowStyle = ProcessWindowStyle.Hidden,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
WorkingDirectory = _remote.FullName
|
||||
};
|
||||
|
||||
using (var process = new Process())
|
||||
{
|
||||
|
||||
process.StartInfo = processStartInfo;
|
||||
process.Start();
|
||||
|
||||
while (!process.StandardOutput.EndOfStream)
|
||||
{
|
||||
var line = process.StandardOutput.ReadLine();
|
||||
if (line != null)
|
||||
{
|
||||
files.Add(new FileInfo(Path.Combine(_remote.FullName, line)));
|
||||
}
|
||||
}
|
||||
process.WaitForExit();
|
||||
}
|
||||
|
||||
return files;
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Private method loads symbol multipliers from algoseek csv file
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
private Dictionary<string, decimal> LoadSymbolMultipliers()
|
||||
{
|
||||
const int columnUnderlying = 0;
|
||||
const int columnMultipleFactor = 2;
|
||||
|
||||
return File.ReadAllLines("AlgoSeekFuturesConverter/AlgoSeek.US.Futures.PriceMultipliers.1.1.csv")
|
||||
.Select(line => line.ToCsvData())
|
||||
// skipping empty fields
|
||||
.Where(line => !string.IsNullOrEmpty(line[columnUnderlying]) &&
|
||||
!string.IsNullOrEmpty(line[columnMultipleFactor]))
|
||||
// skipping header
|
||||
.Skip(1)
|
||||
.ToDictionary(line => line[columnUnderlying],
|
||||
line => line[columnMultipleFactor].ConvertInvariant<decimal>());
|
||||
}
|
||||
|
||||
private void Flush(Processors processors, DateTime time, bool final)
|
||||
{
|
||||
foreach (var symbol in processors.Keys)
|
||||
{
|
||||
processors[symbol].ForEach(p => p.ForEach(x => x.FlushBuffer(time, final)));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compress the queue buffers directly to a zip file. Lightening fast as streaming ram-> compressed zip.
|
||||
/// </summary>
|
||||
public void Package(DateTime date)
|
||||
{
|
||||
var zipper = OS.IsWindows ? "C:/Program Files/7-Zip/7z.exe" : "7z";
|
||||
|
||||
Log.Trace("AlgoSeekFuturesConverter.Package(): Zipping all files ...");
|
||||
|
||||
var destination = Path.Combine(_destination, "future");
|
||||
Directory.CreateDirectory(destination);
|
||||
var dateMask = date.ToStringInvariant(DateFormat.EightCharacter);
|
||||
|
||||
var files =
|
||||
Directory.EnumerateFiles(destination, dateMask + "*.csv", SearchOption.AllDirectories)
|
||||
.GroupBy(x => Directory.GetParent(x).FullName)
|
||||
.ToList();
|
||||
|
||||
// Zip each file massively in parallel
|
||||
Parallel.ForEach(files, file =>
|
||||
//foreach (var file in files)
|
||||
{
|
||||
try
|
||||
{
|
||||
var outputFileName = file.Key + ".zip";
|
||||
|
||||
// Create and open a new ZIP file
|
||||
var filesToCompress = Directory.GetFiles(file.Key, "*.csv", SearchOption.AllDirectories);
|
||||
var zip = ZipFile.Open(outputFileName, ZipArchiveMode.Create);
|
||||
|
||||
foreach (var fileToCompress in filesToCompress)
|
||||
{
|
||||
// Add the entry for each file
|
||||
zip.CreateEntryFromFile(fileToCompress, Path.GetFileName(fileToCompress), CompressionLevel.Optimal);
|
||||
}
|
||||
|
||||
// Dispose of the object when we are done
|
||||
zip.Dispose();
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(file.Key, true);
|
||||
}
|
||||
catch (Exception err)
|
||||
{
|
||||
Log.Error("Directory.Delete returned error: " + err.Message);
|
||||
}
|
||||
}
|
||||
catch (Exception err)
|
||||
{
|
||||
Log.Error("File: {0} Err: {1} Source {2} Stack {3}", file, err.Message, err.Source, err.StackTrace);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using QuantConnect.Data.Consolidators;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.ToolBox.AlgoSeekFuturesConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Processor for caching and consolidating ticks;
|
||||
/// then flushing the ticks in memory to disk when triggered.
|
||||
/// </summary>
|
||||
public class AlgoSeekFuturesProcessor
|
||||
{
|
||||
static private int _curFileCount = 0;
|
||||
private string _zipPath;
|
||||
private string _entryPath;
|
||||
private Symbol _symbol;
|
||||
private TickType _tickType;
|
||||
private Resolution _resolution;
|
||||
private LazyStreamWriter _streamWriter;
|
||||
private string _dataDirectory;
|
||||
private IDataConsolidator _consolidator;
|
||||
private DateTime _referenceDate;
|
||||
private static string[] _windowsRestrictedNames =
|
||||
{
|
||||
"con", "prn", "aux", "nul"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Zip entry name for the futures contract
|
||||
/// </summary>
|
||||
public string EntryPath
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_entryPath == null)
|
||||
{
|
||||
_entryPath = SafeName(LeanData.GenerateZipEntryName(_symbol, _referenceDate, _resolution, _tickType));
|
||||
}
|
||||
return _entryPath;
|
||||
}
|
||||
set { _entryPath = value; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Zip file path for the futures contract collection
|
||||
/// </summary>
|
||||
public string ZipPath
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_zipPath == null)
|
||||
{
|
||||
_zipPath = Path.Combine(_dataDirectory, SafeName(LeanData.GenerateRelativeZipFilePath(Safe(_symbol), _referenceDate, _resolution, _tickType).Replace(".zip", string.Empty))) + ".zip";
|
||||
}
|
||||
return _zipPath;
|
||||
}
|
||||
set { _zipPath = value; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Public access to the processor symbol
|
||||
/// </summary>
|
||||
public Symbol Symbol
|
||||
{
|
||||
get { return _symbol; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Accessor for the final enumerator
|
||||
/// </summary>
|
||||
public Resolution Resolution
|
||||
{
|
||||
get { return _resolution; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Type of this futures processor.
|
||||
/// ASOP's are grouped trade type for file writing.
|
||||
/// </summary>
|
||||
public TickType TickType
|
||||
{
|
||||
get { return _tickType; }
|
||||
set { _tickType = value; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If no data has been consolidated, do not write to disk
|
||||
/// </summary>
|
||||
public bool ShouldWriteToDisk()
|
||||
{
|
||||
return _consolidator.Consolidated != null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a new AlgoSeekFuturesProcessor for enquing consolidated bars and flushing them to disk
|
||||
/// </summary>
|
||||
/// <param name="symbol">Symbol for the processor</param>
|
||||
/// <param name="date">Reference date for the processor</param>
|
||||
/// <param name="tickType">TradeBar or QuoteBar to generate</param>
|
||||
/// <param name="resolution">Resolution to consolidate</param>
|
||||
/// <param name="dataDirectory">Data directory for LEAN</param>
|
||||
public AlgoSeekFuturesProcessor(Symbol symbol, DateTime date, TickType tickType, Resolution resolution, string dataDirectory)
|
||||
{
|
||||
_symbol = Safe(symbol);
|
||||
_tickType = tickType;
|
||||
_referenceDate = date;
|
||||
_resolution = resolution;
|
||||
_dataDirectory = dataDirectory;
|
||||
|
||||
// Setup the consolidator for the requested resolution
|
||||
if (resolution == Resolution.Tick)
|
||||
{
|
||||
_consolidator = new IdentityDataConsolidator<Tick>();
|
||||
}
|
||||
else
|
||||
{
|
||||
switch (tickType)
|
||||
{
|
||||
case TickType.Trade:
|
||||
_consolidator = new TickConsolidator(resolution.ToTimeSpan());
|
||||
break;
|
||||
case TickType.Quote:
|
||||
_consolidator = new TickQuoteBarConsolidator(resolution.ToTimeSpan());
|
||||
break;
|
||||
case TickType.OpenInterest:
|
||||
_consolidator = new OpenInterestConsolidator(resolution.ToTimeSpan());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var path = ZipPath.Replace(".zip", string.Empty);
|
||||
Directory.CreateDirectory(path);
|
||||
|
||||
var file = Path.Combine(path, EntryPath);
|
||||
|
||||
try
|
||||
{
|
||||
_streamWriter = new LazyStreamWriter(file);
|
||||
}
|
||||
catch (Exception err)
|
||||
{
|
||||
// we are unable to open new file - it is already opened due to bug in algoseek data
|
||||
Log.Error("File: {0} Err: {1} Source: {2} Stack: {3}", file, err.Message, err.Source, err.StackTrace);
|
||||
var newRandomizedName = (file + "-" + Math.Abs(file.GetHashCode()).ToStringInvariant()).Replace(".csv", string.Empty) + ".csv";
|
||||
|
||||
// we store the information under different (randomized) name
|
||||
Log.Trace("Changing name from {0} to {1}", file, newRandomizedName);
|
||||
_streamWriter = new LazyStreamWriter(newRandomizedName);
|
||||
}
|
||||
|
||||
// On consolidating the bars put the bar into a queue in memory to be written to disk later.
|
||||
_consolidator.DataConsolidated += (sender, consolidated) =>
|
||||
{
|
||||
_streamWriter.WriteLine(LeanData.GenerateLine(consolidated, SecurityType.Future, Resolution));
|
||||
};
|
||||
|
||||
Interlocked.Add(ref _curFileCount, 1);
|
||||
if (_curFileCount % 1000 == 0)
|
||||
{
|
||||
Log.Trace("Opened more files: {0}", _curFileCount);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process the tick; add to the con
|
||||
/// </summary>
|
||||
/// <param name="data"></param>
|
||||
public void Process(Tick data)
|
||||
{
|
||||
if (data.TickType != _tickType)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_consolidator.Update(data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Write the in memory queues to the disk.
|
||||
/// </summary>
|
||||
/// <param name="frontierTime">Current foremost tick time</param>
|
||||
/// <param name="finalFlush">Indicates is this is the final push to disk at the end of the data</param>
|
||||
public void FlushBuffer(DateTime frontierTime, bool finalFlush)
|
||||
{
|
||||
//Force the consolidation if time has past the bar
|
||||
_consolidator.Scan(frontierTime);
|
||||
|
||||
// If this is the final packet dump it to the queue
|
||||
if (finalFlush)
|
||||
{
|
||||
if (_consolidator.WorkingData != null)
|
||||
{
|
||||
_streamWriter.WriteLine(LeanData.GenerateLine(_consolidator.WorkingData, SecurityType.Future, Resolution));
|
||||
}
|
||||
|
||||
_streamWriter.Flush();
|
||||
_streamWriter.Close();
|
||||
_streamWriter = null;
|
||||
|
||||
Interlocked.Add(ref _curFileCount, -1);
|
||||
if (_curFileCount % 1000 == 0)
|
||||
{
|
||||
Log.Trace("Closed some files: {0}", _curFileCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Add filtering to safe check the symbol for windows environments
|
||||
/// </summary>
|
||||
/// <param name="symbol">Symbol to rename if required</param>
|
||||
/// <returns>Renamed symbol for reserved names</returns>
|
||||
private static Symbol Safe(Symbol symbol)
|
||||
{
|
||||
if (OS.IsWindows)
|
||||
{
|
||||
if (_windowsRestrictedNames.Contains(symbol.Value.ToLowerInvariant()))
|
||||
{
|
||||
symbol = Symbol.CreateFuture(SafeName(symbol.Underlying.Value), symbol.ID.Market, symbol.ID.Date);
|
||||
}
|
||||
}
|
||||
return symbol;
|
||||
}
|
||||
private static string SafeName(string fileName)
|
||||
{
|
||||
if (OS.IsWindows)
|
||||
{
|
||||
foreach (var name in _windowsRestrictedNames)
|
||||
{
|
||||
// The 'con' restricted filename will corrupt the 'seCONed' filepath
|
||||
var restrictedFilePath = Path.DirectorySeparatorChar + name;
|
||||
var safeFilePath = Path.DirectorySeparatorChar + "_" + name;
|
||||
fileName = fileName.Replace(restrictedFilePath, safeFilePath);
|
||||
}
|
||||
}
|
||||
return fileName;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using QuantConnect.Logging;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using QuantConnect.Configuration;
|
||||
using System.Linq;
|
||||
using System.IO;
|
||||
|
||||
namespace QuantConnect.ToolBox.AlgoSeekFuturesConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// AlgoSeek Options Converter: Convert raw OPRA channel files into QuantConnect Options Data Format.
|
||||
/// </summary>
|
||||
public static class AlgoSeekFuturesProgram
|
||||
{
|
||||
public static void AlgoSeekFuturesConverter(string date)
|
||||
{
|
||||
// There are practical file limits we need to override for this to work.
|
||||
// By default programs are only allowed 1024 files open; for futures parsing we need 100k
|
||||
Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "disabled");
|
||||
Log.LogHandler = new CompositeLogHandler(new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("log.txt") });
|
||||
|
||||
// Directory for the data, output and processed cache:
|
||||
var remoteDirectory = Config.Get("futures-remote-directory").Replace("{0}", date);
|
||||
var sourceDirectory = Config.Get("futures-source-directory").Replace("{0}", date);
|
||||
var dataDirectory = Globals.DataFolder;
|
||||
var resolutions = Config.Get("resolutions");
|
||||
var cleanSourceDirectory = Config.GetBool("clean-source-directory", false);
|
||||
|
||||
Log.Trace("CONFIGURATION:");
|
||||
Log.Trace("Processor Count: " + Environment.ProcessorCount);
|
||||
Log.Trace("Remote Directory: " + remoteDirectory);
|
||||
Log.Trace("Source Directory: " + sourceDirectory);
|
||||
Log.Trace("Destination Directory: " + dataDirectory);
|
||||
|
||||
// Date for the option bz files.
|
||||
var referenceDate = DateTime.ParseExact(date, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
|
||||
|
||||
Log.Trace("DateTime: " + referenceDate.Date.ToStringInvariant());
|
||||
|
||||
// checking if remote folder exists
|
||||
if(!Directory.Exists(remoteDirectory))
|
||||
{
|
||||
Log.Error("Remote Directory doesn't exist: " + remoteDirectory);
|
||||
return;
|
||||
}
|
||||
|
||||
// prepare tick types
|
||||
var resolutionList = new[] { Resolution.Minute };
|
||||
|
||||
if (!string.IsNullOrEmpty(resolutions))
|
||||
{
|
||||
var names = resolutions.Split(new[] { ';' });
|
||||
resolutionList =
|
||||
names
|
||||
.Where(x => !string.IsNullOrEmpty(x))
|
||||
.Select(name => (Resolution)Enum.Parse(typeof(Resolution), name, true)).ToArray();
|
||||
}
|
||||
|
||||
Log.Trace("Resolutions: " + string.Join(";", resolutionList.Select(x => x.ToString()).ToArray()));
|
||||
|
||||
// Convert the date:
|
||||
var timer = Stopwatch.StartNew();
|
||||
var converter = new AlgoSeekFuturesConverter(resolutionList.ToList() , referenceDate, remoteDirectory, sourceDirectory, dataDirectory);
|
||||
converter.Convert();
|
||||
Log.Trace($"AlgoSeekFuturesConverter.Main(): {referenceDate.ToStringInvariant()} Conversion finished in time: {timer.Elapsed.ToStringInvariant(null)}");
|
||||
|
||||
// Compress the memory cache to zips.
|
||||
timer.Restart();
|
||||
converter.Package(referenceDate);
|
||||
Log.Trace($"AlgoSeekFuturesConverter.Main(): {referenceDate.ToStringInvariant()} Compression finished in time: {timer.Elapsed.ToStringInvariant(null)}");
|
||||
|
||||
if (cleanSourceDirectory)
|
||||
{
|
||||
Log.Trace($"AlgoSeekFuturesConverter.Main(): Cleaning source directory: {sourceDirectory}");
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(sourceDirectory, true);
|
||||
}
|
||||
catch(Exception err)
|
||||
{
|
||||
Log.Trace($"AlgoSeekFuturesConverter.Main(): Error while cleaning source directory {err.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Securities.Future;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.ToolBox.AlgoSeekFuturesConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Enumerator for converting AlgoSeek futures files into Ticks.
|
||||
/// </summary>
|
||||
public class AlgoSeekFuturesReader : IEnumerator<Tick>
|
||||
{
|
||||
private readonly Stream _stream;
|
||||
private readonly StreamReader _streamReader;
|
||||
private readonly HashSet<string> _symbolFilter;
|
||||
private readonly Dictionary<string, decimal> _symbolMultipliers;
|
||||
private readonly SymbolPropertiesDatabase _symbolProperties;
|
||||
|
||||
private readonly int _columnTimestamp = -1;
|
||||
private readonly int _columnSecID = -1;
|
||||
private readonly int _columnTicker = -1;
|
||||
private readonly int _columnType = -1;
|
||||
private readonly int _columnSide = -1;
|
||||
private readonly int _columnQuantity = -1;
|
||||
private readonly int _columnPrice = -1;
|
||||
private readonly int _columnsCount = -1;
|
||||
|
||||
/// <summary>
|
||||
/// Enumerate through the lines of the algoseek files.
|
||||
/// </summary>
|
||||
/// <param name="file">BZ File for AlgoSeek</param>
|
||||
/// <param name="symbolMultipliers">Symbol price multiplier</param>
|
||||
/// <param name="symbolFilter">Symbol filter to apply, if any</param>
|
||||
public AlgoSeekFuturesReader(string file, Dictionary<string, decimal> symbolMultipliers, HashSet<string> symbolFilter = null)
|
||||
{
|
||||
var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file));
|
||||
_stream = streamProvider.Open(file).First();
|
||||
_streamReader = new StreamReader(_stream);
|
||||
_symbolFilter = symbolFilter;
|
||||
_symbolMultipliers = symbolMultipliers.ToDictionary();
|
||||
_symbolProperties = SymbolPropertiesDatabase.FromDataFolder();
|
||||
|
||||
// detecting column order in the file
|
||||
var headerLine = _streamReader.ReadLine();
|
||||
if (!string.IsNullOrEmpty(headerLine))
|
||||
{
|
||||
var header = headerLine.ToCsv();
|
||||
_columnTimestamp = header.FindIndex(x => x == "Timestamp");
|
||||
_columnTicker = header.FindIndex(x => x == "Ticker");
|
||||
_columnType = header.FindIndex(x => x == "Type");
|
||||
_columnSide = header.FindIndex(x => x == "Side");
|
||||
_columnSecID = header.FindIndex(x => x == "SecurityID");
|
||||
_columnQuantity = header.FindIndex(x => x == "Quantity");
|
||||
_columnPrice = header.FindIndex(x => x == "Price");
|
||||
|
||||
_columnsCount = new[] { _columnTimestamp, _columnTicker, _columnType, _columnSide, _columnSecID, _columnQuantity, _columnPrice }.Max();
|
||||
}
|
||||
//Prime the data pump, set the current.
|
||||
Current = null;
|
||||
MoveNext();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse the next line of the algoseek future file.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public bool MoveNext()
|
||||
{
|
||||
string line;
|
||||
Tick tick = null;
|
||||
while (tick == null && (line = _streamReader.ReadLine()) != null)
|
||||
{
|
||||
// If line is invalid continue looping to find next valid line.
|
||||
tick = Parse(line);
|
||||
}
|
||||
|
||||
Current = tick;
|
||||
return Current != null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Current top of the tick file.
|
||||
/// </summary>
|
||||
public Tick Current { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current element in the collection.
|
||||
/// </summary>
|
||||
/// <returns>
|
||||
/// The current element in the collection.
|
||||
/// </returns>
|
||||
object IEnumerator.Current => Current;
|
||||
|
||||
/// <summary>
|
||||
/// Reset the enumerator for the AlgoSeekFuturesReader
|
||||
/// </summary>
|
||||
public void Reset()
|
||||
{
|
||||
throw new NotImplementedException("Reset not implemented for AlgoSeekFuturesReader.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Dispose of the underlying AlgoSeekFuturesReader
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_stream.Close();
|
||||
_stream.Dispose();
|
||||
_streamReader.Close();
|
||||
_streamReader.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse a string line into a future tick.
|
||||
/// </summary>
|
||||
/// <param name="line"></param>
|
||||
/// <returns></returns>
|
||||
private Tick Parse(string line)
|
||||
{
|
||||
try
|
||||
{
|
||||
const int TradeMask = 2;
|
||||
const int QuoteMask = 1;
|
||||
const int OpenInterestMask = 11;
|
||||
const int MessageTypeMask = 15;
|
||||
|
||||
// parse csv check column count
|
||||
var csv = line.ToCsv();
|
||||
if (csv.Count - 1 < _columnsCount)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var ticker = csv[_columnTicker];
|
||||
|
||||
// we filter out options and spreads
|
||||
if (ticker.IndexOfAny(new [] { ' ', '-' }) != -1)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ticker = ticker.Trim('"');
|
||||
|
||||
if (string.IsNullOrEmpty(ticker))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// ignoring time zones completely -- this is all in the 'data-time-zone'
|
||||
var timeString = csv[_columnTimestamp];
|
||||
var time = DateTime.ParseExact(timeString, "yyyyMMddHHmmssFFF", CultureInfo.InvariantCulture);
|
||||
|
||||
var symbol = SymbolRepresentation.ParseFutureSymbol(ticker, time.Year);
|
||||
|
||||
if (symbol == null || !_symbolMultipliers.ContainsKey(symbol.ID.Symbol) ||
|
||||
_symbolFilter != null && !_symbolFilter.Contains(symbol.ID.Symbol, StringComparer.InvariantCultureIgnoreCase))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// detecting tick type (trade or quote)
|
||||
TickType tickType;
|
||||
bool isAsk = false;
|
||||
|
||||
var type = csv[_columnType].ConvertInvariant<int>();
|
||||
if ((type & MessageTypeMask) == TradeMask)
|
||||
{
|
||||
tickType = TickType.Trade;
|
||||
}
|
||||
else if ((type & MessageTypeMask) == OpenInterestMask)
|
||||
{
|
||||
tickType = TickType.OpenInterest;
|
||||
}
|
||||
else if ((type & MessageTypeMask) == QuoteMask)
|
||||
{
|
||||
tickType = TickType.Quote;
|
||||
|
||||
switch (csv[_columnSide])
|
||||
{
|
||||
case "B":
|
||||
isAsk = false;
|
||||
break;
|
||||
case "S":
|
||||
isAsk = true;
|
||||
break;
|
||||
default:
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// All futures but VIX are delivered with a scale factor of 10000000000.
|
||||
var scaleFactor = symbol.ID.Symbol == "VX" ? decimal.One : 10000000000m;
|
||||
|
||||
var price = csv[_columnPrice].ToDecimal() / scaleFactor;
|
||||
var quantity = csv[_columnQuantity].ToInt32();
|
||||
|
||||
price *= _symbolMultipliers[symbol.ID.Symbol];
|
||||
|
||||
switch (tickType)
|
||||
{
|
||||
case TickType.Quote:
|
||||
|
||||
var tick = new Tick
|
||||
{
|
||||
Symbol = symbol,
|
||||
Time = time,
|
||||
TickType = tickType,
|
||||
Value = price
|
||||
};
|
||||
|
||||
if (isAsk)
|
||||
{
|
||||
tick.AskPrice = price;
|
||||
tick.AskSize = quantity;
|
||||
}
|
||||
else
|
||||
{
|
||||
tick.BidPrice = price;
|
||||
tick.BidSize = quantity;
|
||||
}
|
||||
|
||||
return tick;
|
||||
|
||||
case TickType.Trade:
|
||||
|
||||
tick = new Tick
|
||||
{
|
||||
Symbol = symbol,
|
||||
Time = time,
|
||||
TickType = tickType,
|
||||
Value = price,
|
||||
Quantity = quantity
|
||||
};
|
||||
return tick;
|
||||
|
||||
case TickType.OpenInterest:
|
||||
|
||||
tick = new Tick
|
||||
{
|
||||
Symbol = symbol,
|
||||
Time = time,
|
||||
TickType = tickType,
|
||||
Exchange = symbol.ID.Market,
|
||||
Value = quantity
|
||||
};
|
||||
return tick;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
catch (Exception err)
|
||||
{
|
||||
Log.Error(err);
|
||||
Log.Trace("Line: {0}", line);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using Ionic.BZip2;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
public class Bz2StreamProvider : IStreamProvider
|
||||
{
|
||||
/// <summary>
|
||||
/// Opens the specified source as read to be consumed stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be opened</param>
|
||||
/// <returns>The stream representing the specified source</returns>
|
||||
public IEnumerable<Stream> Open(string source)
|
||||
{
|
||||
yield return new BZip2InputStream(File.OpenRead(source));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Closes the specified source file stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be closed</param>
|
||||
public void Close(string source)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Configuration;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Util;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using QuantConnect.Lean.Engine.DataFeeds;
|
||||
using DateTime = System.DateTime;
|
||||
using Log = QuantConnect.Logging.Log;
|
||||
using QuantConnect.Data.UniverseSelection;
|
||||
using static QuantConnect.Data.UniverseSelection.CoarseFundamentalDataProvider;
|
||||
using QuantConnect.Data.Fundamental;
|
||||
|
||||
namespace QuantConnect.ToolBox.CoarseUniverseGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Coarse
|
||||
/// </summary>
|
||||
public class CoarseUniverseGeneratorProgram
|
||||
{
|
||||
/// <summary>
|
||||
/// Has fundamental data source
|
||||
/// </summary>
|
||||
public const FundamentalProperty HasFundamentalSource = FundamentalProperty.CompanyReference_CompanyId;
|
||||
|
||||
private readonly DirectoryInfo _dailyDataFolder;
|
||||
private readonly DirectoryInfo _destinationFolder;
|
||||
private readonly IMapFileProvider _mapFileProvider;
|
||||
private readonly IFactorFileProvider _factorFileProvider;
|
||||
private readonly string _market;
|
||||
private readonly FileInfo _blackListedTickersFile;
|
||||
|
||||
/// <summary>
|
||||
/// Runs the Coarse universe generator with default values.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public static bool CoarseUniverseGenerator()
|
||||
{
|
||||
var dailyDataFolder = new DirectoryInfo(Path.Combine(Globals.DataFolder, SecurityType.Equity.SecurityTypeToLower(), Market.USA, Resolution.Daily.ResolutionToLower()));
|
||||
var destinationFolder = new DirectoryInfo(Path.Combine(Globals.DataFolder, SecurityType.Equity.SecurityTypeToLower(), Market.USA, "fundamental", "coarse"));
|
||||
var blackListedTickersFile = new FileInfo("blacklisted-tickers.txt");
|
||||
var reservedWordPrefix = Config.Get("reserved-words-prefix", "quantconnect-");
|
||||
var dataProvider = new DefaultDataProvider();
|
||||
var mapFileProvider = new LocalDiskMapFileProvider();
|
||||
mapFileProvider.Initialize(dataProvider);
|
||||
var factorFileProvider = new LocalDiskFactorFileProvider();
|
||||
factorFileProvider.Initialize(mapFileProvider, dataProvider);
|
||||
FundamentalService.Initialize(dataProvider, nameof(CoarseFundamentalDataProvider), false);
|
||||
var generator = new CoarseUniverseGeneratorProgram(dailyDataFolder, destinationFolder, Market.USA, blackListedTickersFile, reservedWordPrefix, mapFileProvider, factorFileProvider);
|
||||
return generator.Run(out _, out _);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="CoarseUniverseGeneratorProgram"/> class.
|
||||
/// </summary>
|
||||
/// <param name="dailyDataFolder">The daily data folder.</param>
|
||||
/// <param name="destinationFolder">The destination folder.</param>
|
||||
/// <param name="market">The market.</param>
|
||||
/// <param name="blackListedTickersFile">The black listed tickers file.</param>
|
||||
/// <param name="reservedWordsPrefix">The reserved words prefix.</param>
|
||||
/// <param name="mapFileProvider">The map file provider.</param>
|
||||
/// <param name="factorFileProvider">The factor file provider.</param>
|
||||
/// <param name="debugEnabled">if set to <c>true</c> [debug enabled].</param>
|
||||
public CoarseUniverseGeneratorProgram(
|
||||
DirectoryInfo dailyDataFolder,
|
||||
DirectoryInfo destinationFolder,
|
||||
string market,
|
||||
FileInfo blackListedTickersFile,
|
||||
string reservedWordsPrefix,
|
||||
IMapFileProvider mapFileProvider,
|
||||
IFactorFileProvider factorFileProvider,
|
||||
bool debugEnabled = false)
|
||||
{
|
||||
_blackListedTickersFile = blackListedTickersFile;
|
||||
_market = market;
|
||||
_factorFileProvider = factorFileProvider;
|
||||
_mapFileProvider = mapFileProvider;
|
||||
_destinationFolder = destinationFolder;
|
||||
_dailyDataFolder = dailyDataFolder;
|
||||
|
||||
Log.DebuggingEnabled = debugEnabled;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs this instance.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public bool Run(out ConcurrentDictionary<SecurityIdentifier, List<CoarseFundamental>> coarsePerSecurity, out DateTime[] dates)
|
||||
{
|
||||
var startTime = DateTime.UtcNow;
|
||||
var success = true;
|
||||
Log.Trace($"CoarseUniverseGeneratorProgram.ProcessDailyFolder(): Processing: {_dailyDataFolder.FullName}");
|
||||
|
||||
var symbolsProcessed = 0;
|
||||
var filesRead = 0;
|
||||
var dailyFilesNotFound = 0;
|
||||
var coarseFilesGenerated = 0;
|
||||
|
||||
var mapFileResolver = _mapFileProvider.Get(new AuxiliaryDataKey(_market, SecurityType.Equity));
|
||||
|
||||
var result = coarsePerSecurity = new();
|
||||
dates = Array.Empty<DateTime>();
|
||||
|
||||
var blackListedTickers = new HashSet<string>();
|
||||
if (_blackListedTickersFile.Exists)
|
||||
{
|
||||
blackListedTickers = File.ReadAllLines(_blackListedTickersFile.FullName).ToHashSet();
|
||||
}
|
||||
|
||||
var securityIdentifierContexts = PopulateSidContex(mapFileResolver, blackListedTickers);
|
||||
var dailyPricesByTicker = new ConcurrentDictionary<string, List<TradeBar>>();
|
||||
var outputCoarseContent = new ConcurrentDictionary<DateTime, List<CoarseFundamental>>();
|
||||
|
||||
var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2) };
|
||||
try
|
||||
{
|
||||
Parallel.ForEach(securityIdentifierContexts, parallelOptions, sidContext =>
|
||||
{
|
||||
var coarseForSecurity = new List<CoarseFundamental>();
|
||||
var symbol = new Symbol(sidContext.SID, sidContext.LastTicker);
|
||||
var symbolCount = Interlocked.Increment(ref symbolsProcessed);
|
||||
Log.Debug($"CoarseUniverseGeneratorProgram.Run(): Processing {symbol} with tickers: '{string.Join(",", sidContext.Tickers)}'");
|
||||
var factorFile = _factorFileProvider.Get(symbol);
|
||||
|
||||
// Populate dailyPricesByTicker with all daily data by ticker for all tickers of this security.
|
||||
foreach (var ticker in sidContext.Tickers)
|
||||
{
|
||||
var pathFile = Path.Combine(_dailyDataFolder.FullName, $"{ticker}.zip");
|
||||
var dailyFile = new FileInfo(pathFile);
|
||||
if (!dailyFile.Exists)
|
||||
{
|
||||
Log.Debug($"CoarseUniverseGeneratorProgram.Run(): {dailyFile.FullName} not found, looking for daily data in data folder");
|
||||
|
||||
dailyFile = new FileInfo(Path.Combine(Globals.DataFolder, "equity", "usa", "daily", $"{ticker}.zip"));
|
||||
if (!dailyFile.Exists)
|
||||
{
|
||||
Log.Error($"CoarseUniverseGeneratorProgram.Run(): {dailyFile} not found!");
|
||||
Interlocked.Increment(ref dailyFilesNotFound);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (!dailyPricesByTicker.ContainsKey(ticker))
|
||||
{
|
||||
dailyPricesByTicker.AddOrUpdate(ticker, ParseDailyFile(dailyFile));
|
||||
Interlocked.Increment(ref filesRead);
|
||||
}
|
||||
}
|
||||
|
||||
// Look for daily data for each ticker of the actual security
|
||||
for (int mapFileRowIndex = sidContext.MapFileRows.Length - 1; mapFileRowIndex >= 1; mapFileRowIndex--)
|
||||
{
|
||||
var ticker = sidContext.MapFileRows[mapFileRowIndex].Item2.ToLowerInvariant();
|
||||
var endDate = sidContext.MapFileRows[mapFileRowIndex].Item1;
|
||||
var startDate = sidContext.MapFileRows[mapFileRowIndex - 1].Item1;
|
||||
List<TradeBar> tickerDailyData;
|
||||
if (!dailyPricesByTicker.TryGetValue(ticker, out tickerDailyData))
|
||||
{
|
||||
Log.Error($"CoarseUniverseGeneratorProgram.Run(): Daily data for ticker {ticker.ToUpperInvariant()} not found!");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get daily data only for the time the ticker was
|
||||
foreach (var tradeBar in tickerDailyData.Where(tb => tb.Time >= startDate && tb.Time <= endDate))
|
||||
{
|
||||
var coarseFundamental = GenerateFactorFileRow(ticker, sidContext, factorFile as CorporateFactorProvider, tradeBar);
|
||||
coarseForSecurity.Add(coarseFundamental);
|
||||
|
||||
outputCoarseContent.AddOrUpdate(tradeBar.Time,
|
||||
new List<CoarseFundamental> { coarseFundamental },
|
||||
(time, list) =>
|
||||
{
|
||||
lock (list)
|
||||
{
|
||||
list.Add(coarseFundamental);
|
||||
return list;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if(coarseForSecurity.Count > 0)
|
||||
{
|
||||
result[sidContext.SID] = coarseForSecurity;
|
||||
}
|
||||
if (symbolCount % 1000 == 0)
|
||||
{
|
||||
var elapsed = DateTime.UtcNow - startTime;
|
||||
Log.Trace($"CoarseUniverseGeneratorProgram.Run(): Processed {symbolCount} in {elapsed:g} at {symbolCount / elapsed.TotalMinutes:F2} symbols/minute ");
|
||||
}
|
||||
});
|
||||
|
||||
_destinationFolder.Create();
|
||||
var startWriting = DateTime.UtcNow;
|
||||
Parallel.ForEach(outputCoarseContent, coarseByDate =>
|
||||
{
|
||||
var filename = $"{coarseByDate.Key.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)}.csv";
|
||||
var filePath = Path.Combine(_destinationFolder.FullName, filename);
|
||||
Log.Debug($"CoarseUniverseGeneratorProgram.Run(): Saving {filename} with {coarseByDate.Value.Count} entries.");
|
||||
File.WriteAllLines(filePath, coarseByDate.Value.Select(x => CoarseFundamental.ToRow(x)).OrderBy(cr => cr));
|
||||
var filesCount = Interlocked.Increment(ref coarseFilesGenerated);
|
||||
if (filesCount % 1000 == 0)
|
||||
{
|
||||
var elapsed = DateTime.UtcNow - startWriting;
|
||||
Log.Trace($"CoarseUniverseGeneratorProgram.Run(): Processed {filesCount} in {elapsed:g} at {filesCount / elapsed.TotalSeconds:F2} files/second ");
|
||||
}
|
||||
});
|
||||
|
||||
dates = outputCoarseContent.Keys.OrderBy(x => x).ToArray();
|
||||
Log.Trace($"\n\nTotal of {coarseFilesGenerated} coarse files generated in {DateTime.UtcNow - startTime:g}:\n" +
|
||||
$"\t => {filesRead} daily data files read.\n");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Error(e, $"CoarseUniverseGeneratorProgram.Run(): FAILED!");
|
||||
success = false;
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates the factor file row.
|
||||
/// </summary>
|
||||
/// <param name="ticker">The ticker.</param>
|
||||
/// <param name="sidContext">The sid context.</param>
|
||||
/// <param name="factorFile">The factor file.</param>
|
||||
/// <param name="tradeBar">The trade bar.</param>
|
||||
/// <param name="fineAvailableDates">The fine available dates.</param>
|
||||
/// <param name="fineFundamentalFolder">The fine fundamental folder.</param>
|
||||
/// <returns></returns>
|
||||
private static CoarseFundamental GenerateFactorFileRow(string ticker, SecurityIdentifierContext sidContext, CorporateFactorProvider factorFile, TradeBar tradeBar)
|
||||
{
|
||||
var date = tradeBar.Time;
|
||||
var factorFileRow = factorFile?.GetScalingFactors(date);
|
||||
var dollarVolume = Math.Truncate((double)(tradeBar.Close * tradeBar.Volume));
|
||||
var priceFactor = factorFileRow?.PriceFactor.Normalize() ?? 1m;
|
||||
var splitFactor = factorFileRow?.SplitFactor.Normalize() ?? 1m;
|
||||
var hasFundamentalData = CheckFundamentalData(date, sidContext.SID);
|
||||
|
||||
// sid,symbol,close,volume,dollar volume,has fundamental data,price factor,split factor
|
||||
return new CoarseFundamentalSource
|
||||
{
|
||||
Symbol = new Symbol(sidContext.SID, ticker),
|
||||
Value = tradeBar.Close.Normalize(),
|
||||
Time = date,
|
||||
VolumeSetter = decimal.ToInt64(tradeBar.Volume),
|
||||
DollarVolumeSetter = dollarVolume,
|
||||
PriceFactorSetter = priceFactor,
|
||||
SplitFactorSetter = splitFactor,
|
||||
HasFundamentalDataSetter = hasFundamentalData
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if there is fundamental data for
|
||||
/// </summary>
|
||||
/// <param name="date">The date.</param>
|
||||
/// <param name="sid">The security identifier.</param>
|
||||
/// <returns>True if fundamental data is available</returns>
|
||||
private static bool CheckFundamentalData(DateTime date, SecurityIdentifier sid)
|
||||
{
|
||||
return !string.IsNullOrEmpty(FundamentalService.Get<string>(date, sid, HasFundamentalSource));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses the daily file.
|
||||
/// </summary>
|
||||
/// <param name="dailyFile">The daily file.</param>
|
||||
/// <returns></returns>
|
||||
private static List<TradeBar> ParseDailyFile(FileInfo dailyFile)
|
||||
{
|
||||
var scaleFactor = 1 / 10000m;
|
||||
|
||||
var output = new List<TradeBar>();
|
||||
using (var fileStream = dailyFile.OpenRead())
|
||||
using (var stream = Compression.UnzipStreamToStreamReader(fileStream))
|
||||
{
|
||||
while (!stream.EndOfStream)
|
||||
{
|
||||
var tradeBar = new TradeBar
|
||||
{
|
||||
Time = stream.GetDateTime(),
|
||||
Open = stream.GetDecimal() * scaleFactor,
|
||||
High = stream.GetDecimal() * scaleFactor,
|
||||
Low = stream.GetDecimal() * scaleFactor,
|
||||
Close = stream.GetDecimal() * scaleFactor,
|
||||
Volume = stream.GetDecimal()
|
||||
};
|
||||
output.Add(tradeBar);
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Populates the sid contex.
|
||||
/// </summary>
|
||||
/// <param name="mapFileResolver">The map file resolver.</param>
|
||||
/// <param name="exclusions">The exclusions.</param>
|
||||
/// <returns></returns>
|
||||
private IEnumerable<SecurityIdentifierContext> PopulateSidContex(MapFileResolver mapFileResolver, HashSet<string> exclusions)
|
||||
{
|
||||
Log.Trace("CoarseUniverseGeneratorProgram.PopulateSidContex(): Generating SID context from QuantQuote's map files.");
|
||||
foreach (var mapFile in mapFileResolver)
|
||||
{
|
||||
if (exclusions.Contains(mapFile.Last().MappedSymbol))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return new SecurityIdentifierContext(mapFile, _market);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using System;
|
||||
using System.Linq;
|
||||
|
||||
namespace QuantConnect.ToolBox.CoarseUniverseGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Auxiliary class for handling map files and SID.
|
||||
/// </summary>
|
||||
internal class SecurityIdentifierContext
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="SecurityIdentifierContext"/> class.
|
||||
/// </summary>
|
||||
/// <param name="mapFile">The map file.</param>
|
||||
/// <param name="market">The market.</param>
|
||||
public SecurityIdentifierContext(MapFile mapFile, string market)
|
||||
{
|
||||
MapFile = mapFile;
|
||||
SID = SecurityIdentifier.GenerateEquity(MapFile.FirstDate, MapFile.FirstTicker, market);
|
||||
MapFileRows = MapFile.Select(mfr => new Tuple<DateTime, string>(mfr.Date, mfr.MappedSymbol)).ToArray();
|
||||
Tickers = MapFile.Select(mfr => mfr.MappedSymbol.ToLowerInvariant()).Distinct().ToArray();
|
||||
LastTicker = MapFile.Last().MappedSymbol.ToLowerInvariant();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the sid.
|
||||
/// </summary>
|
||||
/// <value>
|
||||
/// The sid.
|
||||
/// </value>
|
||||
public SecurityIdentifier SID { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the map file.
|
||||
/// </summary>
|
||||
/// <value>
|
||||
/// The map file.
|
||||
/// </value>
|
||||
public MapFile MapFile { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the map file rows.
|
||||
/// </summary>
|
||||
/// <value>
|
||||
/// The map file rows.
|
||||
/// </value>
|
||||
public Tuple<DateTime, string>[] MapFileRows { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the tickers.
|
||||
/// </summary>
|
||||
/// <value>
|
||||
/// The tickers.
|
||||
/// </value>
|
||||
public string[] Tickers { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the last ticker.
|
||||
/// </summary>
|
||||
/// <value>
|
||||
/// The last ticker.
|
||||
/// </value>
|
||||
public string LastTicker { get; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
ACCPW
|
||||
AFGL
|
||||
AITPU
|
||||
ALIT
|
||||
ALZH
|
||||
AMEHW
|
||||
AMQ
|
||||
ANSBF
|
||||
ARSGZ
|
||||
ASHPF
|
||||
ATEST
|
||||
ATEST.A
|
||||
ATEST.B
|
||||
ATEST.C
|
||||
ATEST.G
|
||||
ATEST.H
|
||||
ATEST.L
|
||||
ATEST.Z
|
||||
BBRX
|
||||
BDFC
|
||||
BGSM
|
||||
BKSJZ
|
||||
BOLTW
|
||||
BOXLW
|
||||
BPTS
|
||||
CBUS
|
||||
CLO
|
||||
CNTX
|
||||
CTEST
|
||||
CTEST.A
|
||||
CTEST.B
|
||||
CTEST.C
|
||||
CTEST.D
|
||||
CTEST.E
|
||||
CTEST.F
|
||||
CTEST.G
|
||||
CTEST.H
|
||||
CTEST.I
|
||||
CTEST.J
|
||||
CTEST.K
|
||||
CTEST.L
|
||||
CTEST.M
|
||||
CTEST.N
|
||||
CTEST.O
|
||||
CTEST.P
|
||||
CTEST.Q
|
||||
CTEST.R
|
||||
CTEST.S
|
||||
CTEST.T
|
||||
CTEST.U
|
||||
CTEST.V
|
||||
CTEST.W
|
||||
CTEST.X
|
||||
CTEST.Y
|
||||
CTEST.Z
|
||||
CVTS
|
||||
DDOC
|
||||
DRMT
|
||||
EFRTF
|
||||
EGRGF
|
||||
EHR
|
||||
EMMSV
|
||||
FJP.B
|
||||
FKLYU
|
||||
FSNV
|
||||
FSPR
|
||||
FTPPP
|
||||
GBLK
|
||||
GCAA
|
||||
GLSS
|
||||
GNST
|
||||
HMU
|
||||
IBEX
|
||||
IDGX
|
||||
IGLEU
|
||||
ILSAP
|
||||
IPOW
|
||||
IPOX
|
||||
JPACZ
|
||||
JPPXZ
|
||||
JPRMZ
|
||||
JPYOZ
|
||||
KLOX
|
||||
KRAT
|
||||
LBUY
|
||||
LBYAV
|
||||
LBYKV
|
||||
LRPRF
|
||||
LXL
|
||||
MBLTY
|
||||
MDQZZ
|
||||
MFCA
|
||||
MKVNF
|
||||
MOLC
|
||||
MOTA
|
||||
MOTAW
|
||||
MTEST
|
||||
MTEST.A
|
||||
MVC.H
|
||||
MVIR
|
||||
NEPG
|
||||
NTEST W
|
||||
NTEST.A
|
||||
NTEST.B
|
||||
NTEST.C
|
||||
NTEST.D
|
||||
NTEST.E
|
||||
NTEST.F
|
||||
NTEST.G
|
||||
NTEST.H
|
||||
NTEST.I
|
||||
NTEST.J
|
||||
NTEST.K
|
||||
NTEST.L
|
||||
NTEST.M
|
||||
NTEST.N
|
||||
NTEST.O
|
||||
NTEST.P
|
||||
NTEST.Q
|
||||
NTEST.R
|
||||
NTEST.S
|
||||
NTEST.T
|
||||
NTEST.U
|
||||
NTEST.V
|
||||
NTEST.W
|
||||
NTEST.X
|
||||
NTEST.Y
|
||||
NTEST.Z
|
||||
OFGIZ
|
||||
OTG
|
||||
PCIMU
|
||||
PCON
|
||||
PEER
|
||||
PEERW
|
||||
PMVAU
|
||||
PNTGV
|
||||
POSHZ
|
||||
PRHR
|
||||
PRKU
|
||||
PRMCF
|
||||
PRMF
|
||||
PSAV
|
||||
PSRT
|
||||
PSWW
|
||||
PTCY
|
||||
PTEST
|
||||
PTEST.A
|
||||
PTEST.B
|
||||
PTEST.W
|
||||
PTEST.X
|
||||
PTEST.Y
|
||||
PTEST.Z
|
||||
REDX
|
||||
SCCI
|
||||
SLWD
|
||||
SNHVV
|
||||
SOHYP
|
||||
TDRRF
|
||||
TESTA
|
||||
TESTB
|
||||
TESTC
|
||||
TESTD
|
||||
TESTE
|
||||
TESTF
|
||||
TESTG
|
||||
TESTH
|
||||
TESTYX
|
||||
TFCPI
|
||||
TFIG
|
||||
UAVSU
|
||||
UREEW
|
||||
VMET
|
||||
VTEC
|
||||
VUZIW
|
||||
WNFM
|
||||
WSPT
|
||||
YAGEY
|
||||
YAGZD
|
||||
YAGZZ
|
||||
YQ
|
||||
ZAZZT
|
||||
ZBZX
|
||||
ZBZZT
|
||||
ZCZZT
|
||||
ZDZZT
|
||||
ZEXIT
|
||||
ZEZZT
|
||||
ZFZZT
|
||||
ZGZZT
|
||||
ZHZZT
|
||||
ZIEXT
|
||||
ZIZZT
|
||||
ZJZZT
|
||||
ZKZZT
|
||||
ZLZZT
|
||||
ZMZZT
|
||||
ZNTRX
|
||||
ZNZZT
|
||||
ZOMHF
|
||||
ZOZZT
|
||||
ZPZZT
|
||||
ZQZZT
|
||||
ZRZZT
|
||||
ZSZZT
|
||||
ZTEST
|
||||
ZTST
|
||||
ZTZZT
|
||||
ZUZZT
|
||||
ZVV
|
||||
ZVZZC
|
||||
ZVZZT
|
||||
ZWZZT
|
||||
ZXIET
|
||||
ZXZZT
|
||||
ZYZZT
|
||||
ZZZZIX
|
||||
ZZZZT
|
||||
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Consolidators;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IDataProcessor"/> that consolidates the data
|
||||
/// stream and forwards the consolidated data to other processors
|
||||
/// </summary>
|
||||
public class ConsolidatorDataProcessor : IDataProcessor
|
||||
{
|
||||
private DateTime _frontier;
|
||||
private readonly IDataProcessor _destination;
|
||||
private readonly Func<IBaseData, IDataConsolidator> _createConsolidator;
|
||||
private readonly Dictionary<Symbol, IDataConsolidator> _consolidators;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ConsolidatorDataProcessor"/> class
|
||||
/// </summary>
|
||||
/// <param name="destination">The receiver of the consolidated data</param>
|
||||
/// <param name="createConsolidator">Function used to create consolidators</param>
|
||||
public ConsolidatorDataProcessor(IDataProcessor destination, Func<IBaseData, IDataConsolidator> createConsolidator)
|
||||
{
|
||||
_destination = destination;
|
||||
_createConsolidator = createConsolidator;
|
||||
_consolidators = new Dictionary<Symbol, IDataConsolidator>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invoked for each piece of data from the source file
|
||||
/// </summary>
|
||||
/// <param name="data">The data to be processed</param>
|
||||
public void Process(IBaseData data)
|
||||
{
|
||||
// grab the correct consolidator for this symbol
|
||||
IDataConsolidator consolidator;
|
||||
if (!_consolidators.TryGetValue(data.Symbol, out consolidator))
|
||||
{
|
||||
consolidator = _createConsolidator(data);
|
||||
consolidator.DataConsolidated += OnDataConsolidated;
|
||||
_consolidators[data.Symbol] = consolidator;
|
||||
}
|
||||
|
||||
consolidator.Update(data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_frontier = DateTime.MaxValue;
|
||||
|
||||
// check the other consolidators to see if they also need to emit their working bars
|
||||
foreach (var consolidator in _consolidators.Values)
|
||||
{
|
||||
consolidator.Scan(_frontier);
|
||||
}
|
||||
|
||||
_destination.Dispose();
|
||||
_consolidators.Clear();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles the <see cref="IDataConsolidator.DataConsolidated"/> event
|
||||
/// </summary>
|
||||
private void OnDataConsolidated(object sender, IBaseData args)
|
||||
{
|
||||
_destination.Process(args);
|
||||
|
||||
// we've already checked this frontier time, so don't scan the consolidators
|
||||
if (_frontier >= args.EndTime) return;
|
||||
_frontier = args.EndTime;
|
||||
|
||||
// check the other consolidators to see if they also need to emit
|
||||
foreach (var consolidator in _consolidators.Values)
|
||||
{
|
||||
// back up the time a single instance, this allows data at exact same
|
||||
// time to still come through
|
||||
consolidator.Scan(args.EndTime.AddTicks(-1));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IDataProcessor"/> that writes the incoming
|
||||
/// stream of data to a csv file.
|
||||
/// </summary>
|
||||
public class CsvDataProcessor : IDataProcessor
|
||||
{
|
||||
private const int TicksPerFlush = 50;
|
||||
private static readonly object DirectoryCreateSync = new object();
|
||||
|
||||
private readonly string _dataDirectory;
|
||||
private readonly Resolution _resolution;
|
||||
private readonly TickType _tickType;
|
||||
private readonly Dictionary<Symbol, Writer> _writers;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="CsvDataProcessor"/> class
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The root data directory, /Data</param>
|
||||
/// <param name="resolution">The resolution being sent into the Process method</param>
|
||||
/// <param name="tickType">The tick type, trade or quote</param>
|
||||
public CsvDataProcessor(string dataDirectory, Resolution resolution, TickType tickType)
|
||||
{
|
||||
_dataDirectory = dataDirectory;
|
||||
_resolution = resolution;
|
||||
_tickType = tickType;
|
||||
_writers = new Dictionary<Symbol, Writer>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invoked for each piece of data from the source file
|
||||
/// </summary>
|
||||
/// <param name="data">The data to be processed</param>
|
||||
public void Process(IBaseData data)
|
||||
{
|
||||
Writer writer;
|
||||
if (!_writers.TryGetValue(data.Symbol, out writer))
|
||||
{
|
||||
writer = CreateTextWriter(data);
|
||||
_writers[data.Symbol] = writer;
|
||||
}
|
||||
|
||||
// flush every so often
|
||||
if (++writer.ProcessCount%TicksPerFlush == 0)
|
||||
{
|
||||
writer.TextWriter.Flush();
|
||||
}
|
||||
|
||||
var line = LeanData.GenerateLine(data, data.Symbol.ID.SecurityType, _resolution);
|
||||
writer.TextWriter.WriteLine(line);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var kvp in _writers)
|
||||
{
|
||||
kvp.Value.TextWriter.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the <see cref="TextWriter"/> that writes data to csv files
|
||||
/// </summary>
|
||||
private Writer CreateTextWriter(IBaseData data)
|
||||
{
|
||||
var entry = LeanData.GenerateZipEntryName(data.Symbol, data.Time.Date, _resolution, _tickType);
|
||||
var relativePath = LeanData.GenerateRelativeZipFilePath(data.Symbol, data.Time.Date, _resolution, _tickType)
|
||||
.Replace(".zip", string.Empty);
|
||||
var path = Path.Combine(Path.Combine(_dataDirectory, relativePath), entry);
|
||||
var directory = new FileInfo(path).Directory.FullName;
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
// lock before checking again
|
||||
lock (DirectoryCreateSync) if (!Directory.Exists(directory)) Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
return new Writer(path, new StreamWriter(path));
|
||||
}
|
||||
|
||||
|
||||
private sealed class Writer
|
||||
{
|
||||
public readonly string Path;
|
||||
public readonly TextWriter TextWriter;
|
||||
public int ProcessCount;
|
||||
public Writer(string path, TextWriter textWriter)
|
||||
{
|
||||
Path = path;
|
||||
TextWriter = textWriter;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Configuration;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Base tool for pulling data from a remote source and updating existing csv file.
|
||||
/// </summary>
|
||||
public class ExchangeInfoUpdater
|
||||
{
|
||||
private readonly IExchangeInfoDownloader _eidl;
|
||||
|
||||
public ExchangeInfoUpdater(IExchangeInfoDownloader eidl)
|
||||
{
|
||||
_eidl = eidl;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Update existing symbol properties database
|
||||
/// </summary>
|
||||
public void Run()
|
||||
{
|
||||
var directory = Path.Combine(Globals.DataFolder, "symbol-properties");
|
||||
var file = Path.Combine(directory, "symbol-properties-database.csv");
|
||||
var baseOutputDirectory = Config.Get("temp-output-directory", "/temp-output-directory");
|
||||
var tempOutputDirectory = Directory.CreateDirectory(Path.Combine(baseOutputDirectory, "symbol-properties"));
|
||||
var tmp = Path.Combine(tempOutputDirectory.FullName, "symbol-properties-database.csv");
|
||||
if (File.Exists(tmp))
|
||||
{
|
||||
file = tmp;
|
||||
}
|
||||
else if (!File.Exists(file))
|
||||
{
|
||||
throw new FileNotFoundException("Unable to locate symbol properties file: " + file);
|
||||
}
|
||||
|
||||
// Read file data before to escape from clash if file == tmp
|
||||
// Dispose off enumerator to free up resource
|
||||
var fileLines = File.ReadLines(file).ToList();
|
||||
|
||||
using (var writer = new StreamWriter(tmp))
|
||||
{
|
||||
var fetch = false;
|
||||
var filter = $"{_eidl.Market},";
|
||||
foreach (var line in fileLines)
|
||||
{
|
||||
if (!line.StartsWithInvariant(filter, true))
|
||||
{
|
||||
writer.WriteLine(line);
|
||||
}
|
||||
else if (!fetch)
|
||||
{
|
||||
WriteData(writer);
|
||||
fetch = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!fetch)
|
||||
{
|
||||
writer.WriteLine(Environment.NewLine);
|
||||
WriteData(writer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteData(StreamWriter writer)
|
||||
{
|
||||
var existingSymbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
|
||||
var entryPerSymbol = _eidl.Get().ToDictionary(newLine => {
|
||||
var splitted = newLine.Split(',');
|
||||
return new SecurityDatabaseKey(splitted[0], splitted[1], (SecurityType)Enum.Parse(typeof(SecurityType), splitted[2], true));
|
||||
});
|
||||
|
||||
foreach (var existingEntry in existingSymbolPropertiesDatabase.GetSymbolPropertiesList(_eidl.Market))
|
||||
{
|
||||
if (!entryPerSymbol.ContainsKey(existingEntry.Key))
|
||||
{
|
||||
// let's keep any existing which is no longer available, to take into account for delistings/removals
|
||||
entryPerSymbol[existingEntry.Key] = $"{existingEntry.Key.Market},{existingEntry.Key.Symbol},{existingEntry.Key.SecurityType.ToLower()},{existingEntry.Value}";
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var upd in entryPerSymbol.OrderBy(x => x.Key.SecurityType).ThenBy(x => x.Key.Symbol))
|
||||
{
|
||||
writer.WriteLine(upd.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Data.Market;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates a factor file from a list of splits and dividends for a specified equity
|
||||
/// </summary>
|
||||
public class FactorFileGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Data for this equity at daily resolution
|
||||
/// </summary>
|
||||
private readonly List<TradeBar> _dailyDataForEquity;
|
||||
|
||||
/// <summary>
|
||||
/// The last date in the _dailyEquityData
|
||||
/// </summary>
|
||||
private readonly DateTime _lastDateFromEquityData;
|
||||
|
||||
/// <summary>
|
||||
/// The symbol for which the factor file is being generated
|
||||
/// </summary>
|
||||
public Symbol Symbol { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Constructor for the FactorFileGenerator
|
||||
/// </summary>
|
||||
/// <param name="symbol">The equity for which the factor file respresents</param>
|
||||
/// <param name="pathForDailyEquityData">The path to the daily data for the specified equity</param>
|
||||
public FactorFileGenerator(Symbol symbol, string pathForDailyEquityData)
|
||||
{
|
||||
Symbol = symbol;
|
||||
_dailyDataForEquity = ReadDailyEquityData(pathForDailyEquityData);
|
||||
_lastDateFromEquityData = _dailyDataForEquity.Last().Time;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create FactorFile instance
|
||||
/// </summary>
|
||||
/// <param name="dividendSplitList">List of Dividends and Splits</param>
|
||||
/// <returns><see cref="FactorFile"/> instance</returns>
|
||||
public CorporateFactorProvider CreateFactorFile(List<BaseData> dividendSplitList)
|
||||
{
|
||||
var orderedDividendSplitQueue = new Queue<BaseData>(
|
||||
CombineIntraDayDividendSplits(dividendSplitList)
|
||||
.OrderByDescending(x => x.Time));
|
||||
|
||||
var factorFileRows = new List<CorporateFactorRow>
|
||||
{
|
||||
// First Factor Row is set far into the future and by definition has 1 for both price and split factors
|
||||
new CorporateFactorRow(
|
||||
Time.EndOfTime,
|
||||
priceFactor: 1,
|
||||
splitFactor: 1
|
||||
)
|
||||
};
|
||||
|
||||
return RecursivlyGenerateFactorFile(orderedDividendSplitQueue, factorFileRows);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If dividend and split occur on the same day,
|
||||
/// combine them into IntraDayDividendSplit object
|
||||
/// </summary>
|
||||
/// <param name="splitDividendList">List of split and dividends</param>
|
||||
/// <returns>A list of splits, dividends with intraday split and dividends combined into <see cref="IntraDayDividendSplit"/></returns>
|
||||
private static List<BaseData> CombineIntraDayDividendSplits(List<BaseData> splitDividendList)
|
||||
{
|
||||
var splitDividendCollection = new Collection<BaseData>(splitDividendList);
|
||||
|
||||
var dateKeysLookup = splitDividendCollection.GroupBy(x => x.Time)
|
||||
.OrderByDescending(x => x.Key)
|
||||
.Select(group => group)
|
||||
.ToList();
|
||||
|
||||
var baseDataList = new List<BaseData>();
|
||||
foreach (var kvpLookup in dateKeysLookup)
|
||||
{
|
||||
if (kvpLookup.Count() > 1)
|
||||
{
|
||||
// Intraday dividend split found
|
||||
var dividend = kvpLookup.First(x => x.GetType() == typeof(Dividend)) as Dividend;
|
||||
var split = kvpLookup.First(x => x.GetType() == typeof(Split)) as Split;
|
||||
baseDataList.Add(new IntraDayDividendSplit(split, dividend));
|
||||
}
|
||||
else
|
||||
{
|
||||
baseDataList.Add(kvpLookup.First());
|
||||
}
|
||||
}
|
||||
|
||||
return baseDataList;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Recursively generate a <see cref="FactorFile"/>
|
||||
/// </summary>
|
||||
/// <param name="orderedDividendSplits">Queue of dividends and splits ordered by date</param>
|
||||
/// <param name="factorFileRows">The list of factor file rows</param>
|
||||
/// <returns><see cref="FactorFile"/> instance</returns>
|
||||
private CorporateFactorProvider RecursivlyGenerateFactorFile(Queue<BaseData> orderedDividendSplits, List<CorporateFactorRow> factorFileRows)
|
||||
{
|
||||
// If there is no more dividends or splits, return
|
||||
if (!orderedDividendSplits.Any())
|
||||
{
|
||||
factorFileRows.Add(CreateLastFactorFileRow(factorFileRows, _dailyDataForEquity.Last().Close));
|
||||
return new CorporateFactorProvider(Symbol.ID.Symbol, factorFileRows);
|
||||
}
|
||||
|
||||
var nextEvent = orderedDividendSplits.Dequeue();
|
||||
|
||||
// If there is no more daily equity data to use, return
|
||||
if (_lastDateFromEquityData > nextEvent.Time)
|
||||
{
|
||||
decimal initialReferencePrice = 1;
|
||||
factorFileRows.Add(CreateLastFactorFileRow(factorFileRows, initialReferencePrice));
|
||||
return new CorporateFactorProvider(Symbol.ID.Symbol, factorFileRows);
|
||||
}
|
||||
|
||||
var nextFactorFileRow = CalculateNextFactorFileRow(factorFileRows, nextEvent);
|
||||
|
||||
if (nextFactorFileRow != null)
|
||||
factorFileRows.Add(nextFactorFileRow);
|
||||
|
||||
return RecursivlyGenerateFactorFile(orderedDividendSplits, factorFileRows);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create the last FileFactorRow.
|
||||
/// Represents the earliest date that the daily equity data contains.
|
||||
/// </summary>
|
||||
/// <param name="factorFileRows">The list of factor file rows</param>
|
||||
/// <returns><see cref="CorporateFactorRow"/></returns>
|
||||
private CorporateFactorRow CreateLastFactorFileRow(List<CorporateFactorRow> factorFileRows, decimal referencePrice)
|
||||
{
|
||||
return new CorporateFactorRow(
|
||||
_dailyDataForEquity.Last().Time.Date,
|
||||
factorFileRows.Last().PriceFactor,
|
||||
factorFileRows.Last().SplitFactor,
|
||||
referencePrice
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates the next <see cref="CorporateFactorRow"/>
|
||||
/// </summary>
|
||||
/// <param name="factorFileRows">The current list of factorFileRows</param>
|
||||
/// <param name="nextEvent">The next dividend, split or intradayDividendSplit</param>
|
||||
/// <returns>A single factor file row</returns>
|
||||
private CorporateFactorRow CalculateNextFactorFileRow(List<CorporateFactorRow> factorFileRows, BaseData nextEvent)
|
||||
{
|
||||
CorporateFactorRow nextCorporateFactorRow;
|
||||
var t = nextEvent.GetType();
|
||||
|
||||
switch (t.Name)
|
||||
{
|
||||
case "Dividend":
|
||||
nextCorporateFactorRow = CalculateNextDividendFactor(nextEvent, factorFileRows.Last());
|
||||
break;
|
||||
case "Split":
|
||||
nextCorporateFactorRow = CalculateNextSplitFactor(nextEvent, factorFileRows.Last());
|
||||
break;
|
||||
case "IntraDayDividendSplit":
|
||||
nextCorporateFactorRow = CalculateIntradayDividendSplit((IntraDayDividendSplit)nextEvent, factorFileRows.Last());
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentException("Unhandled BaseData type for FactorFileGenerator.");
|
||||
}
|
||||
|
||||
return nextCorporateFactorRow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates the <see cref="CorporateFactorRow"/> that represents a intraday dividend split.
|
||||
/// Applies the dividend first.
|
||||
/// </summary>
|
||||
/// <param name="intraDayDividendSplit"><see cref="IntraDayDividendSplit"/> instance that holds the intraday dividend and split information</param>
|
||||
/// <param name="last">The last <see cref="CorporateFactorRow"/> generated recursivly</param>
|
||||
/// <returns><see cref="CorporateFactorRow"/> that represents an intraday dividend and split</returns>
|
||||
private CorporateFactorRow CalculateIntradayDividendSplit(IntraDayDividendSplit intraDayDividendSplit, CorporateFactorRow last)
|
||||
{
|
||||
var row = CalculateNextDividendFactor(intraDayDividendSplit.Dividend, last);
|
||||
return CalculateNextSplitFactor(intraDayDividendSplit.Split, row);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates the price factor of a <see cref="Dividend"/>
|
||||
/// </summary>
|
||||
/// <param name="dividend">The next dividend</param>
|
||||
/// <param name="previousCorporateFactorRow">The previous <see cref="CorporateFactorRow"/> generated</param>
|
||||
/// <returns><see cref="CorporateFactorRow"/> that represents the dividend event</returns>
|
||||
private CorporateFactorRow CalculateNextDividendFactor(BaseData dividend, CorporateFactorRow previousCorporateFactorRow)
|
||||
{
|
||||
var eventDayData = GetDailyDataForDate(dividend.Time);
|
||||
|
||||
// If you don't have the equity data nothing can be calculated
|
||||
if (eventDayData == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
TradeBar previousClosingPrice = FindPreviousTradableDayClosingPrice(eventDayData.Time);
|
||||
|
||||
// adjust the dividend for both price and split factors (!)
|
||||
var priceFactor = previousCorporateFactorRow.PriceFactor *
|
||||
(1 - dividend.Value * previousCorporateFactorRow.SplitFactor / previousClosingPrice.Close);
|
||||
|
||||
return new CorporateFactorRow(
|
||||
previousClosingPrice.Time,
|
||||
priceFactor.RoundToSignificantDigits(7),
|
||||
previousCorporateFactorRow.SplitFactor,
|
||||
previousClosingPrice.Close
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates the split factor of a <see cref="Split"/>
|
||||
/// </summary>
|
||||
/// <param name="split">The next <see cref="Split"/></param>
|
||||
/// <param name="previousCorporateFactorRow">The previous <see cref="CorporateFactorRow"/> generated</param>
|
||||
/// <returns><see cref="CorporateFactorRow"/> that represents the split event</returns>
|
||||
private CorporateFactorRow CalculateNextSplitFactor(BaseData split, CorporateFactorRow previousCorporateFactorRow)
|
||||
{
|
||||
var eventDayData = GetDailyDataForDate(split.Time);
|
||||
|
||||
// If you don't have the equity data nothing can be done
|
||||
if (eventDayData == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
TradeBar previousClosingPrice = FindPreviousTradableDayClosingPrice(eventDayData.Time);
|
||||
|
||||
return new CorporateFactorRow(
|
||||
previousClosingPrice.Time,
|
||||
previousCorporateFactorRow.PriceFactor,
|
||||
(previousCorporateFactorRow.SplitFactor / split.Value).RoundToSignificantDigits(6),
|
||||
previousClosingPrice.Close
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the data for a specified date
|
||||
/// </summary>
|
||||
/// <param name="date">The current specified date</param>
|
||||
/// <returns><see cref="TradeBar"/>representing that date</returns>
|
||||
private TradeBar GetDailyDataForDate(DateTime date)
|
||||
{
|
||||
return _dailyDataForEquity.FirstOrDefault(x => x.Time.Date == date.Date);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the data for the previous tradable day
|
||||
/// </summary>
|
||||
/// <param name="date">The current specified date</param>
|
||||
/// <returns>The last tradeble days data</returns>
|
||||
private TradeBar FindPreviousTradableDayClosingPrice(DateTime date)
|
||||
{
|
||||
TradeBar previousDayData = null;
|
||||
var lastDateforData = _dailyDataForEquity.Last();
|
||||
|
||||
while (previousDayData == null && date > lastDateforData.EndTime)
|
||||
{
|
||||
previousDayData = _dailyDataForEquity.FirstOrDefault(x => x.Time == date.AddDays(-1));
|
||||
date = date.AddDays(-1);
|
||||
}
|
||||
|
||||
return previousDayData;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Read the daily equity date from file
|
||||
/// </summary>
|
||||
/// <param name="pathForDailyEquityData">Path the the daily data</param>
|
||||
/// <returns>A list of <see cref="TradeBar"/> read from file</returns>
|
||||
private static List<TradeBar> ReadDailyEquityData(string pathForDailyEquityData)
|
||||
{
|
||||
var dataReader = new LeanDataReader(pathForDailyEquityData);
|
||||
var bars = dataReader.Parse();
|
||||
return bars.OrderByDescending(x => x.Time)
|
||||
.Select(x => (TradeBar)x)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pairs split and dividend data into one type
|
||||
/// </summary>
|
||||
private class IntraDayDividendSplit : BaseData
|
||||
{
|
||||
public Split Split { get; }
|
||||
public Dividend Dividend { get; }
|
||||
|
||||
public IntraDayDividendSplit(Split split, Dividend dividend)
|
||||
{
|
||||
if (split == null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(split));
|
||||
}
|
||||
|
||||
if (dividend == null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(dividend));
|
||||
}
|
||||
|
||||
Split = split;
|
||||
Dividend = dividend;
|
||||
Time = Split.Time;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IStreamProvider"/> that just returns a file stream
|
||||
/// </summary>
|
||||
public class FileStreamProvider : IStreamProvider
|
||||
{
|
||||
private readonly Dictionary<string, FileStream> _files = new Dictionary<string, FileStream>();
|
||||
|
||||
/// <summary>
|
||||
/// Opens the specified source as read to be consumed stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be opened</param>
|
||||
/// <returns>The stream representing the specified source</returns>
|
||||
public IEnumerable<Stream> Open(string source)
|
||||
{
|
||||
yield return File.OpenRead(source);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Closes the specified source file stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be closed</param>
|
||||
public void Close(string source)
|
||||
{
|
||||
// it's expected that users will dispose the stream
|
||||
// from the open call, this is used to clean up any
|
||||
// other resources, for example a ZipFile stream
|
||||
// when we returned a ZipEntry stream
|
||||
_files.Remove(source);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var kvp in _files)
|
||||
{
|
||||
kvp.Value.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using QuantConnect.Data;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IDataProcessor"/> that filters the incoming
|
||||
/// stream of data before passing it along to the wrapped processor
|
||||
/// </summary>
|
||||
public class FilteredDataProcessor : IDataProcessor
|
||||
{
|
||||
private readonly Func<IBaseData, bool> _predicate;
|
||||
private readonly IDataProcessor _processor;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FilteredDataProcessor"/> class
|
||||
/// </summary>
|
||||
/// <param name="processor">The processor to filter data for</param>
|
||||
/// <param name="predicate">The filtering predicate to be applied</param>
|
||||
public FilteredDataProcessor(IDataProcessor processor, Func<IBaseData, bool> predicate)
|
||||
{
|
||||
_predicate = predicate;
|
||||
_processor = processor;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invoked for each piece of data from the source file
|
||||
/// </summary>
|
||||
/// <param name="data">The data to be processed</param>
|
||||
public void Process(IBaseData data)
|
||||
{
|
||||
if (_predicate(data))
|
||||
{
|
||||
_processor.Process(data);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_processor.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using Ionic.BZip2;
|
||||
using Ionic.Zlib;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
public class GzipStreamProvider : IStreamProvider
|
||||
{
|
||||
private readonly Dictionary<string, Stream> _openedStreams = new Dictionary<string, Stream>(2);
|
||||
|
||||
/// <summary>
|
||||
/// Opens the specified source as read to be consumed stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be opened</param>
|
||||
/// <returns>The stream representing the specified source</returns>
|
||||
public IEnumerable<Stream> Open(string source)
|
||||
{
|
||||
var stream = new GZipStream(File.OpenRead(source), CompressionMode.Decompress);
|
||||
_openedStreams.Add(source, stream);
|
||||
yield return stream;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Closes the specified source file stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be closed</param>
|
||||
public void Close(string source)
|
||||
{
|
||||
Stream stream;
|
||||
if (_openedStreams.TryGetValue(source, out stream))
|
||||
{
|
||||
stream.Close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var keyValuePair in _openedStreams)
|
||||
{
|
||||
keyValuePair.Value.Close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Consolidators;
|
||||
using QuantConnect.Data.Market;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Specifies a piece of processing that should be performed against a source file
|
||||
/// </summary>
|
||||
public interface IDataProcessor : IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Invoked for each piece of data from the source file
|
||||
/// </summary>
|
||||
/// <param name="data">The data to be processed</param>
|
||||
void Process(IBaseData data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides methods for creating data processor stacks
|
||||
/// </summary>
|
||||
public static class DataProcessor
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a new data processor that will filter in input data before piping it into the specified processor
|
||||
/// </summary>
|
||||
public static IDataProcessor FilteredBy(this IDataProcessor processor, Func<IBaseData, bool> predicate)
|
||||
{
|
||||
return new FilteredDataProcessor(processor, predicate);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a data processor that will aggregate and zip the requested resolutions of data
|
||||
/// </summary>
|
||||
public static IDataProcessor Zip(string dataDirectory, IEnumerable<Resolution> resolutions, TickType tickType, bool sourceIsTick)
|
||||
{
|
||||
var set = resolutions.ToHashSet();
|
||||
|
||||
var root = new PipeDataProcessor();
|
||||
|
||||
// only filter tick sources
|
||||
var stack = !sourceIsTick ? root
|
||||
: (IDataProcessor) new FilteredDataProcessor(root, x => ((Tick) x).TickType == tickType);
|
||||
|
||||
if (set.Contains(Resolution.Tick))
|
||||
{
|
||||
// tick is filtered via trade/quote
|
||||
var tick = new CsvDataProcessor(dataDirectory, Resolution.Tick, tickType);
|
||||
root.PipeTo(tick);
|
||||
}
|
||||
if (set.Contains(Resolution.Second))
|
||||
{
|
||||
root = AddResolution(dataDirectory, tickType, root, Resolution.Second, sourceIsTick);
|
||||
sourceIsTick = false;
|
||||
}
|
||||
if (set.Contains(Resolution.Minute))
|
||||
{
|
||||
root = AddResolution(dataDirectory, tickType, root, Resolution.Minute, sourceIsTick);
|
||||
sourceIsTick = false;
|
||||
}
|
||||
if (set.Contains(Resolution.Hour))
|
||||
{
|
||||
root = AddResolution(dataDirectory, tickType, root, Resolution.Hour, sourceIsTick);
|
||||
sourceIsTick = false;
|
||||
}
|
||||
if (set.Contains(Resolution.Daily))
|
||||
{
|
||||
AddResolution(dataDirectory, tickType, root, Resolution.Daily, sourceIsTick);
|
||||
}
|
||||
return stack;
|
||||
}
|
||||
|
||||
private static PipeDataProcessor AddResolution(string dataDirectory, TickType tickType, PipeDataProcessor root, Resolution resolution, bool sourceIsTick)
|
||||
{
|
||||
var second = new CsvDataProcessor(dataDirectory, resolution, tickType);
|
||||
var secondRoot = new PipeDataProcessor(second);
|
||||
var aggregator = new ConsolidatorDataProcessor(secondRoot, data => CreateConsolidator(resolution, tickType, data, sourceIsTick));
|
||||
root.PipeTo(aggregator);
|
||||
return secondRoot;
|
||||
}
|
||||
|
||||
private static IDataConsolidator CreateConsolidator(Resolution resolution, TickType tickType, IBaseData data, bool sourceIsTick)
|
||||
{
|
||||
var securityType = data.Symbol.ID.SecurityType;
|
||||
switch (securityType)
|
||||
{
|
||||
case SecurityType.Base:
|
||||
case SecurityType.Equity:
|
||||
case SecurityType.Cfd:
|
||||
case SecurityType.Forex:
|
||||
return new TickConsolidator(resolution.ToTimeSpan());
|
||||
|
||||
case SecurityType.Option:
|
||||
if (tickType == TickType.Trade)
|
||||
{
|
||||
return sourceIsTick
|
||||
? new TickConsolidator(resolution.ToTimeSpan())
|
||||
: (IDataConsolidator) new TradeBarConsolidator(resolution.ToTimeSpan());
|
||||
}
|
||||
if (tickType == TickType.Quote)
|
||||
{
|
||||
return sourceIsTick
|
||||
? new TickQuoteBarConsolidator(resolution.ToTimeSpan())
|
||||
: (IDataConsolidator) new QuoteBarConsolidator(resolution.ToTimeSpan());
|
||||
}
|
||||
break;
|
||||
}
|
||||
throw new NotImplementedException("Consolidator creation is not defined for " + securityType + " " + tickType);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Exchange Info Downloader Interface for pulling data from a remote source.
|
||||
/// </summary>
|
||||
public interface IExchangeInfoDownloader
|
||||
{
|
||||
/// <summary>
|
||||
/// Market
|
||||
/// </summary>
|
||||
string Market { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Get exchange info coma-separated data
|
||||
/// </summary>
|
||||
/// <returns>Enumerable of exchange info for this market</returns>
|
||||
IEnumerable<string> Get();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using QuantConnect.Data;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a type capable of accepting a stream and parsing it into an enumerable of data
|
||||
/// </summary>
|
||||
public interface IStreamParser : IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Parses the specified input stream into an enumerable of data
|
||||
/// </summary>
|
||||
/// <param name="source">The source of the stream</param>
|
||||
/// <param name="stream">The input stream to be parsed</param>
|
||||
/// <returns>An enumerable of base data</returns>
|
||||
IEnumerable<BaseData> Parse(string source, Stream stream);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Defines how to open/close a source file
|
||||
/// </summary>
|
||||
public interface IStreamProvider : IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Opens the specified source as read to be consumed stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be opened</param>
|
||||
/// <returns>The stream representing the specified source</returns>
|
||||
IEnumerable<Stream> Open(string source);
|
||||
|
||||
/// <summary>
|
||||
/// Closes the specified source file stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be closed</param>
|
||||
void Close(string source);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides factor method for creating an <see cref="IStreamProvider"/> from a file name
|
||||
/// </summary>
|
||||
public static class StreamProvider
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a new <see cref="IStreamProvider"/> capable of reading a file with the specified extenson
|
||||
/// </summary>
|
||||
/// <param name="extension">The file extension</param>
|
||||
/// <returns>A new stream provider capable of reading files with the specified extension</returns>
|
||||
public static IStreamProvider ForExtension(string extension)
|
||||
{
|
||||
var ext = Path.GetExtension(extension);
|
||||
if (ext == ".zip")
|
||||
{
|
||||
return new ZipStreamProvider();
|
||||
}
|
||||
if (ext == ".bz2")
|
||||
{
|
||||
return new Bz2StreamProvider();
|
||||
}
|
||||
if (ext == ".gz")
|
||||
{
|
||||
return new GzipStreamProvider();
|
||||
}
|
||||
|
||||
return new FileStreamProvider();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
*
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Logging;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using ZipEntry = Ionic.Zip.ZipEntry;
|
||||
|
||||
namespace QuantConnect.ToolBox.KaikoDataConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Decompress single entry from Kaiko crypto raw data.
|
||||
/// </summary>
|
||||
public class KaikoDataReader
|
||||
{
|
||||
private Symbol _symbol;
|
||||
private TickType _tickType;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="KaikoDataReader"/> class.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol.</param>
|
||||
/// <param name="tickType">Type of the tick.</param>
|
||||
public KaikoDataReader(Symbol symbol, TickType tickType)
|
||||
{
|
||||
_symbol = symbol;
|
||||
_tickType = tickType;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the ticks from Kaiko file zip entry.
|
||||
/// </summary>
|
||||
/// <param name="zipEntry">The zip entry.</param>
|
||||
/// <returns></returns>
|
||||
public IEnumerable<BaseData> GetTicksFromZipEntry(ZipEntry zipEntry)
|
||||
{
|
||||
var rawData = GetRawDataStreamFromEntry(zipEntry);
|
||||
return _tickType == TickType.Trade ? ParseKaikoTradeFile(rawData) : ParseKaikoQuoteFile(rawData);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the raw data from entry.
|
||||
/// </summary>
|
||||
/// <param name="zipEntry">The zip entry.</param>
|
||||
/// <returns>IEnumerable with the zip entry content.</returns>
|
||||
private IEnumerable<string> GetRawDataStreamFromEntry(ZipEntry zipEntry)
|
||||
{
|
||||
using (var outerStream = new StreamReader(zipEntry.OpenReader()))
|
||||
using (var innerStream = new GZipStream(outerStream.BaseStream, CompressionMode.Decompress))
|
||||
using (var outputStream = new StreamReader(innerStream))
|
||||
{
|
||||
string line;
|
||||
while ((line = outputStream.ReadLine()) != null)
|
||||
{
|
||||
yield return line;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse order book information for Kaiko data files
|
||||
/// </summary>
|
||||
/// <param name="rawDataLines">The raw data lines.</param>
|
||||
/// <returns>
|
||||
/// IEnumerable of ticks representing the Kaiko data
|
||||
/// </returns>
|
||||
private IEnumerable<Tick> ParseKaikoQuoteFile(IEnumerable<string> rawDataLines)
|
||||
{
|
||||
var headerLine = rawDataLines.First();
|
||||
var headerCsv = headerLine.ToCsv();
|
||||
var typeColumn = headerCsv.FindIndex(x => x == "type");
|
||||
var dateColumn = headerCsv.FindIndex(x => x == "date");
|
||||
var priceColumn = headerCsv.FindIndex(x => x == "price");
|
||||
var quantityColumn = headerCsv.FindIndex(x => x == "amount");
|
||||
|
||||
long currentEpoch = 0;
|
||||
var currentEpochTicks = new List<KaikoTick>();
|
||||
|
||||
foreach (var line in rawDataLines.Skip(1))
|
||||
{
|
||||
if (line == null || string.IsNullOrEmpty(line)) continue;
|
||||
|
||||
var lineParts = line.Split(',');
|
||||
|
||||
var tickEpoch = Parse.Long(lineParts[dateColumn]);
|
||||
|
||||
decimal quantity;
|
||||
decimal price;
|
||||
|
||||
try
|
||||
{
|
||||
quantity = ParseScientificNotationToDecimal(lineParts, quantityColumn);
|
||||
price = ParseScientificNotationToDecimal(lineParts, priceColumn);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error($"KaikoDataConverter.ParseKaikoQuoteFile(): Raw data corrupted. Line {string.Join(" ", lineParts)}, Exception {ex}");
|
||||
continue;
|
||||
}
|
||||
|
||||
var currentTick = new KaikoTick
|
||||
{
|
||||
TickType = TickType.Quote,
|
||||
Time = Time.UnixMillisecondTimeStampToDateTime(tickEpoch),
|
||||
Quantity = quantity,
|
||||
Value = price,
|
||||
OrderDirection = lineParts[typeColumn]
|
||||
};
|
||||
|
||||
if (currentEpoch != tickEpoch)
|
||||
{
|
||||
var quoteTick = CreateQuoteTick(Time.UnixMillisecondTimeStampToDateTime(currentEpoch), currentEpochTicks);
|
||||
|
||||
if (quoteTick != null) yield return quoteTick;
|
||||
|
||||
currentEpochTicks.Clear();
|
||||
currentEpoch = tickEpoch;
|
||||
}
|
||||
|
||||
currentEpochTicks.Add(currentTick);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Take a minute snapshot of order book information and make a single Lean quote tick
|
||||
/// </summary>
|
||||
/// <param name="date">The data being processed</param>
|
||||
/// <param name="currentEpcohTicks">The snapshot of bid/ask Kaiko data</param>
|
||||
/// <returns>A single Lean quote tick</returns>
|
||||
private Tick CreateQuoteTick(DateTime date, List<KaikoTick> currentEpcohTicks)
|
||||
{
|
||||
// lowest ask
|
||||
var bestAsk = currentEpcohTicks.Where(x => x.OrderDirection == "a")
|
||||
.OrderBy(x => x.Value)
|
||||
.FirstOrDefault();
|
||||
|
||||
// highest bid
|
||||
var bestBid = currentEpcohTicks.Where(x => x.OrderDirection == "b")
|
||||
.OrderByDescending(x => x.Value)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (bestAsk == null && bestBid == null)
|
||||
{
|
||||
// Did not have enough data to create a tick
|
||||
return null;
|
||||
}
|
||||
|
||||
var tick = new Tick()
|
||||
{
|
||||
Symbol = _symbol,
|
||||
Time = date,
|
||||
TickType = TickType.Quote
|
||||
};
|
||||
|
||||
if (bestBid != null)
|
||||
{
|
||||
tick.BidPrice = bestBid.Price;
|
||||
tick.BidSize = bestBid.Quantity;
|
||||
}
|
||||
|
||||
if (bestAsk != null)
|
||||
{
|
||||
tick.AskPrice = bestAsk.Price;
|
||||
tick.AskSize = bestAsk.Quantity;
|
||||
}
|
||||
|
||||
return tick;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse a kaiko trade file
|
||||
/// </summary>
|
||||
/// <param name="unzippedFile">The path to the unzipped file</param>
|
||||
/// <returns>Lean Ticks in the Kaiko file</returns>
|
||||
private IEnumerable<Tick> ParseKaikoTradeFile(IEnumerable<string> rawDataLines)
|
||||
{
|
||||
var headerLine = rawDataLines.First();
|
||||
var headerCsv = headerLine.ToCsv();
|
||||
var dateColumn = headerCsv.FindIndex(x => x == "date");
|
||||
var priceColumn = headerCsv.FindIndex(x => x == "price");
|
||||
var quantityColumn = headerCsv.FindIndex(x => x == "amount");
|
||||
|
||||
foreach (var line in rawDataLines.Skip(1))
|
||||
{
|
||||
if (line == null || string.IsNullOrEmpty(line)) continue;
|
||||
|
||||
var lineParts = line.Split(',');
|
||||
|
||||
decimal quantity;
|
||||
decimal price;
|
||||
|
||||
try
|
||||
{
|
||||
quantity = ParseScientificNotationToDecimal(lineParts, quantityColumn);
|
||||
price = ParseScientificNotationToDecimal(lineParts, priceColumn);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error($"KaikoDataConverter.ParseKaikoTradeFile(): Raw data corrupted. Line {string.Join(" ", lineParts)}, Exception {ex}");
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return new Tick
|
||||
{
|
||||
Symbol = _symbol,
|
||||
TickType = TickType.Trade,
|
||||
Time = Time.UnixMillisecondTimeStampToDateTime(Parse.Long(lineParts[dateColumn])),
|
||||
Quantity = quantity,
|
||||
Value = price
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse the quantity field of the kaiko ticks - can sometimes be expressed in scientific notation
|
||||
/// </summary>
|
||||
/// <param name="lineParts">The line from the Kaiko file</param>
|
||||
/// <param name="column">The index of the quantity column </param>
|
||||
/// <returns>The quantity as a decimal</returns>
|
||||
private static decimal ParseScientificNotationToDecimal(string[] lineParts, int column)
|
||||
{
|
||||
var value = lineParts[column];
|
||||
if (value.Contains('e', StringComparison.InvariantCulture))
|
||||
{
|
||||
return Parse.Decimal(value, NumberStyles.Float);
|
||||
}
|
||||
|
||||
return lineParts[column].ConvertInvariant<decimal>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Simple class to add order direction to Tick
|
||||
/// used for aggregating Kaiko order book snapshots
|
||||
/// </summary>
|
||||
private class KaikoTick : Tick
|
||||
{
|
||||
public string OrderDirection { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
*
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using QuantConnect.Data;
|
||||
using System.Diagnostics;
|
||||
using QuantConnect.Logging;
|
||||
using System.Collections.Generic;
|
||||
using ZipFile = Ionic.Zip.ZipFile;
|
||||
|
||||
namespace QuantConnect.ToolBox.KaikoDataConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Console application for converting a single day of Kaiko data into Lean data format for high resolutions (tick, second and minute)
|
||||
/// </summary>
|
||||
public static class KaikoDataConverterProgram
|
||||
{
|
||||
/// <summary>
|
||||
/// Kaiko data converter entry point.
|
||||
/// </summary>
|
||||
/// <param name="sourceDirectory">The source directory where all Kaiko zipped files are stored..</param>
|
||||
/// <param name="date">The date to process.</param>
|
||||
/// <param name="exchange">The exchange to process, if not defined, all exchanges will be processed.</param>
|
||||
/// <exception cref="ArgumentException">Source folder does not exists.</exception>
|
||||
/// <remarks>This converter will process automatically data for every exchange and for both tick types if the raw data files are available in the sourceDirectory</remarks>
|
||||
public static void KaikoDataConverter(string sourceDirectory, string date, string exchange = "")
|
||||
{
|
||||
var timer = new Stopwatch();
|
||||
timer.Start();
|
||||
var folderPath = new DirectoryInfo(sourceDirectory);
|
||||
if (!folderPath.Exists)
|
||||
{
|
||||
throw new ArgumentException($"Source folder {folderPath.FullName} not found");
|
||||
}
|
||||
|
||||
exchange = !string.IsNullOrEmpty(exchange) && exchange.ToLowerInvariant() == "gdax" ? "coinbase" : exchange;
|
||||
|
||||
var processingDate = Parse.DateTimeExact(date, DateFormat.EightCharacter);
|
||||
foreach (var filePath in folderPath.EnumerateFiles("*.zip"))
|
||||
{
|
||||
// Do not process exchanges other than the one defined.
|
||||
if (!string.IsNullOrEmpty(exchange) && !filePath.Name.ToLowerInvariant().Contains(exchange.ToLowerInvariant())) continue;
|
||||
|
||||
Log.Trace($"KaikoDataConverter(): Starting data conversion from source {filePath.Name} for date {processingDate:yyyy_MM_dd}... ");
|
||||
using (var zip = new ZipFile(filePath.FullName))
|
||||
{
|
||||
var targetDayEntries = zip.Entries.Where(e => e.FileName.Contains($"{processingDate.ToStringInvariant("yyyy_MM_dd")}")).ToList();
|
||||
|
||||
if (!targetDayEntries.Any())
|
||||
{
|
||||
Log.Error($"KaikoDataConverter(): Date {processingDate:yyyy_MM_dd} not found in source file {filePath.FullName}.");
|
||||
}
|
||||
|
||||
foreach (var zipEntry in targetDayEntries)
|
||||
{
|
||||
var nameParts = zipEntry.FileName.Split(new char[] { '/' }).Last().Split(new char[] { '_' });
|
||||
var market = nameParts[0] == "Coinbase" ? "GDAX" : nameParts[0];
|
||||
var ticker = nameParts[1];
|
||||
var tickType = nameParts[2] == "trades" ? TickType.Trade : TickType.Quote;
|
||||
var symbol = Symbol.Create(ticker, SecurityType.Crypto, market);
|
||||
|
||||
Log.Trace($"KaikoDataConverter(): Processing {symbol.Value} {tickType}");
|
||||
|
||||
// Generate ticks from raw data and write them to disk
|
||||
|
||||
var reader = new KaikoDataReader(symbol, tickType);
|
||||
var ticks = reader.GetTicksFromZipEntry(zipEntry);
|
||||
|
||||
var writer = new LeanDataWriter(Resolution.Tick, symbol, Globals.DataFolder, tickType);
|
||||
writer.Write(ticks);
|
||||
|
||||
try
|
||||
{
|
||||
Log.Trace($"KaikoDataConverter(): Starting consolidation for {symbol.Value} {tickType}");
|
||||
List<TickAggregator> consolidators = new List<TickAggregator>();
|
||||
|
||||
if (tickType == TickType.Trade)
|
||||
{
|
||||
consolidators.AddRange(new[]
|
||||
{
|
||||
new TradeTickAggregator(Resolution.Second),
|
||||
new TradeTickAggregator(Resolution.Minute),
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
consolidators.AddRange(new[]
|
||||
{
|
||||
new QuoteTickAggregator(Resolution.Second),
|
||||
new QuoteTickAggregator(Resolution.Minute),
|
||||
});
|
||||
}
|
||||
|
||||
foreach (var tick in ticks)
|
||||
{
|
||||
foreach (var consolidator in consolidators)
|
||||
{
|
||||
consolidator.Update(tick);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var consolidator in consolidators)
|
||||
{
|
||||
writer = new LeanDataWriter(consolidator.Resolution, symbol, Globals.DataFolder, tickType);
|
||||
writer.Write(consolidator.Flush());
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Error($"KaikoDataConverter(): Error processing entry {zipEntry.FileName}. Exception {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Log.Trace($"KaikoDataConverter(): Finished in {timer.Elapsed}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
using System.IO;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// This class wraps a <see cref="StreamWriter"/> so that the StreamWriter is only
|
||||
/// instantiated until WriteLine() is called. This ensures that the file the StreamWriter is
|
||||
/// writing to is only created if something is written to it. A StreamWriter will create a empty file
|
||||
/// as soon as it is instantiated.
|
||||
/// </summary>
|
||||
public class LazyStreamWriter
|
||||
{
|
||||
private StreamWriter _streamWriter;
|
||||
private readonly string _path;
|
||||
|
||||
/// <summary>
|
||||
/// Constructor for the <see cref="LazyStreamWriter"/>
|
||||
/// </summary>
|
||||
/// <param name="path">Path to the file that should be created</param>
|
||||
public LazyStreamWriter(string path)
|
||||
{
|
||||
_path = path;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wraps the WriteLine method of the StreamWriter.
|
||||
/// </summary>
|
||||
/// <param name="line">The line to write</param>
|
||||
/// <remarks>Will instantiate the StreamWriter if this is the first time this method is called</remarks>
|
||||
public void WriteLine(string line)
|
||||
{
|
||||
PrepareStreamWriter();
|
||||
|
||||
_streamWriter.WriteLine(line);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wraps the <see cref="StreamWriter.Flush()"/> method
|
||||
/// </summary>
|
||||
public void Flush()
|
||||
{
|
||||
if (_streamWriter != null)
|
||||
{
|
||||
_streamWriter.Flush();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wraps the <see cref="StreamWriter.Close()"/> method
|
||||
/// </summary>
|
||||
public void Close()
|
||||
{
|
||||
if (_streamWriter != null)
|
||||
{
|
||||
_streamWriter.Close();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the StreamWriter is instantiated. If not, it will instantiate the StreamWriter
|
||||
/// </summary>
|
||||
private void PrepareStreamWriter()
|
||||
{
|
||||
if (_streamWriter == null)
|
||||
{
|
||||
_streamWriter = new StreamWriter(_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Ionic.Zip;
|
||||
using NodaTime;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// This class reads data directly from disk and returns the data without the data
|
||||
/// entering the Lean data enumeration stack
|
||||
/// </summary>
|
||||
public class LeanDataReader
|
||||
{
|
||||
private readonly DateTime _date;
|
||||
private readonly string _zipPath;
|
||||
private readonly string _zipentry;
|
||||
private readonly SubscriptionDataConfig _config;
|
||||
|
||||
/// <summary>
|
||||
/// The LeanDataReader constructor
|
||||
/// </summary>
|
||||
/// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
|
||||
/// <param name="symbol">The <see cref="Symbol"/> that will be read</param>
|
||||
/// <param name="resolution">The <see cref="Resolution"/> that will be read</param>
|
||||
/// <param name="date">The <see cref="DateTime"/> that will be read</param>
|
||||
/// <param name="dataFolder">The root data folder</param>
|
||||
public LeanDataReader(SubscriptionDataConfig config, Symbol symbol, Resolution resolution, DateTime date, string dataFolder)
|
||||
{
|
||||
_date = date;
|
||||
_zipPath = LeanData.GenerateZipFilePath(dataFolder, symbol, date, resolution, config.TickType);
|
||||
_zipentry = LeanData.GenerateZipEntryName(symbol, date, resolution, config.TickType);
|
||||
_config = config;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize a instance of LeanDataReader from a path to a zipped data file.
|
||||
/// It also supports declaring the zip entry CSV file for options and futures.
|
||||
/// </summary>
|
||||
/// <param name="filepath">Absolute or relative path to a zipped data file, optionally the zip entry file can be declared by using '#' as separator.</param>
|
||||
/// <example>
|
||||
/// var dataReader = LeanDataReader("../relative/path/to/file.zip")
|
||||
/// var dataReader = LeanDataReader("absolute/path/to/file.zip#zipEntry.csv")
|
||||
/// </example>
|
||||
public LeanDataReader(string filepath)
|
||||
{
|
||||
Symbol symbol;
|
||||
DateTime date;
|
||||
Resolution resolution;
|
||||
string zipEntry = null;
|
||||
|
||||
var isFutureOrOption = filepath.Contains('#', StringComparison.InvariantCulture);
|
||||
|
||||
if (isFutureOrOption)
|
||||
{
|
||||
zipEntry = filepath.Split('#')[1];
|
||||
filepath = filepath.Split('#')[0];
|
||||
}
|
||||
|
||||
var fileInfo = new FileInfo(filepath);
|
||||
if (!LeanData.TryParsePath(fileInfo.FullName, out symbol, out date, out resolution, out var tickType, out var dataType))
|
||||
{
|
||||
throw new ArgumentException($"File {filepath} cannot be parsed.");
|
||||
}
|
||||
|
||||
if (isFutureOrOption)
|
||||
{
|
||||
symbol = LeanData.ReadSymbolFromZipEntry(symbol, resolution, zipEntry);
|
||||
}
|
||||
|
||||
var marketHoursDataBase = MarketHoursDatabase.FromDataFolder();
|
||||
var dataTimeZone = marketHoursDataBase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
|
||||
var exchangeTimeZone = marketHoursDataBase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
|
||||
|
||||
var config = new SubscriptionDataConfig(dataType, symbol, resolution,
|
||||
dataTimeZone, exchangeTimeZone, tickType: tickType,
|
||||
fillForward: false, extendedHours: true, isInternalFeed: true);
|
||||
|
||||
_date = date;
|
||||
_zipPath = fileInfo.FullName;
|
||||
_zipentry = zipEntry;
|
||||
_config = config;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enumerate over the tick zip file and return a list of BaseData.
|
||||
/// </summary>
|
||||
/// <returns>IEnumerable of ticks</returns>
|
||||
public IEnumerable<BaseData> Parse()
|
||||
{
|
||||
if (!File.Exists(_zipPath))
|
||||
{
|
||||
Log.Error($"LeanDataReader.Parse(): File does not exist: {_zipPath}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
var factory = (BaseData) ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]);
|
||||
|
||||
if (_config.Type.ImplementsStreamReader())
|
||||
{
|
||||
using (var zip = new ZipFile(_zipPath))
|
||||
{
|
||||
foreach (var zipEntry in zip.Where(x => _zipentry == null || string.Equals(x.FileName, _zipentry, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
// we get the contract symbol from the zip entry if not already provided with the zip entry
|
||||
var symbol = _config.Symbol;
|
||||
if(_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption()))
|
||||
{
|
||||
symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, zipEntry.FileName);
|
||||
}
|
||||
using (var entryReader = new StreamReader(zipEntry.OpenReader()))
|
||||
{
|
||||
while (!entryReader.EndOfStream)
|
||||
{
|
||||
var dataPoint = factory.Reader(_config, entryReader, _date, false);
|
||||
dataPoint.Symbol = symbol;
|
||||
yield return dataPoint;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// for futures and options if no entry was provided we just read all
|
||||
else if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption()))
|
||||
{
|
||||
foreach (var entries in Compression.Unzip(_zipPath))
|
||||
{
|
||||
// we get the contract symbol from the zip entry
|
||||
var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key);
|
||||
foreach (var line in entries.Value)
|
||||
{
|
||||
var dataPoint = factory.Reader(_config, line, _date, false);
|
||||
dataPoint.Symbol = symbol;
|
||||
yield return dataPoint;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ZipFile zipFile;
|
||||
using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile))
|
||||
{
|
||||
if (unzipped == null)
|
||||
yield break;
|
||||
string line;
|
||||
while ((line = unzipped.ReadLine()) != null)
|
||||
{
|
||||
yield return factory.Reader(_config, line, _date, false);
|
||||
}
|
||||
}
|
||||
zipFile.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the data time zone
|
||||
/// </summary>
|
||||
/// <returns><see cref="NodaTime.DateTimeZone"/> representing the data timezone</returns>
|
||||
public DateTimeZone GetDataTimeZone()
|
||||
{
|
||||
return _config.DataTimeZone;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the Exchange time zone
|
||||
/// </summary>
|
||||
/// <returns><see cref="NodaTime.DateTimeZone"/> representing the exchange timezone</returns>
|
||||
public DateTimeZone GetExchangeTimeZone()
|
||||
{
|
||||
return _config.ExchangeTimeZone;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a single instrument as listed in the file instruments.txt
|
||||
/// </summary>
|
||||
public class LeanInstrument
|
||||
{
|
||||
/// <summary>
|
||||
/// The symbol of the instrument
|
||||
/// </summary>
|
||||
public string Symbol { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The name/description of the instrument
|
||||
/// </summary>
|
||||
public string Name { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The instrument type
|
||||
/// </summary>
|
||||
public SecurityType Type { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The point value
|
||||
/// </summary>
|
||||
public double PointValue { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IStreamParser"/> that reads files in the lean format
|
||||
/// </summary>
|
||||
public class LeanParser : IStreamParser
|
||||
{
|
||||
/// <summary>
|
||||
/// Parses the specified input stream into an enumerable of data
|
||||
/// </summary>
|
||||
/// <param name="source">The source file corresponding the the stream</param>
|
||||
/// <param name="stream">The input stream to be parsed</param>
|
||||
/// <returns>An enumerable of base data</returns>
|
||||
public IEnumerable<BaseData> Parse(string source, Stream stream)
|
||||
{
|
||||
var pathComponents = LeanDataPathComponents.Parse(source);
|
||||
var tickType = pathComponents.Filename.ToLowerInvariant().Contains("_trade")
|
||||
? TickType.Trade
|
||||
: TickType.Quote;
|
||||
|
||||
var dataType = GetDataType(pathComponents.SecurityType, pathComponents.Resolution, tickType);
|
||||
var factory = (BaseData) Activator.CreateInstance(dataType);
|
||||
|
||||
// ignore time zones here, i.e, we're going to emit data in the data time zone
|
||||
var config = new SubscriptionDataConfig(dataType, pathComponents.Symbol, pathComponents.Resolution, TimeZones.Utc, TimeZones.Utc, false, true, false);
|
||||
using (var reader = new StreamReader(stream))
|
||||
{
|
||||
string line;
|
||||
while ((line = reader.ReadLine()) != null)
|
||||
{
|
||||
yield return factory.Reader(config, line, pathComponents.Date, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
private Type GetDataType(SecurityType securityType, Resolution resolution, TickType tickType)
|
||||
{
|
||||
if (resolution == Resolution.Tick)
|
||||
{
|
||||
return typeof (Tick);
|
||||
}
|
||||
|
||||
switch (securityType)
|
||||
{
|
||||
case SecurityType.Base:
|
||||
case SecurityType.Equity:
|
||||
return typeof (TradeBar);
|
||||
|
||||
case SecurityType.Cfd:
|
||||
case SecurityType.Forex:
|
||||
case SecurityType.Crypto:
|
||||
return typeof (QuoteBar);
|
||||
|
||||
case SecurityType.Option:
|
||||
case SecurityType.FutureOption:
|
||||
case SecurityType.IndexOption:
|
||||
if (tickType == TickType.Trade) return typeof (TradeBar);
|
||||
if (tickType == TickType.Quote) return typeof (QuoteBar);
|
||||
break;
|
||||
}
|
||||
var parameters = string.Join(" | ", securityType, resolution, tickType);
|
||||
throw new NotImplementedException("LeanParser.GetDataType has not yet implemented: " + parameters);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using QuantConnect.Data;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IDataProcessor"/> that simply forwards all
|
||||
/// received data to other attached processors
|
||||
/// </summary>
|
||||
public class PipeDataProcessor : IDataProcessor
|
||||
{
|
||||
private readonly HashSet<IDataProcessor> _processors;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PipeDataProcessor"/> class
|
||||
/// </summary>
|
||||
/// <param name="processors">The processors to pipe the data to</param>
|
||||
public PipeDataProcessor(IEnumerable<IDataProcessor> processors)
|
||||
{
|
||||
_processors = processors.ToHashSet();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PipeDataProcessor"/> class
|
||||
/// </summary>
|
||||
/// <param name="processors">The processors to pipe the data to</param>
|
||||
public PipeDataProcessor(params IDataProcessor[] processors)
|
||||
: this((IEnumerable<IDataProcessor>)processors)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds the specified processor to the output pipe
|
||||
/// </summary>
|
||||
/// <param name="processor">Processor to receive data from this pipe</param>
|
||||
public void PipeTo(IDataProcessor processor)
|
||||
{
|
||||
_processors.Add(processor);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invoked for each piece of data from the source file
|
||||
/// </summary>
|
||||
/// <param name="data">The data to be processed</param>
|
||||
public void Process(IBaseData data)
|
||||
{
|
||||
foreach (var processor in _processors)
|
||||
{
|
||||
processor.Process(data);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var processor in _processors)
|
||||
{
|
||||
processor.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
using QuantConnect.Configuration;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.ToolBox.AlgoSeekFuturesConverter;
|
||||
using QuantConnect.ToolBox.CoarseUniverseGenerator;
|
||||
using QuantConnect.ToolBox.KaikoDataConverter;
|
||||
using QuantConnect.ToolBox.RandomDataGenerator;
|
||||
using QuantConnect.Util;
|
||||
using System;
|
||||
using System.IO;
|
||||
using static QuantConnect.Configuration.ApplicationParser;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
public class Program
|
||||
{
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
Log.DebuggingEnabled = Config.GetBool("debug-mode");
|
||||
var destinationDir = Config.Get("results-destination-folder");
|
||||
if (!string.IsNullOrEmpty(destinationDir))
|
||||
{
|
||||
Directory.CreateDirectory(destinationDir);
|
||||
Log.FilePath = Path.Combine(destinationDir, "log.txt");
|
||||
}
|
||||
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName<ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
|
||||
|
||||
var optionsObject = ToolboxArgumentParser.ParseArguments(args);
|
||||
if (optionsObject.Count == 0)
|
||||
{
|
||||
PrintMessageAndExit();
|
||||
}
|
||||
|
||||
var dataProvider
|
||||
= Composer.Instance.GetExportedValueByTypeName<IDataProvider>(Config.Get("data-provider", "DefaultDataProvider"));
|
||||
var mapFileProvider
|
||||
= Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider"));
|
||||
var factorFileProvider
|
||||
= Composer.Instance.GetExportedValueByTypeName<IFactorFileProvider>(Config.Get("factor-file-provider", "LocalDiskFactorFileProvider"));
|
||||
|
||||
mapFileProvider.Initialize(dataProvider);
|
||||
factorFileProvider.Initialize(mapFileProvider, dataProvider);
|
||||
|
||||
var targetApp = GetParameterOrExit(optionsObject, "app").ToLowerInvariant();
|
||||
if (targetApp.Contains("download") || targetApp.EndsWith("dl"))
|
||||
{
|
||||
var fromDate = Parse.DateTimeExact(GetParameterOrExit(optionsObject, "from-date"), "yyyyMMdd-HH:mm:ss");
|
||||
var resolution = optionsObject.ContainsKey("resolution") ? optionsObject["resolution"].ToString() : "";
|
||||
var market = optionsObject.ContainsKey("market") ? optionsObject["market"].ToString() : "";
|
||||
var securityType = optionsObject.ContainsKey("security-type") ? optionsObject["security-type"].ToString() : "";
|
||||
var tickers = ToolboxArgumentParser.GetTickers(optionsObject);
|
||||
var toDate = optionsObject.ContainsKey("to-date")
|
||||
? Parse.DateTimeExact(optionsObject["to-date"].ToString(), "yyyyMMdd-HH:mm:ss")
|
||||
: DateTime.UtcNow;
|
||||
switch (targetApp)
|
||||
{
|
||||
default:
|
||||
PrintMessageAndExit(1, "ERROR: Unrecognized --app value");
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
switch (targetApp)
|
||||
{
|
||||
case "asfc":
|
||||
case "algoseekfuturesconverter":
|
||||
AlgoSeekFuturesProgram.AlgoSeekFuturesConverter(GetParameterOrExit(optionsObject, "date"));
|
||||
break;
|
||||
case "kdc":
|
||||
case "kaikodataconverter":
|
||||
KaikoDataConverterProgram.KaikoDataConverter(GetParameterOrExit(optionsObject, "source-dir"),
|
||||
GetParameterOrExit(optionsObject, "date"),
|
||||
GetParameterOrDefault(optionsObject, "exchange", string.Empty));
|
||||
break;
|
||||
case "cug":
|
||||
case "coarseuniversegenerator":
|
||||
CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
|
||||
break;
|
||||
case "rdg":
|
||||
case "randomdatagenerator":
|
||||
var tickers = ToolboxArgumentParser.GetTickers(optionsObject);
|
||||
RandomDataGeneratorProgram.RandomDataGenerator(
|
||||
GetParameterOrExit(optionsObject, "start"),
|
||||
GetParameterOrExit(optionsObject, "end"),
|
||||
GetParameterOrDefault(optionsObject, "symbol-count", null),
|
||||
GetParameterOrDefault(optionsObject, "market", null),
|
||||
GetParameterOrDefault(optionsObject, "security-type", "Equity"),
|
||||
GetParameterOrDefault(optionsObject, "resolution", "Minute"),
|
||||
GetParameterOrDefault(optionsObject, "data-density", "Dense"),
|
||||
GetParameterOrDefault(optionsObject, "include-coarse", "true"),
|
||||
GetParameterOrDefault(optionsObject, "quote-trade-ratio", "1"),
|
||||
GetParameterOrDefault(optionsObject, "random-seed", null),
|
||||
GetParameterOrDefault(optionsObject, "ipo-percentage", "5.0"),
|
||||
GetParameterOrDefault(optionsObject, "rename-percentage", "30.0"),
|
||||
GetParameterOrDefault(optionsObject, "splits-percentage", "15.0"),
|
||||
GetParameterOrDefault(optionsObject, "dividends-percentage", "60.0"),
|
||||
GetParameterOrDefault(optionsObject, "dividend-every-quarter-percentage", "30.0"),
|
||||
GetParameterOrDefault(optionsObject, "option-price-engine", "BaroneAdesiWhaleyApproximationEngine"),
|
||||
GetParameterOrDefault(optionsObject, "volatility-model-resolution", "Daily"),
|
||||
GetParameterOrDefault(optionsObject, "chain-symbol-count", "1"),
|
||||
tickers
|
||||
);
|
||||
break;
|
||||
|
||||
default:
|
||||
PrintMessageAndExit(1, "ERROR: Unrecognized --app value");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("QuantConnect.ToolBox")]
|
||||
[assembly: AssemblyProduct("QuantConnect.ToolBox")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("5c3d0688-07ac-4cd4-8f2e-e74ebcf32a88")]
|
||||
[assembly: InternalsVisibleTo("QuantConnect.Tests")]
|
||||
@@ -0,0 +1,56 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<OutputType>Exe</OutputType>
|
||||
<RootNamespace>QuantConnect.ToolBox</RootNamespace>
|
||||
<AssemblyName>QuantConnect.ToolBox</AssemblyName>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AnalysisMode>AllEnabledByDefault</AnalysisMode>
|
||||
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
|
||||
<OutputPath>bin\$(Configuration)\</OutputPath>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
|
||||
<Description>QuantConnect LEAN Engine: ToolBox Project - A collection of data downloaders and converters</Description>
|
||||
<NoWarn>CA1062</NoWarn>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup>
|
||||
<StartupObject>QuantConnect.ToolBox.Program</StartupObject>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup>
|
||||
<PackageLicenseFile>LICENSE</PackageLicenseFile>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="..\LICENSE">
|
||||
<Pack>True</Pack>
|
||||
<PackagePath></PackagePath>
|
||||
</None>
|
||||
<None Include="AlgoSeekFuturesConverter\AlgoSeek.US.Futures.PriceMultipliers.1.1.csv">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<Content Include="CoarseUniverseGenerator\blacklisted-tickers.txt">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Engine\QuantConnect.Lean.Engine.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,55 @@
|
||||
 Lean Data ToolBox
|
||||
=========
|
||||
[](https://www.quantconnect.com/slack)
|
||||
|
||||
[Lean Home][1] | [Documentation][2] | [Download Lean][3]
|
||||
----------
|
||||
|
||||
## Introduction ##
|
||||
|
||||
Lean Engine is an open-source algorithmic trading engine built for easy strategy research, backtesting and live trading. We integrate with common data providers and brokerages so you can quickly deploy algorithmic trading strategies.
|
||||
|
||||
The ToolBox project is a command line program which wraps over 15 tools.
|
||||
|
||||
## Usage
|
||||
|
||||
Each tool requires a different set of parameters, the only **required argument is '--app='**, which defines the target tool and is case insensitive.
|
||||
|
||||
Help information is available using the '--help' parameter.
|
||||
|
||||
Example: --app=RandomDataGenerator --tickers=SPY,AAPL --resolution=Daily --from-date=yyyyMMdd-HH:mm:ss --to-date=yyyyMMdd-HH:mm:ss
|
||||
|
||||
#### Available downloaders
|
||||
|
||||
- **'--app='**
|
||||
- GDAXDownloader or GDAXDL
|
||||
- IBDownloader or IBDL
|
||||
- BitfinexDownloader or BFXDL
|
||||
- **'--from-date=yyyyMMdd-HH:mm:ss'** required
|
||||
- **'--tickers=SPY,AAPL,etc'** required
|
||||
- **'--resolution=Tick/Second/Minute/Hour/Daily/All'** required. **Case sensitive. Not all downloaders support all resolutions**, send empty for more information.
|
||||
- **'--to-date=yyyyMMdd-HH:mm:ss'** optional. If not provided 'DateTime.UtcNow' will be used
|
||||
|
||||
#### Available Converters
|
||||
|
||||
- **'--app='**
|
||||
- AlgoSeekFuturesConverter or ASFC
|
||||
- **'--date=yyyyMMdd'** reference date.
|
||||
- AlgoSeekOptionsConverter or ASOC
|
||||
- **'--date=yyyyMMdd'** reference date.
|
||||
- KaikoDataConverter or KDC
|
||||
- **'--market='** the exchange the data represents.
|
||||
- **'--tick-type=Quote/Trade'** the tick type being processed. Case insensitive.
|
||||
- **'--source-dir='** path to the raw Kaiko data.
|
||||
- QuantQuoteConverter or QQC
|
||||
- **'--source-dir='** directory where your QuantQuote order is extracted.
|
||||
- **'--destination-dir='** directory where Lean Data is located "Lean/Data".
|
||||
- **'--resolution='** resolution of the QuantQuote data.
|
||||
|
||||
#### Other tools
|
||||
- **'--app='**
|
||||
- CoarseUniverseGenerator or CUG
|
||||
|
||||
[1]: https://lean.quantconnect.com "Lean Open Source Home Page"
|
||||
[2]: https://lean.quantconnect.com/docs "Lean Documentation"
|
||||
[3]: https://github.com/QuantConnect/Lean/archive/master.zip
|
||||
@@ -0,0 +1,253 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Util;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Provide the base symbol generator implementation
|
||||
/// </summary>
|
||||
public abstract class BaseSymbolGenerator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IRandomValueGenerator"/> instance producing random values for use in random data generation
|
||||
/// </summary>
|
||||
protected IRandomValueGenerator Random { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Settings of current random data generation run
|
||||
/// </summary>
|
||||
protected RandomDataGeneratorSettings Settings { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Exchange hours and raw data times zones in various markets
|
||||
/// </summary>
|
||||
protected MarketHoursDatabase MarketHoursDatabase { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Access to specific properties for various symbols
|
||||
/// </summary>
|
||||
protected SymbolPropertiesDatabase SymbolPropertiesDatabase { get; }
|
||||
|
||||
// used to prevent generating duplicates, but also caps
|
||||
// the memory allocated to checking for duplicates
|
||||
private readonly FixedSizeHashQueue<Symbol> _symbols;
|
||||
|
||||
/// <summary>
|
||||
/// Base constructor implementation for Symbol generator
|
||||
/// </summary>
|
||||
/// <param name="settings">random data generation run settings</param>
|
||||
/// <param name="random">produces random values for use in random data generation</param>
|
||||
protected BaseSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
|
||||
{
|
||||
Settings = settings;
|
||||
Random = random;
|
||||
_symbols = new FixedSizeHashQueue<Symbol>(1000);
|
||||
SymbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
|
||||
MarketHoursDatabase = MarketHoursDatabase.FromDataFolder();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a ad-hoc symbol generator depending on settings
|
||||
/// </summary>
|
||||
/// <param name="settings">random data generator settings</param>
|
||||
/// <param name="random">produces random values for use in random data generation</param>
|
||||
/// <returns>New symbol generator</returns>
|
||||
public static BaseSymbolGenerator Create(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
|
||||
{
|
||||
if (settings is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(settings), "Settings cannot be null or empty");
|
||||
}
|
||||
|
||||
if (random is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(random), "Randomizer cannot be null");
|
||||
}
|
||||
|
||||
switch (settings.SecurityType)
|
||||
{
|
||||
case SecurityType.Option:
|
||||
return new OptionSymbolGenerator(settings, random, 100m, 75m);
|
||||
|
||||
case SecurityType.Future:
|
||||
return new FutureSymbolGenerator(settings, random);
|
||||
|
||||
default:
|
||||
return new DefaultSymbolGenerator(settings, random);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates specified number of symbols
|
||||
/// </summary>
|
||||
/// <returns>Set of random symbols</returns>
|
||||
public IEnumerable<Symbol> GenerateRandomSymbols()
|
||||
{
|
||||
if (!Settings.Tickers.IsNullOrEmpty())
|
||||
{
|
||||
foreach (var symbol in Settings.Tickers.SelectMany(GenerateAsset))
|
||||
{
|
||||
yield return symbol;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (var i = 0; i < Settings.SymbolCount; i++)
|
||||
{
|
||||
foreach (var symbol in GenerateAsset())
|
||||
{
|
||||
yield return symbol;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random asset
|
||||
/// </summary>
|
||||
/// <param name="ticker">Optionally can provide a ticker that should be used</param>
|
||||
/// <returns>Random asset</returns>
|
||||
protected abstract IEnumerable<Symbol> GenerateAsset(string ticker = null);
|
||||
|
||||
/// <summary>
|
||||
/// Generates random symbol, used further down for asset
|
||||
/// </summary>
|
||||
/// <param name="securityType">security type</param>
|
||||
/// <param name="market">market</param>
|
||||
/// <param name="ticker">Optionally can provide a ticker to use</param>
|
||||
/// <returns>Random symbol</returns>
|
||||
public Symbol NextSymbol(SecurityType securityType, string market, string ticker = null)
|
||||
{
|
||||
if (securityType == SecurityType.Option || securityType == SecurityType.Future)
|
||||
{
|
||||
throw new ArgumentException("Please use OptionSymbolGenerator or FutureSymbolGenerator for SecurityType.Option and SecurityType.Future respectively.");
|
||||
}
|
||||
|
||||
if (ticker == null)
|
||||
{
|
||||
// we must return a Symbol matching an entry in the Symbol properties database
|
||||
// if there is a wildcard entry, we can generate a truly random Symbol
|
||||
// if there is no wildcard entry, the symbols we can generate are limited by the entries in the database
|
||||
if (SymbolPropertiesDatabase.ContainsKey(market, SecurityDatabaseKey.Wildcard, securityType))
|
||||
{
|
||||
// let's make symbols all have 3 chars as it's acceptable for all security types with wildcard entries
|
||||
ticker = NextUpperCaseString(3, 3);
|
||||
}
|
||||
else
|
||||
{
|
||||
ticker = NextTickerFromSymbolPropertiesDatabase(securityType, market);
|
||||
}
|
||||
}
|
||||
|
||||
// by chance we may generate a ticker that actually exists, and if map files exist that match this
|
||||
// ticker then we'll end up resolving the first trading date for use in the SID, otherwise, all
|
||||
// generated Symbol will have a date equal to SecurityIdentifier.DefaultDate
|
||||
var symbol = Symbol.Create(ticker, securityType, market);
|
||||
if (_symbols.Add(symbol))
|
||||
{
|
||||
return symbol;
|
||||
}
|
||||
|
||||
// lo' and behold, we created a duplicate --recurse to find a unique value
|
||||
// this is purposefully done as the last statement to enable the compiler to
|
||||
// unroll this method into a tail-recursion loop :)
|
||||
return NextSymbol(securityType, market);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Return a Ticker matching an entry in the Symbol properties database
|
||||
/// </summary>
|
||||
/// <param name="securityType">security type</param>
|
||||
/// <param name="market"></param>
|
||||
/// <returns>Random Ticker matching an entry in the Symbol properties database</returns>
|
||||
protected string NextTickerFromSymbolPropertiesDatabase(SecurityType securityType, string market)
|
||||
{
|
||||
// prevent returning a ticker matching any previously generated Symbol
|
||||
var existingTickers = _symbols
|
||||
.Where(sym => sym.ID.Market == market && sym.ID.SecurityType == securityType)
|
||||
.Select(sym => sym.Value);
|
||||
|
||||
// get the available tickers from the Symbol properties database and remove previously generated tickers
|
||||
var availableTickers = Enumerable.Except(SymbolPropertiesDatabase.GetSymbolPropertiesList(market, securityType)
|
||||
.Select(kvp => kvp.Key.Symbol), existingTickers)
|
||||
.ToList();
|
||||
|
||||
// there is a limited number of entries in the Symbol properties database so we may run out of tickers
|
||||
if (availableTickers.Count == 0)
|
||||
{
|
||||
throw new NoTickersAvailableException(securityType, market);
|
||||
}
|
||||
|
||||
return availableTickers[Random.NextInt(availableTickers.Count)];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates random expiration date on a friday within specified time range
|
||||
/// </summary>
|
||||
/// <param name="marketHours">market hours</param>
|
||||
/// <param name="minExpiry">minimum expiration date</param>
|
||||
/// <param name="maxExpiry">maximum expiration date</param>
|
||||
/// <returns>Random date on a friday within specified time range</returns>
|
||||
protected DateTime GetRandomExpiration(SecurityExchangeHours marketHours, DateTime minExpiry, DateTime maxExpiry)
|
||||
{
|
||||
// generate a random expiration date on a friday
|
||||
var expiry = Random.NextDate(minExpiry, maxExpiry, DayOfWeek.Friday);
|
||||
|
||||
// check to see if we're open on this date and if not, back track until we are
|
||||
// we're using the equity market hours as a proxy since we haven't generated the option Symbol yet
|
||||
while (!marketHours.IsDateOpen(expiry))
|
||||
{
|
||||
expiry = expiry.AddDays(-1);
|
||||
}
|
||||
|
||||
return expiry;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random <see cref="string"/> within the specified lengths.
|
||||
/// </summary>
|
||||
/// <param name="minLength">The minimum length, inclusive</param>
|
||||
/// <param name="maxLength">The maximum length, inclusive</param>
|
||||
/// <returns>A new upper case string within the specified lengths</returns>
|
||||
public string NextUpperCaseString(int minLength, int maxLength)
|
||||
{
|
||||
var str = string.Empty;
|
||||
var length = Random.NextInt(minLength, maxLength);
|
||||
for (int i = 0; i < length; i++)
|
||||
{
|
||||
// A=65 - inclusive lower bound
|
||||
// Z=90 - inclusive upper bound
|
||||
var c = (char)Random.NextInt(65, 91);
|
||||
str += c;
|
||||
}
|
||||
|
||||
return str;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the number of symbols with the specified parameters can be generated.
|
||||
/// Returns int.MaxValue if there is no limit for the given parameters.
|
||||
/// </summary>
|
||||
/// <returns>The number of available symbols for the given parameters, or int.MaxValue if no limit</returns>
|
||||
public abstract int GetAvailableSymbolCount();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Specifies how dense data should be generated
|
||||
/// </summary>
|
||||
public enum DataDensity
|
||||
{
|
||||
/// <summary>
|
||||
/// At least once per resolution step
|
||||
/// </summary>
|
||||
Dense,
|
||||
|
||||
/// <summary>
|
||||
/// At least once per 5 resolution steps
|
||||
/// </summary>
|
||||
Sparse,
|
||||
|
||||
/// <summary>
|
||||
/// At least once per 50 resolution steps
|
||||
/// </summary>
|
||||
VerySparse
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using QuantConnect.Securities;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates a new random <see cref="Symbol"/> object of the specified security type.
|
||||
/// All returned symbols have a matching entry in the Symbol properties database.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A valid implementation will keep track of generated Symbol objects to ensure duplicates
|
||||
/// are not generated.
|
||||
/// </remarks>
|
||||
public class DefaultSymbolGenerator : BaseSymbolGenerator
|
||||
{
|
||||
private readonly string _market;
|
||||
private readonly SecurityType _securityType;
|
||||
|
||||
/// <summary>
|
||||
/// Creates <see cref="DefaultSymbolGenerator"/> instance
|
||||
/// </summary>
|
||||
/// <param name="settings">random data generation run settings</param>
|
||||
/// <param name="random">produces random values for use in random data generation</param>
|
||||
public DefaultSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
|
||||
: base(settings, random)
|
||||
{
|
||||
_market = settings.Market;
|
||||
_securityType = settings.SecurityType;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a single-item list at a time using base random implementation
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
protected override IEnumerable<Symbol> GenerateAsset(string ticker = null)
|
||||
{
|
||||
yield return NextSymbol(Settings.SecurityType, Settings.Market, ticker);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the number of symbols with the specified parameters can be generated.
|
||||
/// Returns int.MaxValue if there is no limit for the given parameters.
|
||||
/// </summary>
|
||||
/// <returns>The number of available symbols for the given parameters, or int.MaxValue if no limit</returns>
|
||||
public override int GetAvailableSymbolCount()
|
||||
{
|
||||
// check the Symbol properties database to determine how many symbols we can generate
|
||||
// if there is a wildcard entry, we can generate as many symbols as we want
|
||||
// if there is no wildcard entry, we can only generate as many symbols as there are entries
|
||||
return SymbolPropertiesDatabase.ContainsKey(_market, SecurityDatabaseKey.Wildcard, _securityType)
|
||||
? int.MaxValue
|
||||
: SymbolPropertiesDatabase.GetSymbolPropertiesList(_market, _securityType).Count();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates random splits, random dividends, and map file
|
||||
/// </summary>
|
||||
public class DividendSplitMapGenerator
|
||||
{
|
||||
private const double _minimumFinalSplitFactorAllowed = 0.001;
|
||||
|
||||
/// <summary>
|
||||
/// The final factor to adjust all prices with in order to maintain price continuity.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Set default equal to 1 so that we can use it even in the event of no splits
|
||||
/// </remarks>
|
||||
public decimal FinalSplitFactor { get; set; } = 1m;
|
||||
|
||||
/// <summary>
|
||||
/// Stores <see cref="MapFileRow"/> instances
|
||||
/// </summary>
|
||||
public List<MapFileRow> MapRows { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Stores <see cref="CorporateFactorRow"/> instances
|
||||
/// </summary>
|
||||
public List<CorporateFactorRow> DividendsSplits { get; set; } = new List<CorporateFactorRow>();
|
||||
|
||||
/// <summary>
|
||||
/// Current Symbol value. Can be renamed
|
||||
/// </summary>
|
||||
public Symbol CurrentSymbol { get; private set; }
|
||||
|
||||
private readonly RandomValueGenerator _randomValueGenerator;
|
||||
private readonly Random _random;
|
||||
private readonly RandomDataGeneratorSettings _settings;
|
||||
private readonly DateTime _delistDate;
|
||||
private readonly bool _willBeDelisted;
|
||||
private readonly BaseSymbolGenerator _symbolGenerator;
|
||||
|
||||
public DividendSplitMapGenerator(
|
||||
Symbol symbol,
|
||||
RandomDataGeneratorSettings settings,
|
||||
RandomValueGenerator randomValueGenerator,
|
||||
BaseSymbolGenerator symbolGenerator,
|
||||
Random random,
|
||||
DateTime delistDate,
|
||||
bool willBeDelisted)
|
||||
{
|
||||
CurrentSymbol = symbol;
|
||||
_settings = settings;
|
||||
_randomValueGenerator = randomValueGenerator;
|
||||
_random = random;
|
||||
_delistDate = delistDate;
|
||||
_willBeDelisted = willBeDelisted;
|
||||
_symbolGenerator = symbolGenerator;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates the splits, dividends, and maps.
|
||||
/// Writes necessary output to public variables
|
||||
/// </summary>
|
||||
/// <param name="tickHistory"></param>
|
||||
public void GenerateSplitsDividends(IEnumerable<Tick> tickHistory)
|
||||
{
|
||||
var previousMonth = -1;
|
||||
var monthsTrading = 0;
|
||||
|
||||
var hasRename = _randomValueGenerator.NextBool(_settings.HasRenamePercentage);
|
||||
var hasSplits = _randomValueGenerator.NextBool(_settings.HasSplitsPercentage);
|
||||
var hasDividends = _randomValueGenerator.NextBool(_settings.HasDividendsPercentage);
|
||||
var dividendEveryQuarter = _randomValueGenerator.NextBool(_settings.DividendEveryQuarterPercentage);
|
||||
|
||||
var previousX = _random.NextDouble();
|
||||
|
||||
// Since the largest equity value we can obtain is 1 000 000, if we want this price divided by the FinalSplitFactor
|
||||
// to be upper bounded by 1 000 000 000 we need to make sure the FinalSplitFactor is lower bounded by 0.001. Therefore,
|
||||
// since in the worst of the cases FinalSplitFactor = (previousSplitFactor)^(2m), where m is the number of months
|
||||
// in the time span, we need to lower bound previousSplitFactor by (0.001)^(1/(2m))
|
||||
//
|
||||
// On the other hand, if the upper bound for the previousSplitFactor is 1, then the FinalSplitFactor will be, in the
|
||||
// worst of the cases as small as the minimum equity value we can obtain
|
||||
|
||||
var months = (int)Math.Round(_settings.End.Subtract(_settings.Start).Days / (365.25 / 12));
|
||||
months = months != 0 ? months : 1;
|
||||
var minPreviousSplitFactor = GetLowerBoundForPreviousSplitFactor(months);
|
||||
var maxPreviousSplitFactor = 1;
|
||||
var previousSplitFactor = hasSplits ? GetNextPreviousSplitFactor(_random, minPreviousSplitFactor, maxPreviousSplitFactor) : 1;
|
||||
var previousPriceFactor = hasDividends ? (decimal)Math.Tanh(previousX) : 1;
|
||||
|
||||
var splitDates = new List<DateTime>();
|
||||
var dividendDates = new List<DateTime>();
|
||||
|
||||
var firstTick = true;
|
||||
|
||||
// Iterate through all ticks and generate splits and dividend data
|
||||
if (_settings.SecurityType == SecurityType.Equity)
|
||||
{
|
||||
foreach (var tick in tickHistory)
|
||||
{
|
||||
// On the first trading day write relevant starting data to factor and map files
|
||||
if (firstTick)
|
||||
{
|
||||
DividendsSplits.Add(new CorporateFactorRow(tick.Time,
|
||||
previousPriceFactor,
|
||||
previousSplitFactor,
|
||||
tick.Value));
|
||||
|
||||
MapRows.Add(new MapFileRow(tick.Time, CurrentSymbol.Value));
|
||||
}
|
||||
|
||||
// Add the split to the DividendsSplits list if we have a pending
|
||||
// split. That way, we can use the correct referencePrice in the split event.
|
||||
if (splitDates.Count != 0)
|
||||
{
|
||||
var deleteDates = new List<DateTime>();
|
||||
|
||||
foreach (var splitDate in splitDates)
|
||||
{
|
||||
if (tick.Time > splitDate)
|
||||
{
|
||||
DividendsSplits.Add(new CorporateFactorRow(
|
||||
splitDate,
|
||||
previousPriceFactor,
|
||||
previousSplitFactor,
|
||||
tick.Value / FinalSplitFactor));
|
||||
|
||||
FinalSplitFactor *= previousSplitFactor;
|
||||
deleteDates.Add(splitDate);
|
||||
}
|
||||
}
|
||||
|
||||
// Deletes dates we've already looped over
|
||||
splitDates.RemoveAll(x => deleteDates.Contains(x));
|
||||
}
|
||||
|
||||
if (dividendDates.Count != 0)
|
||||
{
|
||||
var deleteDates = new List<DateTime>();
|
||||
|
||||
foreach (var dividendDate in dividendDates)
|
||||
{
|
||||
if (tick.Time > dividendDate)
|
||||
{
|
||||
DividendsSplits.Add(new CorporateFactorRow(
|
||||
dividendDate,
|
||||
previousPriceFactor,
|
||||
previousSplitFactor,
|
||||
tick.Value / FinalSplitFactor));
|
||||
|
||||
deleteDates.Add(dividendDate);
|
||||
}
|
||||
}
|
||||
|
||||
dividendDates.RemoveAll(x => deleteDates.Contains(x));
|
||||
}
|
||||
|
||||
if (tick.Time.Month != previousMonth)
|
||||
{
|
||||
// Every quarter, try to generate dividend events
|
||||
if (hasDividends && (tick.Time.Month - 1) % 3 == 0)
|
||||
{
|
||||
// Make it so there's a 10% chance that dividends occur if there is no dividend every quarter
|
||||
if (dividendEveryQuarter || _randomValueGenerator.NextBool(10.0))
|
||||
{
|
||||
do
|
||||
{
|
||||
previousX += _random.NextDouble() / 10;
|
||||
previousPriceFactor = (decimal)Math.Tanh(previousX);
|
||||
} while (previousPriceFactor >= 1.0m || previousPriceFactor <= 0m);
|
||||
|
||||
dividendDates.Add(_randomValueGenerator.NextDate(tick.Time, tick.Time.AddMonths(1), (DayOfWeek)_random.Next(1, 5)));
|
||||
}
|
||||
}
|
||||
// Have a 5% chance of a split every month
|
||||
if (hasSplits && _randomValueGenerator.NextBool(_settings.MonthSplitPercentage))
|
||||
{
|
||||
// Produce another split factor that is also bounded by the min and max split factors allowed
|
||||
if (_randomValueGenerator.NextBool(5.0)) // Add the possibility of a reverse split
|
||||
{
|
||||
// A reverse split is a split that is smaller than the current previousSplitFactor
|
||||
// Update previousSplitFactor with a smaller value that is still bounded below by minPreviousSplitFactor
|
||||
previousSplitFactor = GetNextPreviousSplitFactor(_random, minPreviousSplitFactor, previousSplitFactor);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Update previousSplitFactor with a higher value that is still bounded by maxPreviousSplitFactor
|
||||
// Usually, the split factor tends to grow across the time span(See /Data/Equity/usa/factor_files/aapl for instance)
|
||||
previousSplitFactor = GetNextPreviousSplitFactor(_random, previousSplitFactor, maxPreviousSplitFactor);
|
||||
}
|
||||
|
||||
splitDates.Add(_randomValueGenerator.NextDate(tick.Time, tick.Time.AddMonths(1), (DayOfWeek)_random.Next(1, 5)));
|
||||
}
|
||||
// 10% chance of being renamed every month
|
||||
if (hasRename && _randomValueGenerator.NextBool(10.0))
|
||||
{
|
||||
var randomDate = _randomValueGenerator.NextDate(tick.Time, tick.Time.AddMonths(1), (DayOfWeek)_random.Next(1, 5));
|
||||
MapRows.Add(new MapFileRow(randomDate, CurrentSymbol.Value));
|
||||
|
||||
CurrentSymbol = _symbolGenerator.NextSymbol(_settings.SecurityType, _settings.Market);
|
||||
}
|
||||
|
||||
previousMonth = tick.Time.Month;
|
||||
monthsTrading++;
|
||||
}
|
||||
|
||||
if (monthsTrading >= 6 && _willBeDelisted && tick.Time > _delistDate)
|
||||
{
|
||||
MapRows.Add(new MapFileRow(tick.Time, CurrentSymbol.Value));
|
||||
break;
|
||||
}
|
||||
|
||||
firstTick = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a lower bound that guarantees the FinalSplitFactor, in all the possible
|
||||
/// cases, will never be smaller than the _minimumFinalSplitFactorAllowed (0.001)
|
||||
/// </summary>
|
||||
/// <param name="months">The lower bound for the previous split factor is based on
|
||||
/// the number of months between the start and end date from ticksHistory <see cref="GenerateSplitsDividends(IEnumerable{Tick})"></param>
|
||||
/// <returns>A valid lower bound that guarantees the FinalSplitFactor is always higher
|
||||
/// than the _minimumFinalSplitFactorAllowed</returns>
|
||||
public static decimal GetLowerBoundForPreviousSplitFactor(int months)
|
||||
{
|
||||
return (decimal)(Math.Pow(_minimumFinalSplitFactorAllowed, 1 / (double)(2 * months)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a new valid previousSplitFactor that is still bounded by the given upper and lower
|
||||
/// bounds
|
||||
/// </summary>
|
||||
/// <param name="random">Random number generator</param>
|
||||
/// <param name="lowerBound">Minimum allowed value to obtain</param>
|
||||
/// <param name="upperBound">Maximum allowed value to obtain</param>
|
||||
/// <returns>A new valid previousSplitFactor that is still bounded by the given upper and lower
|
||||
/// bounds</returns>
|
||||
public static decimal GetNextPreviousSplitFactor(Random random, decimal lowerBound, decimal upperBound)
|
||||
{
|
||||
return ((decimal)random.NextDouble()) * (upperBound - lowerBound) + lowerBound;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using QuantConnect.Securities.Future;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates a new random future <see cref="Symbol"/>. The generates future contract Symbol will have an
|
||||
/// expiry between the specified time range.
|
||||
/// </summary>
|
||||
public class FutureSymbolGenerator : BaseSymbolGenerator
|
||||
{
|
||||
private readonly DateTime _minExpiry;
|
||||
private readonly DateTime _maxExpiry;
|
||||
private readonly string _market;
|
||||
|
||||
public FutureSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
|
||||
: base(settings, random)
|
||||
{
|
||||
_minExpiry = settings.Start;
|
||||
_maxExpiry = settings.End;
|
||||
_market = settings.Market;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a new random future <see cref="Symbol"/>. The generates future contract Symbol will have an
|
||||
/// expiry between the specified minExpiry and maxExpiry.
|
||||
/// </summary>
|
||||
/// <param name="ticker">Optionally can provide a ticker that should be used</param>
|
||||
/// <returns>A new future contract Symbol with the specified expiration parameters</returns>
|
||||
protected override IEnumerable<Symbol> GenerateAsset(string ticker = null)
|
||||
{
|
||||
if (ticker == null)
|
||||
{
|
||||
// get a valid ticker from the Symbol properties database
|
||||
ticker = NextTickerFromSymbolPropertiesDatabase(SecurityType.Future, _market);
|
||||
}
|
||||
|
||||
var marketHours = MarketHoursDatabase.GetExchangeHours(_market, ticker, SecurityType.Future);
|
||||
var expiry = GetRandomExpiration(marketHours, _minExpiry, _maxExpiry);
|
||||
|
||||
// Try to get the specific expiry function for this future, if available
|
||||
var symbol = Symbol.CreateFuture(ticker, _market, SecurityIdentifier.DefaultDate);
|
||||
if (!FuturesExpiryFunctions.FuturesExpiryDictionary.TryGetValue(symbol, out var expiryFunction))
|
||||
{
|
||||
// If no expiry function is found, return the future using the previously chosen expiry
|
||||
yield return Symbol.CreateFuture(ticker, _market, expiry);
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Get all valid expiries in range using the expiry function
|
||||
// HashSet ensures unique expiry dates since multiple reference dates may map to same expiry
|
||||
var validExpiries = new HashSet<DateTime>();
|
||||
|
||||
// Extends range by ±1 month to catch all potential expiries.
|
||||
// Some futures (like NG) calculate expiry based on next month's date
|
||||
// (e.g., "3 business days before 1st of next month"), so we need to look ahead.
|
||||
// This buffer ensures we don't miss expiries near range boundaries.
|
||||
for (var date = _minExpiry.AddMonths(-1); date <= _maxExpiry.AddMonths(1); date = date.AddDays(1))
|
||||
{
|
||||
// Calculate expiry date using the futures-specific function
|
||||
var newExpiry = expiryFunction(date);
|
||||
|
||||
// Only include expiries within our target range
|
||||
if (_minExpiry < newExpiry && newExpiry <= _maxExpiry)
|
||||
{
|
||||
// Add to set of valid expiries (automatically handles duplicates)
|
||||
validExpiries.Add(newExpiry);
|
||||
}
|
||||
}
|
||||
|
||||
if (validExpiries.Count == 0)
|
||||
{
|
||||
yield return Symbol.CreateFuture(ticker, _market, expiry);
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Randomly select one expiry from the valid set
|
||||
var skip = Random.NextInt(validExpiries.Count);
|
||||
expiry = validExpiries.Skip(skip).First();
|
||||
|
||||
// Return the future contract using the randomly selected valid expiry
|
||||
yield return Symbol.CreateFuture(ticker, _market, expiry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// There is no limit for the future symbols.
|
||||
/// </summary>
|
||||
/// <returns>Returns int.MaxValue</returns>
|
||||
public override int GetAvailableSymbolCount() => int.MaxValue;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Defines a type capable of producing random prices
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Any parameters referenced as a percentage value are always in 'percent space', meaning 1 is 1%.
|
||||
/// </remarks>
|
||||
public interface IPriceGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates an asset price
|
||||
/// </summary>
|
||||
/// <param name="maximumPercentDeviation">The maximum percent deviation. This value is in percent space,
|
||||
/// so a value of 1m is equal to 1%.</param>
|
||||
/// <param name="referenceDate">date used in price calculation</param>
|
||||
/// <returns>Returns a new decimal as price</returns>
|
||||
public decimal NextValue(decimal maximumPercentDeviation, DateTime referenceDate);
|
||||
|
||||
/// <summary>
|
||||
/// Indicates Price generator warmed up and ready to generate new values
|
||||
/// </summary>
|
||||
public bool WarmedUp { get; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Securities;
|
||||
using System;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Defines a type capable of producing random values for use in random data generation
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Any parameters referenced as a percentage value are always in 'percent space', meaning 1 is 1%.
|
||||
/// </remarks>
|
||||
public interface IRandomValueGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Randomly return a <see cref="bool"/> value with the specified odds of being true
|
||||
/// </summary>
|
||||
/// <param name="percentOddsForTrue">The percent odds of being true in percent space, so 10 => 10%</param>
|
||||
/// <returns>True or false</returns>
|
||||
bool NextBool(double percentOddsForTrue);
|
||||
|
||||
/// <summary>
|
||||
/// Returns a random floating-point number that is greater than or equal to 0.0, and less than 1.0
|
||||
/// </summary>
|
||||
/// <returns>A double-precision floating point number that is greater than or equal to 0.0, and less than 1.0.</returns>
|
||||
double NextDouble();
|
||||
|
||||
/// <summary>
|
||||
/// Returns a random integer that is within a specified range.
|
||||
/// </summary>
|
||||
/// <param name="minValue">the inclusive lower bound of the random number returned</param>
|
||||
/// <param name="maxValue">the exclusive upper bound of the random number returned</param>
|
||||
/// <returns>A 32-bit signed integer greater than or equal to minValue and less than maxValue.</returns>
|
||||
int NextInt(int minValue, int maxValue);
|
||||
|
||||
/// <summary>
|
||||
/// Returns a non-negative random integer that is less than the specified maximum.
|
||||
/// </summary>
|
||||
/// <param name="maxValue">the exclusive upper bound of the random number to be generated.</param>
|
||||
/// <returns>A 32-bit signed integer that is greater than or equal to 0, and less than maxValue.</returns>
|
||||
int NextInt(int maxValue);
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random <see cref="DateTime"/> between the specified <paramref name="minDateTime"/> and
|
||||
/// <paramref name="maxDateTime"/>. <paramref name="dayOfWeek"/> is optionally specified to force the
|
||||
/// result to a particular day of the week
|
||||
/// </summary>
|
||||
/// <param name="minDateTime">The minimum date time, inclusive</param>
|
||||
/// <param name="maxDateTime">The maximum date time, inclusive</param>
|
||||
/// <param name="dayOfWeek">Optional. The day of week to force</param>
|
||||
/// <returns>A new <see cref="DateTime"/> within the specified range and optionally of the specified day of week</returns>
|
||||
DateTime NextDate(DateTime minDateTime, DateTime maxDateTime, DayOfWeek? dayOfWeek);
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random <see cref="decimal"/> suitable as a price. This should observe minimum price
|
||||
/// variations if available in <see cref="SymbolPropertiesDatabase"/>, and if not, truncating to 2
|
||||
/// decimal places.
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">Throw when the <paramref name="referencePrice"/> or <paramref name="maximumPercentDeviation"/>
|
||||
/// is less than or equal to zero.</exception>
|
||||
/// <param name="securityType">The security type the price is being generated for</param>
|
||||
/// <param name="market">The market of the security the price is being generated for</param>
|
||||
/// <param name="referencePrice">The reference price used as the mean of random price generation</param>
|
||||
/// <param name="maximumPercentDeviation">The maximum percent deviation. This value is in percent space,
|
||||
/// so a value of 1m is equal to 1%.</param>
|
||||
/// <returns>A new decimal suitable for usage as price within the specified deviation from the reference price</returns>
|
||||
decimal NextPrice(SecurityType securityType, string market, decimal referencePrice, decimal maximumPercentDeviation);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using QuantConnect.Data.Market;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Describes main methods for <see cref="TickGenerator"/>
|
||||
/// </summary>
|
||||
public interface ITickGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates and enumerates data points for current symbol
|
||||
/// </summary>
|
||||
IEnumerable<Tick> GenerateTicks();
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random <see cref="Tick"/> that is at most the specified <paramref name="maximumPercentDeviation"/> away from the
|
||||
/// previous price and is of the requested <paramref name="tickType"/>
|
||||
/// </summary>
|
||||
/// <param name="dateTime">The time of the generated tick</param>
|
||||
/// <param name="tickType">The type of <see cref="Tick"/> to be generated</param>
|
||||
/// <param name="maximumPercentDeviation">The maximum percentage to deviate from the
|
||||
/// previous price, for example, 1 would indicate a maximum of 1% deviation from the
|
||||
/// previous price. For a previous price of 100, this would yield a price between 99 and 101 inclusive</param>
|
||||
/// <returns>A random <see cref="Tick"/> value that is within the specified <paramref name="maximumPercentDeviation"/>
|
||||
/// from the previous price</returns>
|
||||
Tick NextTick(
|
||||
DateTime dateTime,
|
||||
TickType tickType,
|
||||
decimal maximumPercentDeviation
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random <see cref="DateTime"/> suitable for use as a tick's emit time.
|
||||
/// If the density provided is <see cref="DataDensity.Dense"/>, then at least one tick will be generated per <paramref name="resolution"/> step.
|
||||
/// If the density provided is <see cref="DataDensity.Sparse"/>, then at least one tick will be generated every 5 <paramref name="resolution"/> steps.
|
||||
/// if the density provided is <see cref="DataDensity.VerySparse"/>, then at least one tick will be generated every 50 <paramref name="resolution"/> steps.
|
||||
/// Times returned are guaranteed to be within market hours for the specified Symbol
|
||||
/// </summary>
|
||||
/// <param name="previous">The previous tick time</param>
|
||||
/// <param name="resolution">The requested resolution of data</param>
|
||||
/// <param name="density">The requested data density</param>
|
||||
/// <returns>A new <see cref="DateTime"/> that is after <paramref name="previous"/> according to the specified <paramref name="resolution"/>
|
||||
/// and <paramref name="density"/> specified</returns>
|
||||
DateTime NextTickTime(DateTime previous, Resolution resolution, DataDensity density);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Exception thrown when there are no tickers left to generate for a certain combination of security type and market.
|
||||
/// </summary>
|
||||
public class NoTickersAvailableException : RandomValueGeneratorException
|
||||
{
|
||||
public NoTickersAvailableException(SecurityType securityType, string market)
|
||||
: base($"Failed to generate {securityType} symbol for {market}, there are no tickers left")
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Securities;
|
||||
using System;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Securities.Option;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Pricing model used to determine the fair price or theoretical value for a call or a put option price
|
||||
/// by default using the Black-Scholes-Merton model
|
||||
/// </summary>
|
||||
public class OptionPriceModelPriceGenerator : IPriceGenerator
|
||||
{
|
||||
private readonly Option _option;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="RandomPriceGenerator"/> is always ready to generate new price values as it does not depend on volatility model
|
||||
/// </summary>
|
||||
public bool WarmedUp => _option.PriceModel is QLOptionPriceModel optionPriceModel && optionPriceModel.VolatilityEstimatorWarmedUp || _option.PriceModel is not QLOptionPriceModel;
|
||||
|
||||
/// <summary>
|
||||
/// Creates instance of <see cref="OptionPriceModelPriceGenerator"/>
|
||||
/// </summary>
|
||||
///<param name="security"><see cref="Security"/> object for which to generate price data</param>
|
||||
public OptionPriceModelPriceGenerator(Security security)
|
||||
{
|
||||
if (security == null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(security), "security cannot be null");
|
||||
}
|
||||
|
||||
if (!security.Symbol.SecurityType.IsOption())
|
||||
{
|
||||
throw new ArgumentException($"{nameof(OptionPriceModelPriceGenerator)} model cannot be applied to non-option security.");
|
||||
}
|
||||
|
||||
_option = security as Option;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// For Black-Scholes-Merton model price calculation relies <see cref="IOptionPriceModel"/> of the security
|
||||
/// </summary>
|
||||
/// <param name="maximumPercentDeviation">The maximum percent deviation. This value is in percent space,
|
||||
/// so a value of 1m is equal to 1%.</param>
|
||||
/// <param name="referenceDate">current reference date</param>
|
||||
/// <returns>A new decimal suitable for usage as new security price</returns>
|
||||
public decimal NextValue(decimal maximumPercentDeviation, DateTime referenceDate)
|
||||
{
|
||||
var underlying = _option.Underlying;
|
||||
var price = underlying.Price;
|
||||
|
||||
var tick = new Tick(referenceDate, underlying.Symbol, price, price);
|
||||
var contract = OptionContract.Create(referenceDate, _option, tick);
|
||||
|
||||
var parameters = new OptionPriceModelParameters(_option, null, contract);
|
||||
return _option.PriceModel.Evaluate(parameters).TheoreticalPrice;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates a new random option <see cref="Symbol"/>.
|
||||
/// </summary>
|
||||
public class OptionSymbolGenerator : BaseSymbolGenerator
|
||||
{
|
||||
private readonly DateTime _minExpiry;
|
||||
private readonly DateTime _maxExpiry;
|
||||
private readonly string _market;
|
||||
private readonly int _symbolChainSize;
|
||||
private readonly decimal _underlyingPrice;
|
||||
private readonly decimal _maximumStrikePriceDeviation;
|
||||
private readonly SecurityType _underlyingSecurityType = SecurityType.Equity;
|
||||
|
||||
public OptionSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random, decimal underlyingPrice, decimal maximumStrikePriceDeviation)
|
||||
: base(settings, random)
|
||||
{
|
||||
// We add seven days more because TickGenerator for options needs first three underlying data points to warm up
|
||||
// the price generator, so if the expiry date is before settings.Start plus three days no quote or trade data is
|
||||
// generated for this option
|
||||
_minExpiry = (settings.Start).AddDays(7);
|
||||
_maxExpiry = (settings.End).AddDays(7);
|
||||
_market = settings.Market;
|
||||
_underlyingPrice = underlyingPrice;
|
||||
_symbolChainSize = settings.ChainSymbolCount;
|
||||
_maximumStrikePriceDeviation = maximumStrikePriceDeviation;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a new random option <see cref="Symbol"/>. The generated option contract Symbol will have an
|
||||
/// expiry between the specified min and max expiration. The strike
|
||||
/// price will be within the specified maximum strike price deviation of the underlying symbol price
|
||||
/// and should be rounded to reasonable value for the given price. For example, a price of 100 dollars would round
|
||||
/// to 5 dollar increments and a price of 5 dollars would round to 50 cent increments
|
||||
/// </summary>
|
||||
/// <param name="ticker">Optionally can provide a ticker that should be used</param>
|
||||
/// <remarks>
|
||||
/// Standard contracts expiry on the third Friday.
|
||||
/// Weekly contracts expiry every week on Friday
|
||||
/// </remarks>
|
||||
/// <returns>A new option contract Symbol within the specified expiration and strike price parameters along with its underlying symbol</returns>
|
||||
protected override IEnumerable<Symbol> GenerateAsset(string ticker = null)
|
||||
{
|
||||
// first generate the underlying
|
||||
var underlying = NextSymbol(_underlyingSecurityType, _market, ticker);
|
||||
yield return underlying;
|
||||
|
||||
var marketHours = MarketHoursDatabase.GetExchangeHours(_market, underlying, _underlyingSecurityType);
|
||||
var expiry = GetRandomExpiration(marketHours, _minExpiry, _maxExpiry);
|
||||
|
||||
var strikes = new HashSet<decimal>();
|
||||
for (var i = 0; i < _symbolChainSize; i++)
|
||||
{
|
||||
decimal strike;
|
||||
do
|
||||
{
|
||||
// generate a random strike while respecting the maximum deviation from the underlying's price
|
||||
// since these are underlying prices, use Equity as the security type
|
||||
strike = Random.NextPrice(_underlyingSecurityType, _market, _underlyingPrice,
|
||||
_maximumStrikePriceDeviation);
|
||||
|
||||
// round the strike price to something reasonable
|
||||
var order = 1 + Math.Log10((double)strike);
|
||||
strike = strike.RoundToSignificantDigits((int)order);
|
||||
}
|
||||
// don't allow duplicate strikes
|
||||
while (!strikes.Add(strike));
|
||||
|
||||
foreach (var optionRight in new [] { OptionRight.Put, OptionRight.Call })
|
||||
{
|
||||
// when providing a null option w/ an expiry, it will automatically create the OSI ticker string for the Value
|
||||
yield return Symbol.CreateOption(underlying, _market, underlying.SecurityType.DefaultOptionStyle(), optionRight, strike, expiry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the number of symbols with the specified parameters can be generated.
|
||||
/// There is no limit for the options.
|
||||
/// </summary>
|
||||
/// <returns>returns int.MaxValue</returns>
|
||||
public override int GetAvailableSymbolCount() => int.MaxValue;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Securities;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
|
||||
using QuantConnect.Logging;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates random data according to the specified parameters
|
||||
/// </summary>
|
||||
public class RandomDataGenerator
|
||||
{
|
||||
private RandomDataGeneratorSettings _settings;
|
||||
private SecurityManager _securityManager;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes <see cref="RandomDataGenerator"/> instance fields
|
||||
/// </summary>
|
||||
/// <param name="settings">random data generation settings</param>
|
||||
/// <param name="securityManager">security management</param>
|
||||
public void Init(RandomDataGeneratorSettings settings, SecurityManager securityManager)
|
||||
{
|
||||
_settings = settings;
|
||||
_securityManager = securityManager;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts data generation
|
||||
/// </summary>
|
||||
public void Run()
|
||||
{
|
||||
var tickTypesPerSecurityType = SubscriptionManager.DefaultDataTypes();
|
||||
// can specify a seed value in this ctor if determinism is desired
|
||||
var random = new Random();
|
||||
var randomValueGenerator = new RandomValueGenerator();
|
||||
if (_settings.RandomSeedSet)
|
||||
{
|
||||
random = new Random(_settings.RandomSeed);
|
||||
randomValueGenerator = new RandomValueGenerator(_settings.RandomSeed);
|
||||
}
|
||||
|
||||
var symbolGenerator = BaseSymbolGenerator.Create(_settings, randomValueGenerator);
|
||||
|
||||
var maxSymbolCount = symbolGenerator.GetAvailableSymbolCount();
|
||||
if (_settings.SymbolCount > maxSymbolCount)
|
||||
{
|
||||
Log.Error($"RandomDataGenerator.Run(): Limiting Symbol count to {maxSymbolCount}, we don't have more {_settings.SecurityType} tickers for {_settings.Market}");
|
||||
_settings.SymbolCount = maxSymbolCount;
|
||||
}
|
||||
|
||||
Log.Trace($"RandomDataGenerator.Run(): Begin data generation of {_settings.SymbolCount} randomly generated {_settings.SecurityType} assets...");
|
||||
|
||||
// iterate over our randomly generated symbols
|
||||
var count = 0;
|
||||
var progress = 0d;
|
||||
var previousMonth = -1;
|
||||
|
||||
foreach (var (symbolRef, currentSymbolGroup) in symbolGenerator.GenerateRandomSymbols()
|
||||
.GroupBy(s => s.HasUnderlying ? s.Underlying : s)
|
||||
.Select(g => (g.Key, g.OrderBy(s => s.HasUnderlying).ToList())))
|
||||
{
|
||||
Log.Trace($"RandomDataGenerator.Run(): Symbol[{++count}]: {symbolRef} Progress: {progress:0.0}% - Generating data...");
|
||||
|
||||
var tickGenerators = new List<IEnumerator<Tick>>();
|
||||
var tickHistories = new Dictionary<Symbol, List<Tick>>();
|
||||
Security underlyingSecurity = null;
|
||||
foreach (var currentSymbol in currentSymbolGroup)
|
||||
{
|
||||
if (!_securityManager.TryGetValue(currentSymbol, out var security))
|
||||
{
|
||||
security = _securityManager.CreateSecurity(
|
||||
currentSymbol,
|
||||
new List<SubscriptionDataConfig>(),
|
||||
underlying: underlyingSecurity);
|
||||
_securityManager.Add(security);
|
||||
}
|
||||
|
||||
underlyingSecurity ??= security;
|
||||
|
||||
tickGenerators.Add(
|
||||
new TickGenerator(_settings, tickTypesPerSecurityType[currentSymbol.SecurityType].ToArray(), security, randomValueGenerator)
|
||||
.GenerateTicks()
|
||||
.GetEnumerator());
|
||||
|
||||
tickHistories.Add(
|
||||
currentSymbol,
|
||||
new List<Tick>());
|
||||
}
|
||||
|
||||
using var sync = new SynchronizingBaseDataEnumerator(tickGenerators);
|
||||
|
||||
var lastLoggedProgress = 0.0;
|
||||
Log.Trace("[0%] Initializing tick data generation");
|
||||
while (sync.MoveNext())
|
||||
{
|
||||
var dataPoint = sync.Current;
|
||||
if (!_securityManager.TryGetValue(dataPoint.Symbol, out var security))
|
||||
{
|
||||
Log.Error($"RandomDataGenerator.Run(): Could not find security for symbol {sync.Current.Symbol}");
|
||||
continue;
|
||||
}
|
||||
|
||||
tickHistories[security.Symbol].Add(dataPoint as Tick);
|
||||
security.Update(new List<BaseData> { dataPoint }, dataPoint.GetType(), false);
|
||||
|
||||
// Calculate and log progress percentage when it increases by more than 3%
|
||||
var currentProgress = RandomDataGeneratorHelper.GetProgressAsPercentage(_settings.Start, _settings.End, dataPoint.EndTime);
|
||||
if (currentProgress - lastLoggedProgress >= 3.0)
|
||||
{
|
||||
Log.Trace($"[{currentProgress:0.00}%] Generating tick data");
|
||||
lastLoggedProgress = currentProgress;
|
||||
}
|
||||
}
|
||||
Log.Trace("[100%] Tick data generation completed successfully.");
|
||||
foreach (var (currentSymbol, tickHistory) in tickHistories)
|
||||
{
|
||||
var symbol = currentSymbol;
|
||||
|
||||
// This is done so that we can update the Symbol in the case of a rename event
|
||||
var delistDate = GetDelistingDate(_settings.Start, _settings.End, randomValueGenerator);
|
||||
var willBeDelisted = randomValueGenerator.NextBool(1.0);
|
||||
|
||||
// Companies rarely IPO then disappear within 6 months
|
||||
if (willBeDelisted && tickHistory.Select(tick => tick.Time.Month).Distinct().Count() <= 6)
|
||||
{
|
||||
willBeDelisted = false;
|
||||
}
|
||||
|
||||
var dividendsSplitsMaps = new DividendSplitMapGenerator(
|
||||
symbol,
|
||||
_settings,
|
||||
randomValueGenerator,
|
||||
symbolGenerator,
|
||||
random,
|
||||
delistDate,
|
||||
willBeDelisted);
|
||||
|
||||
// Keep track of renamed symbols and the time they were renamed.
|
||||
var renamedSymbols = new Dictionary<Symbol, DateTime>();
|
||||
|
||||
if (_settings.SecurityType == SecurityType.Equity)
|
||||
{
|
||||
dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
|
||||
|
||||
if (!willBeDelisted)
|
||||
{
|
||||
dividendsSplitsMaps.DividendsSplits.Add(new CorporateFactorRow(new DateTime(2050, 12, 31), 1m, 1m));
|
||||
|
||||
if (dividendsSplitsMaps.MapRows.Count > 1)
|
||||
{
|
||||
// Remove the last element if we're going to have a 20501231 entry
|
||||
dividendsSplitsMaps.MapRows.RemoveAt(dividendsSplitsMaps.MapRows.Count - 1);
|
||||
}
|
||||
dividendsSplitsMaps.MapRows.Add(new MapFileRow(new DateTime(2050, 12, 31), dividendsSplitsMaps.CurrentSymbol.Value));
|
||||
}
|
||||
|
||||
// If the Symbol value has changed, update the current Symbol
|
||||
if (symbol != dividendsSplitsMaps.CurrentSymbol)
|
||||
{
|
||||
// Add all Symbol rename events to dictionary
|
||||
// We skip the first row as it contains the listing event instead of a rename event
|
||||
foreach (var renameEvent in dividendsSplitsMaps.MapRows.Skip(1))
|
||||
{
|
||||
// Symbol.UpdateMappedSymbol does not update the underlying security ID Symbol, which
|
||||
// is used to create the hash code. Create a new equity Symbol from scratch instead.
|
||||
symbol = Symbol.Create(renameEvent.MappedSymbol, SecurityType.Equity, _settings.Market);
|
||||
renamedSymbols.Add(symbol, renameEvent.Date);
|
||||
|
||||
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} will be renamed on {renameEvent.Date}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// This ensures that ticks will be written for the current Symbol up until 9999-12-31
|
||||
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
|
||||
}
|
||||
|
||||
symbol = dividendsSplitsMaps.CurrentSymbol;
|
||||
|
||||
// Write Splits and Dividend events to directory factor_files
|
||||
var factorFile = new CorporateFactorProvider(symbol.Value, dividendsSplitsMaps.DividendsSplits, _settings.Start);
|
||||
var mapFile = new MapFile(symbol.Value, dividendsSplitsMaps.MapRows);
|
||||
|
||||
factorFile.WriteToFile(symbol);
|
||||
mapFile.WriteToCsv(_settings.Market, symbol.SecurityType);
|
||||
|
||||
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} Dividends, splits, and map files have been written to disk.");
|
||||
}
|
||||
else
|
||||
{
|
||||
// This ensures that ticks will be written for the current Symbol up until 9999-12-31
|
||||
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
|
||||
}
|
||||
|
||||
// define aggregators via settings
|
||||
var aggregators = CreateAggregators(_settings, tickTypesPerSecurityType[currentSymbol.SecurityType].ToArray()).ToList();
|
||||
Symbol previousSymbol = null;
|
||||
var currentCount = 0;
|
||||
var monthsTrading = 0;
|
||||
|
||||
foreach (var renamed in renamedSymbols)
|
||||
{
|
||||
var previousRenameDate = previousSymbol == null ? new DateTime(1, 1, 1) : renamedSymbols[previousSymbol];
|
||||
var previousRenameDateDay = new DateTime(previousRenameDate.Year, previousRenameDate.Month, previousRenameDate.Day);
|
||||
var renameDate = renamed.Value;
|
||||
var renameDateDay = new DateTime(renameDate.Year, renameDate.Month, renameDate.Day);
|
||||
|
||||
foreach (var tick in tickHistory.Where(tick => tick.Time >= previousRenameDate && previousRenameDateDay != TickDay(tick)))
|
||||
{
|
||||
// Prevents the aggregator from being updated with ticks after the rename event
|
||||
if (TickDay(tick) > renameDateDay)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (tick.Time.Month != previousMonth)
|
||||
{
|
||||
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: Month: {tick.Time:MMMM}");
|
||||
previousMonth = tick.Time.Month;
|
||||
monthsTrading++;
|
||||
}
|
||||
|
||||
foreach (var item in aggregators)
|
||||
{
|
||||
tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
|
||||
item.Consolidator.Update(tick);
|
||||
}
|
||||
|
||||
if (monthsTrading >= 6 && willBeDelisted && tick.Time > delistDate)
|
||||
{
|
||||
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {renamed.Key} delisted at {tick.Time:MMMM yyyy}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// count each stage as a point, so total points is 2*Symbol-count
|
||||
// and the current progress is twice the current, but less one because we haven't finished writing data yet
|
||||
progress = 100 * (2 * count - 1) / (2.0 * _settings.SymbolCount);
|
||||
|
||||
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {renamed.Key} Progress: {progress:0.0}% - Saving data in LEAN format");
|
||||
|
||||
// persist consolidated data to disk
|
||||
foreach (var item in aggregators)
|
||||
{
|
||||
var writer = new LeanDataWriter(item.Resolution, renamed.Key, Globals.DataFolder, item.TickType);
|
||||
|
||||
// send the flushed data into the writer. pulling the flushed list is very important,
|
||||
// lest we likely wouldn't get the last piece of data stuck in the consolidator
|
||||
// Filter out the data we're going to write here because filtering them in the consolidator update phase
|
||||
// makes it write all dates for some unknown reason
|
||||
writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != DataDay(data)));
|
||||
}
|
||||
|
||||
// update progress
|
||||
progress = 100 * (2 * count) / (2.0 * _settings.SymbolCount);
|
||||
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
|
||||
|
||||
previousSymbol = renamed.Key;
|
||||
currentCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Log.Trace("RandomDataGenerator.Run(): Random data generation has completed.");
|
||||
|
||||
DateTime TickDay(Tick tick) => new(tick.Time.Year, tick.Time.Month, tick.Time.Day);
|
||||
DateTime DataDay(BaseData data) => new(data.Time.Year, data.Time.Month, data.Time.Day);
|
||||
}
|
||||
|
||||
public static DateTime GetDateMidpoint(DateTime start, DateTime end)
|
||||
{
|
||||
TimeSpan span = end.Subtract(start);
|
||||
int span_time = (int)span.TotalMinutes;
|
||||
double diff_span = -(span_time / 2.0);
|
||||
DateTime start_time = end.AddMinutes(Math.Round(diff_span, 2, MidpointRounding.ToEven));
|
||||
|
||||
//Returns a DateTime object that is halfway between start and end
|
||||
return start_time;
|
||||
}
|
||||
|
||||
public static DateTime GetDelistingDate(DateTime start, DateTime end, RandomValueGenerator randomValueGenerator)
|
||||
{
|
||||
var mid_point = GetDateMidpoint(start, end);
|
||||
var delist_Date = randomValueGenerator.NextDate(mid_point, end, null);
|
||||
|
||||
//Returns a DateTime object that is a random value between the mid_point and end
|
||||
return delist_Date;
|
||||
}
|
||||
|
||||
public static IEnumerable<TickAggregator> CreateAggregators(RandomDataGeneratorSettings settings, TickType[] tickTypes)
|
||||
{
|
||||
// create default aggregators for tick type/resolution
|
||||
foreach (var tickAggregator in TickAggregator.ForTickTypes(settings.SecurityType, settings.Resolution, tickTypes))
|
||||
{
|
||||
yield return tickAggregator;
|
||||
}
|
||||
|
||||
|
||||
// ensure we have a daily consolidator when coarse is enabled
|
||||
if (settings.IncludeCoarse && settings.Resolution != Resolution.Daily)
|
||||
{
|
||||
// prefer trades for coarse - in practice equity only does trades, but leaving this as configurable
|
||||
if (tickTypes.Contains(TickType.Trade))
|
||||
{
|
||||
yield return TickAggregator.ForTickTypes(settings.SecurityType, Resolution.Daily, TickType.Trade).Single();
|
||||
}
|
||||
else
|
||||
{
|
||||
yield return TickAggregator.ForTickTypes(settings.SecurityType, Resolution.Daily, TickType.Quote).Single();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides helper methods for the Random Data Generator
|
||||
/// </summary>
|
||||
public static class RandomDataGeneratorHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Calculates the progress percentage of the current time between a start and end time.
|
||||
/// </summary>
|
||||
/// <param name="start">The start time of the process.</param>
|
||||
/// <param name="end">The end time of the process.</param>
|
||||
/// <param name="currentTime">The current time to evaluate progress.</param>
|
||||
/// <returns>The progress as a percentage, rounded to two decimal places.</returns>
|
||||
public static double GetProgressAsPercentage(DateTime start, DateTime end, DateTime currentTime)
|
||||
{
|
||||
var totalDuration = end - start;
|
||||
return Math.Round((currentTime - start).TotalMilliseconds * 1.0 / totalDuration.TotalMilliseconds * 100, 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Configuration;
|
||||
using QuantConnect.Data.Auxiliary;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Securities.Option;
|
||||
using QuantConnect.ToolBox.CoarseUniverseGenerator;
|
||||
using QuantConnect.Util;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Data;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates and starts <see cref="RandomDataGenerator"/> instance
|
||||
/// </summary>
|
||||
public static class RandomDataGeneratorProgram
|
||||
{
|
||||
private static readonly IRiskFreeInterestRateModel _interestRateProvider = new InterestRateProvider();
|
||||
|
||||
public static void RandomDataGenerator(
|
||||
string startDateString,
|
||||
string endDateString,
|
||||
string symbolCountString,
|
||||
string market,
|
||||
string securityTypeString,
|
||||
string resolutionString,
|
||||
string dataDensityString,
|
||||
string includeCoarseString,
|
||||
string quoteTradeRatioString,
|
||||
string randomSeed,
|
||||
string hasIpoPercentageString,
|
||||
string hasRenamePercentageString,
|
||||
string hasSplitsPercentageString,
|
||||
string hasDividendsPercentageString,
|
||||
string dividendEveryQuarterPercentageString,
|
||||
string optionPriceEngineName,
|
||||
string volatilityModelResolutionString,
|
||||
string chainSymbolCountString,
|
||||
List<string> tickers
|
||||
)
|
||||
{
|
||||
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
|
||||
startDateString,
|
||||
endDateString,
|
||||
symbolCountString,
|
||||
market,
|
||||
securityTypeString,
|
||||
resolutionString,
|
||||
dataDensityString,
|
||||
includeCoarseString,
|
||||
quoteTradeRatioString,
|
||||
randomSeed,
|
||||
hasIpoPercentageString,
|
||||
hasRenamePercentageString,
|
||||
hasSplitsPercentageString,
|
||||
hasDividendsPercentageString,
|
||||
dividendEveryQuarterPercentageString,
|
||||
optionPriceEngineName,
|
||||
volatilityModelResolutionString,
|
||||
chainSymbolCountString,
|
||||
tickers
|
||||
);
|
||||
|
||||
if (settings.Start.Year < 1998)
|
||||
{
|
||||
Log.Error($"RandomDataGeneratorProgram(): Required parameter --start must be at least 19980101");
|
||||
Environment.Exit(1);
|
||||
}
|
||||
|
||||
var securityManager = new SecurityManager(new TimeKeeper(settings.Start, new[] { TimeZones.Utc }));
|
||||
var securityService = new SecurityService(
|
||||
new CashBook(),
|
||||
MarketHoursDatabase.FromDataFolder(),
|
||||
SymbolPropertiesDatabase.FromDataFolder(),
|
||||
new SecurityInitializerProvider(new FuncSecurityInitializer(security =>
|
||||
{
|
||||
// init price
|
||||
security.SetMarketPrice(new Tick(settings.Start, security.Symbol, 100, 100));
|
||||
security.SetMarketPrice(new OpenInterest(settings.Start, security.Symbol, 10000));
|
||||
|
||||
// from settings
|
||||
security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(settings.VolatilityModelResolution);
|
||||
|
||||
// from settings
|
||||
if (security is Option option)
|
||||
{
|
||||
option.PriceModel = OptionPriceModels.QuantLib.Create(settings.OptionPriceEngineName,
|
||||
_interestRateProvider.GetRiskFreeRate(settings.Start, settings.End));
|
||||
}
|
||||
})),
|
||||
RegisteredSecurityDataTypesProvider.Null,
|
||||
new SecurityCacheProvider(
|
||||
new SecurityPortfolioManager(securityManager, new SecurityTransactionManager(null, securityManager), new AlgorithmSettings())),
|
||||
new MapFilePrimaryExchangeProvider(Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider")))
|
||||
);
|
||||
securityManager.SetSecurityService(securityService);
|
||||
|
||||
var generator = new RandomDataGenerator();
|
||||
generator.Init(settings, securityManager);
|
||||
generator.Run();
|
||||
|
||||
if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
|
||||
{
|
||||
Log.Trace("RandomDataGeneratorProgram(): Launching coarse data generator...");
|
||||
|
||||
CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
|
||||
}
|
||||
|
||||
if (!Console.IsInputRedirected)
|
||||
{
|
||||
Log.Trace("RandomDataGeneratorProgram(): Press any key to exit...");
|
||||
Console.ReadKey();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Brokerages;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
public class RandomDataGeneratorSettings
|
||||
{
|
||||
private static int MarketCode = 100;
|
||||
private static readonly string[] DateFormats = { DateFormat.EightCharacter, DateFormat.YearMonth, "yyyy-MM-dd" };
|
||||
|
||||
public DateTime Start { get; init; }
|
||||
public DateTime End { get; init; }
|
||||
public SecurityType SecurityType { get; init; } = SecurityType.Equity;
|
||||
public DataDensity DataDensity { get; init; } = DataDensity.Dense;
|
||||
public Resolution Resolution { get; init; } = Resolution.Minute;
|
||||
public string Market { get; init; }
|
||||
public bool IncludeCoarse { get; init; } = true;
|
||||
public int SymbolCount { get; set; }
|
||||
public double QuoteTradeRatio { get; init; } = 1;
|
||||
public int RandomSeed { get; init; }
|
||||
public bool RandomSeedSet { get; init; }
|
||||
public double HasIpoPercentage { get; init; }
|
||||
public double HasRenamePercentage { get; init; }
|
||||
public double HasSplitsPercentage { get; init; }
|
||||
public double MonthSplitPercentage { get; init; }
|
||||
public double HasDividendsPercentage { get; init; }
|
||||
public double DividendEveryQuarterPercentage { get; init; }
|
||||
public string OptionPriceEngineName { get; init; }
|
||||
public int ChainSymbolCount { get; init; } = 1;
|
||||
public Resolution VolatilityModelResolution { get; init; } = Resolution.Daily;
|
||||
public List<string> Tickers { get; init; }
|
||||
public static RandomDataGeneratorSettings FromCommandLineArguments(
|
||||
string startDateString,
|
||||
string endDateString,
|
||||
string symbolCountString,
|
||||
string market,
|
||||
string securityTypeString,
|
||||
string resolutionString,
|
||||
string dataDensityString,
|
||||
string includeCoarseString,
|
||||
string quoteTradeRatioString,
|
||||
string randomSeedString,
|
||||
string hasIpoPercentageString,
|
||||
string hasRenamePercentageString,
|
||||
string hasSplitsPercentageString,
|
||||
string hasDividendsPercentageString,
|
||||
string dividendEveryQuarterPercentageString,
|
||||
string optionPriceEngineName,
|
||||
string volatilityModelResolutionString,
|
||||
string chainSymbolCountString,
|
||||
List<string> tickers,
|
||||
double monthSplitPercentage = 5.0
|
||||
)
|
||||
{
|
||||
var randomSeedSet = true;
|
||||
|
||||
int randomSeed;
|
||||
int symbolCount;
|
||||
int chainSymbolCount;
|
||||
bool includeCoarse;
|
||||
Resolution resolution;
|
||||
double quoteTradeRatio;
|
||||
DataDensity dataDensity;
|
||||
SecurityType securityType;
|
||||
DateTime startDate, endDate;
|
||||
double hasIpoPercentage;
|
||||
double hasRenamePercentage;
|
||||
double hasSplitsPercentage;
|
||||
double hasDividendsPercentage;
|
||||
double dividendEveryQuarterPercentage;
|
||||
Resolution volatilityModelResolution;
|
||||
|
||||
var failed = false;
|
||||
// --start
|
||||
if (!DateTime.TryParseExact(startDateString, DateFormats, null, DateTimeStyles.None, out startDate))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Required parameter --from-date was incorrectly formatted. Please specify in yyyyMMdd format. Value provided: '{startDateString}'");
|
||||
}
|
||||
|
||||
// --end
|
||||
if (!DateTime.TryParseExact(endDateString, DateFormats, null, DateTimeStyles.None, out endDate))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Required parameter --to-date was incorrectly formatted. Please specify in yyyyMMdd format. Value provided: '{endDateString}'");
|
||||
}
|
||||
|
||||
// --tickers
|
||||
if (!tickers.IsNullOrEmpty())
|
||||
{
|
||||
symbolCount = tickers.Count;
|
||||
Log.Trace("RandomDataGeneratorSettings(): Ignoring symbol count will use provided tickers");
|
||||
}
|
||||
// --symbol-count
|
||||
else if (!int.TryParse(symbolCountString, out symbolCount) || symbolCount <= 0)
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Required parameter --symbol-count was incorrectly formatted. Please specify a valid integer greater than zero. Value provided: '{symbolCountString}'");
|
||||
}
|
||||
|
||||
// --chain-symbol-count
|
||||
if (!int.TryParse(chainSymbolCountString, out chainSymbolCount) || chainSymbolCount <= 0)
|
||||
{
|
||||
chainSymbolCount = 10;
|
||||
Log.Trace($"RandomDataGeneratorSettings(): Using default value of '{chainSymbolCount}' for --chain-symbol-count");
|
||||
}
|
||||
|
||||
// --resolution
|
||||
if (string.IsNullOrEmpty(resolutionString))
|
||||
{
|
||||
resolution = Resolution.Minute;
|
||||
Log.Trace($"RandomDataGeneratorSettings(): Using default value of '{resolution}' for --resolution");
|
||||
}
|
||||
else if (!Enum.TryParse(resolutionString, true, out resolution))
|
||||
{
|
||||
var validValues = string.Join(", ", Enum.GetValues(typeof(Resolution)).Cast<Resolution>());
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --resolution was incorrectly formatted. Default is Minute. Please specify a valid Resolution. Value provided: '{resolutionString}' Valid values: {validValues}");
|
||||
}
|
||||
|
||||
// --standard deviation volatility period span
|
||||
if (string.IsNullOrEmpty(volatilityModelResolutionString))
|
||||
{
|
||||
volatilityModelResolution = Resolution.Daily;
|
||||
Log.Trace($"RandomDataGeneratorSettings():Using default value of '{resolution}' for --resolution");
|
||||
}
|
||||
else if (!Enum.TryParse(volatilityModelResolutionString, true, out volatilityModelResolution))
|
||||
{
|
||||
var validValues = string.Join(", ", Enum.GetValues(typeof(Resolution)).Cast<Resolution>());
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --volatility-model-resolution was incorrectly formatted. Default is Daily. Please specify a valid Resolution. Value provided: '{volatilityModelResolutionString}' Valid values: {validValues}");
|
||||
}
|
||||
|
||||
// --security-type
|
||||
if (string.IsNullOrEmpty(securityTypeString))
|
||||
{
|
||||
securityType = SecurityType.Equity;
|
||||
Log.Trace($"RandomDataGeneratorSettings(): Using default value of '{securityType}' for --security-type");
|
||||
}
|
||||
else if (!Enum.TryParse(securityTypeString, true, out securityType))
|
||||
{
|
||||
var validValues = string.Join(", ", Enum.GetValues(typeof(SecurityType)).Cast<SecurityType>());
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --security-type is invalid. Default is Equity. Please specify a valid SecurityType. Value provided: '{securityTypeString}' Valid values: {validValues}");
|
||||
}
|
||||
|
||||
if (securityType == SecurityType.Option && resolution != Resolution.Minute)
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): When using --security-type=Option you must specify --resolution=Minute");
|
||||
}
|
||||
|
||||
// --market
|
||||
if (string.IsNullOrEmpty(market))
|
||||
{
|
||||
market = DefaultBrokerageModel.DefaultMarketMap[securityType];
|
||||
Log.Trace($"RandomDataGeneratorSettings(): Using default value of '{market}' for --market and --security-type={securityType}");
|
||||
}
|
||||
else if (QuantConnect.Market.Encode(market) == null)
|
||||
{
|
||||
// be sure to add a reference to the unknown market, otherwise we won't be able to decode it coming out
|
||||
QuantConnect.Market.Add(market, Interlocked.Increment(ref MarketCode));
|
||||
Log.Trace($"RandomDataGeneratorSettings(): Please verify that the specified market value is correct: '{market}' This value is not known has been added to the market value map. If this is an error, stop the application immediately using Ctrl+C");
|
||||
}
|
||||
|
||||
// --include-coarse
|
||||
if (string.IsNullOrEmpty(includeCoarseString))
|
||||
{
|
||||
includeCoarse = securityType == SecurityType.Equity;
|
||||
if (securityType != SecurityType.Equity)
|
||||
{
|
||||
Log.Trace($"RandomDataGeneratorSettings(): Using default value of '{includeCoarse}' for --security-type={securityType}");
|
||||
}
|
||||
}
|
||||
else if (!bool.TryParse(includeCoarseString, out includeCoarse))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --include-coarse was incorrectly formatted. Please specify a valid boolean. Value provided: '{includeCoarseString}'. Valid values: 'true' or 'false'");
|
||||
}
|
||||
else if (includeCoarse && securityType != SecurityType.Equity)
|
||||
{
|
||||
Log.Trace("RandomDataGeneratorSettings(): Optional parameter --include-coarse will be ignored because it only applies to --security-type=Equity");
|
||||
}
|
||||
|
||||
// --data-density
|
||||
if (string.IsNullOrEmpty(dataDensityString))
|
||||
{
|
||||
dataDensity = DataDensity.Dense;
|
||||
if (securityType == SecurityType.Option)
|
||||
{
|
||||
dataDensity = DataDensity.Sparse;
|
||||
}
|
||||
Log.Trace($"RandomDataGeneratorSettings(): Using default value of '{dataDensity}' for --data-density");
|
||||
}
|
||||
else if (!Enum.TryParse(dataDensityString, true, out dataDensity))
|
||||
{
|
||||
var validValues = string.Join(", ", Enum.GetValues(typeof(DataDensity))).Cast<DataDensity>();
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --data-density was incorrectly formatted. Please specify a valid DataDensity. Value provided: '{dataDensityString}'. Valid values: {validValues}");
|
||||
}
|
||||
|
||||
// --quote-trade-ratio
|
||||
if (string.IsNullOrEmpty(quoteTradeRatioString))
|
||||
{
|
||||
quoteTradeRatio = 1;
|
||||
}
|
||||
else if (!double.TryParse(quoteTradeRatioString, out quoteTradeRatio))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --quote-trade-ratio was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{quoteTradeRatioString}'");
|
||||
}
|
||||
|
||||
// --random-seed
|
||||
if (string.IsNullOrEmpty(randomSeedString))
|
||||
{
|
||||
randomSeed = 0;
|
||||
randomSeedSet = false;
|
||||
}
|
||||
else if (!int.TryParse(randomSeedString, out randomSeed))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --random-seed was incorrectly formatted. Please specify a valid integer");
|
||||
}
|
||||
|
||||
// --ipo-percentage
|
||||
if (string.IsNullOrEmpty(hasIpoPercentageString))
|
||||
{
|
||||
hasIpoPercentage = 5.0;
|
||||
}
|
||||
else if (!double.TryParse(hasIpoPercentageString, out hasIpoPercentage))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --ipo-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasIpoPercentageString}'");
|
||||
}
|
||||
|
||||
// --rename-percentage
|
||||
if (string.IsNullOrEmpty(hasRenamePercentageString))
|
||||
{
|
||||
hasRenamePercentage = 30.0;
|
||||
}
|
||||
else if (!double.TryParse(hasRenamePercentageString, out hasRenamePercentage))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --rename-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasRenamePercentageString}'");
|
||||
}
|
||||
|
||||
// --splits-percentage
|
||||
if (string.IsNullOrEmpty(hasSplitsPercentageString))
|
||||
{
|
||||
hasSplitsPercentage = 15.0;
|
||||
}
|
||||
else if (!double.TryParse(hasSplitsPercentageString, out hasSplitsPercentage))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --splits-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasSplitsPercentageString}'");
|
||||
}
|
||||
|
||||
// --dividends-percentage
|
||||
if (string.IsNullOrEmpty(hasDividendsPercentageString))
|
||||
{
|
||||
hasDividendsPercentage = 60.0;
|
||||
}
|
||||
else if (!double.TryParse(hasDividendsPercentageString, out hasDividendsPercentage))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --dividends-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasDividendsPercentageString}'");
|
||||
}
|
||||
|
||||
// --dividend-every-quarter-percentage
|
||||
if (string.IsNullOrEmpty(dividendEveryQuarterPercentageString))
|
||||
{
|
||||
dividendEveryQuarterPercentage = 30.0;
|
||||
}
|
||||
else if (!double.TryParse(dividendEveryQuarterPercentageString, out dividendEveryQuarterPercentage))
|
||||
{
|
||||
failed = true;
|
||||
Log.Error($"RandomDataGeneratorSettings(): Optional parameter --dividend-ever-quarter-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{dividendEveryQuarterPercentageString}'");
|
||||
}
|
||||
|
||||
if (failed)
|
||||
{
|
||||
Log.Error("RandomDataGeneratorSettings(): Please address the errors and run the application again.");
|
||||
Environment.Exit(-1);
|
||||
}
|
||||
|
||||
return new RandomDataGeneratorSettings
|
||||
{
|
||||
End = endDate,
|
||||
Start = startDate,
|
||||
|
||||
Market = market,
|
||||
SymbolCount = symbolCount,
|
||||
SecurityType = securityType,
|
||||
QuoteTradeRatio = quoteTradeRatio,
|
||||
ChainSymbolCount = chainSymbolCount,
|
||||
|
||||
Resolution = resolution,
|
||||
|
||||
DataDensity = dataDensity,
|
||||
IncludeCoarse = includeCoarse,
|
||||
RandomSeed = randomSeed,
|
||||
RandomSeedSet = randomSeedSet,
|
||||
|
||||
HasIpoPercentage = hasIpoPercentage,
|
||||
HasRenamePercentage = hasRenamePercentage,
|
||||
HasSplitsPercentage = hasSplitsPercentage,
|
||||
MonthSplitPercentage = monthSplitPercentage,
|
||||
HasDividendsPercentage = hasDividendsPercentage,
|
||||
DividendEveryQuarterPercentage = dividendEveryQuarterPercentage,
|
||||
OptionPriceEngineName = optionPriceEngineName,
|
||||
VolatilityModelResolution = volatilityModelResolution,
|
||||
Tickers = tickers
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using QuantConnect.Securities;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Random pricing model used to determine the fair price or theoretical value for a call or a put option
|
||||
/// </summary>
|
||||
public class RandomPriceGenerator : IPriceGenerator
|
||||
{
|
||||
private readonly Security _security;
|
||||
private readonly IRandomValueGenerator _random;
|
||||
|
||||
/// <summary>
|
||||
/// Creates instance of <see cref="RandomPriceGenerator"/>
|
||||
/// </summary>
|
||||
///<param name="security"><see cref="Security"/> object for which to generate price data</param>
|
||||
/// <param name="random"><see cref="IRandomValueGenerator"/> type capable of producing random values</param>
|
||||
public RandomPriceGenerator(Security security, IRandomValueGenerator random)
|
||||
{
|
||||
_security = security;
|
||||
_random = random;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="RandomPriceGenerator"/> is always ready to generate new price values as it does not depend on volatility model
|
||||
/// </summary>
|
||||
public bool WarmedUp => true;
|
||||
|
||||
/// <summary>
|
||||
/// Generates an asset price
|
||||
/// </summary>
|
||||
/// <param name="maximumPercentDeviation">The maximum percent deviation. This value is in percent space,
|
||||
/// so a value of 1m is equal to 1%.</param>
|
||||
/// <param name="referenceDate">date used in price calculation</param>
|
||||
/// <returns>Returns a new decimal as price</returns>
|
||||
public decimal NextValue(decimal maximumPercentDeviation, DateTime referenceDate)
|
||||
=> _random.NextPrice(_security.Symbol.SecurityType, _security.Symbol.ID.Market, _security.Price, maximumPercentDeviation);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Securities;
|
||||
using QuantConnect.Util;
|
||||
using System;
|
||||
using System.Linq;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IRandomValueGenerator"/> that uses
|
||||
/// <see cref="Random"/> to generate random values
|
||||
/// </summary>
|
||||
public class RandomValueGenerator : IRandomValueGenerator
|
||||
{
|
||||
private readonly Random _random;
|
||||
private readonly MarketHoursDatabase _marketHoursDatabase;
|
||||
private readonly SymbolPropertiesDatabase _symbolPropertiesDatabase;
|
||||
private const decimal _maximumPriceAllowed = 1000000m;
|
||||
|
||||
|
||||
public RandomValueGenerator()
|
||||
: this(new Random())
|
||||
{ }
|
||||
|
||||
public RandomValueGenerator(int seed)
|
||||
: this(new Random(seed))
|
||||
{ }
|
||||
|
||||
public RandomValueGenerator(Random random)
|
||||
: this(random, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder())
|
||||
{ }
|
||||
|
||||
public RandomValueGenerator(
|
||||
int seed,
|
||||
MarketHoursDatabase marketHoursDatabase,
|
||||
SymbolPropertiesDatabase symbolPropertiesDatabase
|
||||
)
|
||||
: this(new Random(seed), marketHoursDatabase, symbolPropertiesDatabase)
|
||||
{ }
|
||||
|
||||
public RandomValueGenerator(Random random, MarketHoursDatabase marketHoursDatabase, SymbolPropertiesDatabase symbolPropertiesDatabase)
|
||||
{
|
||||
_random = random;
|
||||
_marketHoursDatabase = marketHoursDatabase;
|
||||
_symbolPropertiesDatabase = symbolPropertiesDatabase;
|
||||
}
|
||||
|
||||
public bool NextBool(double percentOddsForTrue)
|
||||
{
|
||||
return _random.NextDouble() <= percentOddsForTrue / 100;
|
||||
}
|
||||
|
||||
public virtual DateTime NextDate(DateTime minDateTime, DateTime maxDateTime, DayOfWeek? dayOfWeek)
|
||||
{
|
||||
if (maxDateTime < minDateTime)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
"The maximum date time must be less than or equal to the minimum date time specified"
|
||||
);
|
||||
}
|
||||
|
||||
// compute a random date time value
|
||||
var rangeInDays = (int)maxDateTime.Subtract(minDateTime).TotalDays;
|
||||
var daysOffsetFromMin = _random.Next(0, rangeInDays);
|
||||
var dateTime = minDateTime.AddDays(daysOffsetFromMin);
|
||||
|
||||
var currentDayOfWeek = dateTime.DayOfWeek;
|
||||
if (!dayOfWeek.HasValue || currentDayOfWeek == dayOfWeek.Value)
|
||||
{
|
||||
// either DOW wasn't specified or we got REALLY lucky, although, I suppose it'll happen 1/7 (~14%) of the time
|
||||
return dateTime;
|
||||
}
|
||||
|
||||
var nextDayOfWeek = Enumerable.Range(0, 7)
|
||||
.Select(i => dateTime.AddDays(i))
|
||||
.First(dt => dt.DayOfWeek == dayOfWeek.Value);
|
||||
|
||||
var previousDayOfWeek = Enumerable.Range(0, 7)
|
||||
.Select(i => dateTime.AddDays(-i))
|
||||
.First(dt => dt.DayOfWeek == dayOfWeek.Value);
|
||||
|
||||
// both are valid dates, so chose one randomly
|
||||
if (IsWithinRange(nextDayOfWeek, minDateTime, maxDateTime) &&
|
||||
IsWithinRange(previousDayOfWeek, minDateTime, maxDateTime)
|
||||
)
|
||||
{
|
||||
return _random.Next(0, 1) == 0
|
||||
? previousDayOfWeek
|
||||
: nextDayOfWeek;
|
||||
}
|
||||
|
||||
if (IsWithinRange(nextDayOfWeek, minDateTime, maxDateTime))
|
||||
{
|
||||
return nextDayOfWeek;
|
||||
}
|
||||
|
||||
if (IsWithinRange(previousDayOfWeek, minDateTime, maxDateTime))
|
||||
{
|
||||
return previousDayOfWeek;
|
||||
}
|
||||
|
||||
throw new ArgumentException("The provided min and max dates do not have the requested day of week between them");
|
||||
}
|
||||
|
||||
public double NextDouble() => _random.NextDouble();
|
||||
|
||||
public int NextInt(int minValue, int maxValue) => _random.Next(minValue, maxValue);
|
||||
|
||||
public int NextInt(int maxValue) => _random.Next(maxValue);
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random <see cref="decimal"/> suitable as a price. This should observe minimum price
|
||||
/// variations if available in <see cref="SymbolPropertiesDatabase"/>, and if not, truncating to 2
|
||||
/// decimal places.
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">Throw when the <paramref name="referencePrice"/> or <paramref name="maximumPercentDeviation"/>
|
||||
/// is less than or equal to zero.</exception>
|
||||
/// <param name="securityType">The security type the price is being generated for</param>
|
||||
/// <param name="market">The market of the security the price is being generated for</param>
|
||||
/// <param name="referencePrice">The reference price used as the mean of random price generation</param>
|
||||
/// <param name="maximumPercentDeviation">The maximum percent deviation. This value is in percent space,
|
||||
/// so a value of 1m is equal to 1%.</param>
|
||||
/// <returns>A new decimal suitable for usage as price within the specified deviation from the reference price</returns>
|
||||
public virtual decimal NextPrice(SecurityType securityType, string market, decimal referencePrice, decimal maximumPercentDeviation)
|
||||
{
|
||||
if (referencePrice <= 0)
|
||||
{
|
||||
if (securityType == SecurityType.Option && referencePrice == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
throw new ArgumentException("The provided reference price must be a positive number.");
|
||||
}
|
||||
|
||||
if (maximumPercentDeviation <= 0)
|
||||
{
|
||||
throw new ArgumentException("The provided maximum percent deviation must be a positive number");
|
||||
}
|
||||
|
||||
// convert from percent space to decimal space
|
||||
maximumPercentDeviation /= 100m;
|
||||
|
||||
var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(market, null, securityType, "USD");
|
||||
var minimumPriceVariation = symbolProperties.MinimumPriceVariation;
|
||||
|
||||
decimal price;
|
||||
var attempts = 0;
|
||||
var increaseProbabilityFactor = 0.5;
|
||||
do
|
||||
{
|
||||
// what follows is a simple model of browning motion that
|
||||
// limits the walk to the specified percent deviation
|
||||
|
||||
var deviation = referencePrice * maximumPercentDeviation * (decimal)(NextDouble() - increaseProbabilityFactor);
|
||||
deviation = Math.Sign(deviation) * Math.Max(Math.Abs(deviation), minimumPriceVariation);
|
||||
price = referencePrice + deviation;
|
||||
price = RoundPrice(price, minimumPriceVariation);
|
||||
|
||||
if (price < 20 * minimumPriceVariation)
|
||||
{
|
||||
// The price should not be to close to the minimum price variation.
|
||||
// Invalidate the price to try again and increase the probability of it to going up
|
||||
price = -1m;
|
||||
increaseProbabilityFactor = Math.Max(increaseProbabilityFactor - 0.05, 0);
|
||||
}
|
||||
|
||||
if (price > (_maximumPriceAllowed / 10m))
|
||||
{
|
||||
// The price should not be too higher
|
||||
// Decrease the probability of it to going up
|
||||
increaseProbabilityFactor = increaseProbabilityFactor + 0.05;
|
||||
}
|
||||
|
||||
if (price > _maximumPriceAllowed)
|
||||
{
|
||||
// The price should not be too higher
|
||||
// Invalidate the price to try again
|
||||
price = -1;
|
||||
}
|
||||
} while (!IsPriceValid(securityType, price) && ++attempts < 10);
|
||||
|
||||
if (!IsPriceValid(securityType, price))
|
||||
{
|
||||
// if still invalid, use the last price
|
||||
price = referencePrice;
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
|
||||
private static decimal RoundPrice(decimal price, decimal minimumPriceVariation)
|
||||
{
|
||||
if (minimumPriceVariation == 0) return minimumPriceVariation;
|
||||
return Math.Round(price / minimumPriceVariation) * minimumPriceVariation;
|
||||
}
|
||||
|
||||
private bool IsWithinRange(DateTime value, DateTime min, DateTime max)
|
||||
{
|
||||
return value >= min && value <= max;
|
||||
}
|
||||
|
||||
private static bool IsPriceValid(SecurityType securityType, decimal price)
|
||||
{
|
||||
switch (securityType)
|
||||
{
|
||||
case SecurityType.Option:
|
||||
{
|
||||
return price >= 0;
|
||||
}
|
||||
default:
|
||||
{
|
||||
return price > 0 && price < _maximumPriceAllowed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using System;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides a base class for exceptions thrown by implementations of <see cref="IRandomValueGenerator"/>
|
||||
/// </summary>
|
||||
public class RandomValueGeneratorException : ApplicationException
|
||||
{
|
||||
public RandomValueGeneratorException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Securities;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
public class SecurityInitializerProvider : ISecurityInitializerProvider
|
||||
{
|
||||
public ISecurityInitializer SecurityInitializer { get; }
|
||||
|
||||
public SecurityInitializerProvider(ISecurityInitializer securityInitializer)
|
||||
{
|
||||
SecurityInitializer = securityInitializer;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Securities;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using QuantConnect.Logging;
|
||||
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates random tick data according to the settings provided
|
||||
/// </summary>
|
||||
public class TickGenerator : ITickGenerator
|
||||
{
|
||||
private readonly IPriceGenerator _priceGenerator;
|
||||
private Symbol Symbol => Security.Symbol;
|
||||
|
||||
private readonly IRandomValueGenerator _random;
|
||||
private readonly RandomDataGeneratorSettings _settings;
|
||||
private readonly TickType[] _tickTypes;
|
||||
|
||||
private MarketHoursDatabase MarketHoursDatabase { get; }
|
||||
private SymbolPropertiesDatabase SymbolPropertiesDatabase { get; }
|
||||
private Security Security { get; }
|
||||
|
||||
public TickGenerator(RandomDataGeneratorSettings settings, TickType[] tickTypes, Security security, IRandomValueGenerator random)
|
||||
{
|
||||
_random = random;
|
||||
_settings = settings;
|
||||
_tickTypes = tickTypes;
|
||||
Security = security;
|
||||
SymbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
|
||||
MarketHoursDatabase = MarketHoursDatabase.FromDataFolder();
|
||||
|
||||
if (Symbol.SecurityType.IsOption())
|
||||
{
|
||||
_priceGenerator = new OptionPriceModelPriceGenerator(security);
|
||||
}
|
||||
else
|
||||
{
|
||||
_priceGenerator = new RandomPriceGenerator(security, random);
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<Tick> GenerateTicks()
|
||||
{
|
||||
var current = _settings.Start;
|
||||
|
||||
// There is a possibility that even though this succeeds, the DateTime
|
||||
// generated may be the same as the starting DateTime, although the probability
|
||||
// of this happening diminishes the longer the period we're generating data for is
|
||||
if (_random.NextBool(_settings.HasIpoPercentage))
|
||||
{
|
||||
current = _random.NextDate(_settings.Start, _settings.End, null);
|
||||
Log.Trace($"\tSymbol: {Symbol} has delayed IPO at date {current:yyyy MMMM dd}");
|
||||
}
|
||||
|
||||
// creates a max deviation that scales parabolically as resolution decreases (lower frequency)
|
||||
var deviation = GetMaximumDeviation(_settings.Resolution);
|
||||
while (current <= _settings.End)
|
||||
{
|
||||
|
||||
var next = NextTickTime(current, _settings.Resolution, _settings.DataDensity);
|
||||
// The current date can be the last one of the last day before the market closes
|
||||
// so the next date could be beyond de end date
|
||||
if (next > _settings.End)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (_tickTypes.Contains(TickType.OpenInterest))
|
||||
{
|
||||
if (next.Date != current.Date)
|
||||
{
|
||||
// 5% deviation in daily OI
|
||||
var openInterest = NextTick(next.Date, TickType.OpenInterest, 5m);
|
||||
yield return openInterest;
|
||||
}
|
||||
}
|
||||
|
||||
Tick nextTick = null;
|
||||
// keeps quotes close to the trades for consistency
|
||||
if (_tickTypes.Contains(TickType.Trade) &&
|
||||
_tickTypes.Contains(TickType.Quote))
|
||||
{
|
||||
// %odds of getting a trade tick, for example, a quote:trade ratio of 2 means twice as likely
|
||||
// to get a quote, which means you have a 33% chance of getting a trade => 1/3
|
||||
var tradeChancePercent = 100 / (1 + _settings.QuoteTradeRatio);
|
||||
nextTick = NextTick(
|
||||
next,
|
||||
_random.NextBool(tradeChancePercent)
|
||||
? TickType.Trade
|
||||
: TickType.Quote,
|
||||
deviation);
|
||||
}
|
||||
else if (_tickTypes.Contains(TickType.Trade))
|
||||
{
|
||||
nextTick = NextTick(next, TickType.Trade, deviation);
|
||||
|
||||
}
|
||||
else if (_tickTypes.Contains(TickType.Quote))
|
||||
{
|
||||
nextTick = NextTick(next, TickType.Quote, deviation);
|
||||
}
|
||||
|
||||
if (nextTick != null && _priceGenerator.WarmedUp)
|
||||
{
|
||||
yield return nextTick;
|
||||
}
|
||||
|
||||
// advance to the next time step
|
||||
current = next;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random <see cref="Tick"/> that is at most the specified <paramref name="maximumPercentDeviation"/> away from the
|
||||
/// previous price and is of the requested <paramref name="tickType"/>
|
||||
/// </summary>
|
||||
/// <param name="dateTime">The time of the generated tick</param>
|
||||
/// <param name="tickType">The type of <see cref="Tick"/> to be generated</param>
|
||||
/// <param name="maximumPercentDeviation">The maximum percentage to deviate from the
|
||||
/// previous price for example, 1 would indicate a maximum of 1% deviation from the
|
||||
/// previous price. For a previous price of 100, this would yield a price between 99 and 101 inclusive</param>
|
||||
/// <returns>A random <see cref="Tick"/> value that is within the specified <paramref name="maximumPercentDeviation"/>
|
||||
/// from the previous price</returns>
|
||||
public virtual Tick NextTick(DateTime dateTime, TickType tickType, decimal maximumPercentDeviation)
|
||||
{
|
||||
var next = _priceGenerator.NextValue(maximumPercentDeviation, dateTime);
|
||||
var tick = new Tick
|
||||
{
|
||||
Time = dateTime,
|
||||
Symbol = Symbol,
|
||||
TickType = tickType,
|
||||
Value = next
|
||||
};
|
||||
|
||||
switch (tickType)
|
||||
{
|
||||
case TickType.OpenInterest:
|
||||
return NextOpenInterest(dateTime, Security.OpenInterest, maximumPercentDeviation);
|
||||
|
||||
case TickType.Trade:
|
||||
tick.Quantity = _random.NextInt(1, 1500);
|
||||
return tick;
|
||||
|
||||
case TickType.Quote:
|
||||
var bid = _random.NextPrice(Symbol.SecurityType, Symbol.ID.Market, tick.Value, maximumPercentDeviation);
|
||||
if (bid > tick.Value)
|
||||
{
|
||||
bid = tick.Value - (bid - tick.Value);
|
||||
}
|
||||
var ask = _random.NextPrice(Symbol.SecurityType, Symbol.ID.Market, tick.Value, maximumPercentDeviation);
|
||||
if (ask < tick.Value)
|
||||
{
|
||||
ask = tick.Value + (tick.Value - ask);
|
||||
}
|
||||
|
||||
tick.BidPrice = bid;
|
||||
tick.BidSize = _random.NextInt(1, 1500);
|
||||
tick.AskPrice = ask;
|
||||
tick.AskSize = _random.NextInt(1, 1500);
|
||||
return tick;
|
||||
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(tickType), tickType, null);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random <see cref="Tick"/> that is at most the specified <paramref name="maximumPercentDeviation"/> away from the
|
||||
/// <paramref name="previousValue"/> and is of the Open Interest
|
||||
/// </summary>
|
||||
/// <param name="dateTime">The time of the generated tick</param>
|
||||
/// <param name="previousValue">The previous price, used as a reference for generating
|
||||
/// new random prices for the next time step</param>
|
||||
/// <param name="maximumPercentDeviation">The maximum percentage to deviate from the
|
||||
/// <paramref name="previousValue"/>, for example, 1 would indicate a maximum of 1% deviation from the
|
||||
/// <paramref name="previousValue"/>. For a previous price of 100, this would yield a price between 99 and 101 inclusive</param>
|
||||
/// <returns>A random <see cref="Tick"/> value that is within the specified <paramref name="maximumPercentDeviation"/>
|
||||
/// from the <paramref name="previousValue"/></returns>
|
||||
public Tick NextOpenInterest(DateTime dateTime, decimal previousValue, decimal maximumPercentDeviation)
|
||||
{
|
||||
var next = (long)_random.NextPrice(Symbol.SecurityType, Symbol.ID.Market, previousValue, maximumPercentDeviation);
|
||||
return new OpenInterest
|
||||
{
|
||||
Time = dateTime,
|
||||
Symbol = Symbol,
|
||||
TickType = TickType.OpenInterest,
|
||||
Value = next,
|
||||
Quantity = next
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a random <see cref="DateTime"/> suitable for use as a tick's emit time.
|
||||
/// If the density provided is <see cref="DataDensity.Dense"/>, then at least one tick will be generated per <paramref name="resolution"/> step.
|
||||
/// If the density provided is <see cref="DataDensity.Sparse"/>, then at least one tick will be generated every 5 <paramref name="resolution"/> steps.
|
||||
/// if the density provided is <see cref="DataDensity.VerySparse"/>, then at least one tick will be generated every 50 <paramref name="resolution"/> steps.
|
||||
/// Times returned are guaranteed to be within market hours for the specified Symbol
|
||||
/// </summary>
|
||||
/// <param name="previous">The previous tick time</param>
|
||||
/// <param name="resolution">The requested resolution of data</param>
|
||||
/// <param name="density">The requested data density</param>
|
||||
/// <returns>A new <see cref="DateTime"/> that is after <paramref name="previous"/> according to the specified <paramref name="resolution"/>
|
||||
/// and <paramref name="density"/> specified</returns>
|
||||
public virtual DateTime NextTickTime(DateTime previous, Resolution resolution, DataDensity density)
|
||||
{
|
||||
var increment = resolution.ToTimeSpan();
|
||||
if (increment == TimeSpan.Zero)
|
||||
{
|
||||
increment = TimeSpan.FromMilliseconds(500);
|
||||
}
|
||||
|
||||
double steps;
|
||||
switch (density)
|
||||
{
|
||||
case DataDensity.Dense:
|
||||
steps = 0.5 * _random.NextDouble();
|
||||
break;
|
||||
|
||||
case DataDensity.Sparse:
|
||||
steps = 5 * _random.NextDouble();
|
||||
break;
|
||||
|
||||
case DataDensity.VerySparse:
|
||||
steps = 50 * _random.NextDouble();
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(density), density, null);
|
||||
}
|
||||
|
||||
var delta = TimeSpan.FromTicks((long)(steps * increment.Ticks));
|
||||
var tickTime = previous.Add(delta);
|
||||
if (tickTime == previous)
|
||||
{
|
||||
tickTime = tickTime.Add(increment);
|
||||
}
|
||||
|
||||
var barStart = tickTime.Subtract(increment);
|
||||
var marketHours = MarketHoursDatabase.GetExchangeHours(Symbol.ID.Market, Symbol, Symbol.SecurityType);
|
||||
if (!marketHours.IsDateOpen(tickTime) || !marketHours.IsOpen(barStart, tickTime, false))
|
||||
{
|
||||
// we ended up outside of market hours, emit a new tick at market open
|
||||
var nextMarketOpen = marketHours.GetNextMarketOpen(tickTime, false);
|
||||
if (resolution == Resolution.Tick)
|
||||
{
|
||||
resolution = Resolution.Second;
|
||||
}
|
||||
|
||||
// emit a new tick somewhere in the next trading day at a step higher resolution to guarantee a hit
|
||||
return NextTickTime(nextMarketOpen, resolution - 1, density);
|
||||
}
|
||||
|
||||
return tickTime;
|
||||
}
|
||||
|
||||
private static decimal GetMaximumDeviation(Resolution resolution)
|
||||
{
|
||||
var incr = ((int)resolution) + 0.15m;
|
||||
var deviation = incr * incr * 0.1m;
|
||||
return deviation;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace QuantConnect.ToolBox.RandomDataGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Exception thrown when multiple attempts to generate a valid random value end in failure
|
||||
/// </summary>
|
||||
public class TooManyFailedAttemptsException : RandomValueGeneratorException
|
||||
{
|
||||
public TooManyFailedAttemptsException(string method, int attempts)
|
||||
: base($"Failed to generate a valid value for '{method}' after {attempts} attempts.")
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using QuantConnect.Logging;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Processing harness used to read files in, parse them, and process them.
|
||||
/// </summary>
|
||||
public class RawFileProcessor : IDisposable
|
||||
{
|
||||
private DateTime? _start;
|
||||
private readonly IStreamProvider _streamProvider;
|
||||
private readonly IStreamParser _parser;
|
||||
private readonly IDataProcessor[] _processors;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a name used for logging
|
||||
/// </summary>
|
||||
public string Name { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="RawFileProcessor"/> class
|
||||
/// </summary>
|
||||
public RawFileProcessor(IStreamProvider streamProvider, IStreamParser parser, params IDataProcessor[] processors)
|
||||
{
|
||||
_streamProvider = streamProvider;
|
||||
_parser = parser;
|
||||
_processors = processors;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the raw file processor on the specified files
|
||||
/// </summary>
|
||||
/// <param name="name">A name for the processor used for logging</param>
|
||||
/// <param name="sources">The raw files to be processed</param>
|
||||
/// <param name="streamProvider">Instance capable of reading the sources into a stream</param>
|
||||
/// <param name="streamParser">Instance capable of parsing the provided stream</param>
|
||||
/// <param name="processors">The data processors to process the parsed data</param>
|
||||
/// <returns>True if the operation completed without error, otherwise false</returns>
|
||||
public static bool Run(string name, IEnumerable<string> sources, IStreamProvider streamProvider, IStreamParser streamParser, params IDataProcessor[] processors)
|
||||
{
|
||||
using (var processor = new RawFileProcessor(streamProvider, streamParser, processors) { Name = name })
|
||||
{
|
||||
foreach (var zip in sources)
|
||||
{
|
||||
try
|
||||
{
|
||||
processor.Process(zip);
|
||||
}
|
||||
catch (Exception err)
|
||||
{
|
||||
Log.Error(err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Perform processing on the specified source file
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be processed</param>
|
||||
public void Process(string source)
|
||||
{
|
||||
_start = _start ?? DateTime.UtcNow;
|
||||
|
||||
// process the source file
|
||||
foreach (var stream in _streamProvider.Open(source))
|
||||
{
|
||||
using (stream)
|
||||
{
|
||||
foreach (var data in _parser.Parse(source, stream))
|
||||
{
|
||||
foreach (var processor in _processors)
|
||||
{
|
||||
processor.Process(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Log.Trace("RawFileProcessor.Process({0}): Finished.", source);
|
||||
_streamProvider.Close(source);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_streamProvider.Dispose();
|
||||
_parser.Dispose();
|
||||
foreach (var processor in _processors)
|
||||
{
|
||||
processor.Dispose();
|
||||
}
|
||||
|
||||
if (_start.HasValue)
|
||||
{
|
||||
var stop = DateTime.UtcNow;
|
||||
Log.Trace("RawFileProcessor.Dispose({0}): Elapsed {1}", Name, stop - _start);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Helper method that provides and cleans given temporary paths
|
||||
/// </summary>
|
||||
public static class TemporaryPathProvider
|
||||
{
|
||||
private static readonly Queue<string> TemporaryPaths = new Queue<string>();
|
||||
|
||||
// Gets a new temporary path
|
||||
public static string Get()
|
||||
{
|
||||
var newPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToStringInvariant(null));
|
||||
lock (TemporaryPaths)
|
||||
{
|
||||
TemporaryPaths.Enqueue(newPath);
|
||||
}
|
||||
return newPath;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Recursively deletes all the given temporary paths
|
||||
/// </summary>
|
||||
public static void Delete()
|
||||
{
|
||||
List<string> paths;
|
||||
lock (TemporaryPaths)
|
||||
{
|
||||
paths = TemporaryPaths.ToList(s => s);
|
||||
TemporaryPaths.Clear();
|
||||
}
|
||||
Parallel.ForEach(paths, path =>
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(path, recursive: true);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// pass
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using QuantConnect.Data;
|
||||
using QuantConnect.Data.Consolidators;
|
||||
using QuantConnect.Data.Market;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Class that uses consolidators to aggregate tick data data
|
||||
/// </summary>
|
||||
public abstract class TickAggregator
|
||||
{
|
||||
protected TickAggregator(Resolution resolution, TickType tickType)
|
||||
{
|
||||
TickType = tickType;
|
||||
Resolution = resolution;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the tick type of the consolidator
|
||||
/// </summary>
|
||||
public TickType TickType { get; protected set; }
|
||||
|
||||
/// <summary>
|
||||
/// The consolidator used to aggregate data from
|
||||
/// higher resolutions to data in lower resolutions
|
||||
/// </summary>
|
||||
public IDataConsolidator Consolidator { get; protected set; }
|
||||
|
||||
/// <summary>
|
||||
/// The consolidated data
|
||||
/// </summary>
|
||||
public List<BaseData> Consolidated { get; protected set; }
|
||||
|
||||
/// <summary>
|
||||
/// The resolution that the data is being aggregated into
|
||||
/// </summary>
|
||||
public Resolution Resolution { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Updates the consolidator with the specified bar.
|
||||
/// </summary>
|
||||
/// <param name="data">The latest data observation.</param>
|
||||
public virtual void Update(BaseData data)
|
||||
{
|
||||
Consolidator.Update(data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Return all the consolidated data as well as the
|
||||
/// bar the consolidator is currently working on
|
||||
/// </summary>
|
||||
public List<BaseData> Flush()
|
||||
{
|
||||
var data = new List<BaseData>(Consolidated);
|
||||
if (Consolidator.WorkingData != null)
|
||||
{
|
||||
data.Add(Consolidator.WorkingData as BaseData);
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the correct <see cref="TickAggregator"/> instances for the specified tick types and resolution.
|
||||
/// <see cref="QuantConnect.TickType.OpenInterest"/> will ignore <paramref name="resolution"/> and use <see cref="QuantConnect.Resolution.Daily"/>
|
||||
/// </summary>
|
||||
public static IEnumerable<TickAggregator> ForTickTypes(SecurityType securityType, Resolution resolution, params TickType[] tickTypes)
|
||||
{
|
||||
if (resolution == Resolution.Tick)
|
||||
{
|
||||
foreach (var tickType in tickTypes.Where(t => LeanData.IsValidConfiguration(securityType, resolution, t)))
|
||||
{
|
||||
// OI is special
|
||||
if (tickType == TickType.OpenInterest)
|
||||
{
|
||||
yield return new OpenInterestTickAggregator(resolution);
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return new IdentityTickAggregator(tickType);
|
||||
}
|
||||
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var tickType in tickTypes.Where(t => LeanData.IsValidConfiguration(securityType, resolution, t)))
|
||||
{
|
||||
switch (tickType)
|
||||
{
|
||||
case TickType.Trade:
|
||||
yield return new TradeTickAggregator(resolution);
|
||||
break;
|
||||
|
||||
case TickType.Quote:
|
||||
yield return new QuoteTickAggregator(resolution);
|
||||
break;
|
||||
|
||||
case TickType.OpenInterest:
|
||||
yield return new OpenInterestTickAggregator(resolution);
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(tickType), tickType, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Use <see cref="TickQuoteBarConsolidator"/> to consolidate quote ticks into a specified resolution
|
||||
/// </summary>
|
||||
public class QuoteTickAggregator : TickAggregator
|
||||
{
|
||||
public QuoteTickAggregator(Resolution resolution)
|
||||
: base(resolution, TickType.Quote)
|
||||
{
|
||||
Consolidated = new List<BaseData>();
|
||||
Consolidator = new TickQuoteBarConsolidator(resolution.ToTimeSpan());
|
||||
Consolidator.DataConsolidated += (sender, consolidated) =>
|
||||
{
|
||||
Consolidated.Add(consolidated as QuoteBar);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Use <see cref="TickQuoteBarConsolidator"/> to consolidate trade ticks into a specified resolution
|
||||
/// </summary>
|
||||
public class TradeTickAggregator : TickAggregator
|
||||
{
|
||||
public TradeTickAggregator(Resolution resolution)
|
||||
: base(resolution, TickType.Trade)
|
||||
{
|
||||
Consolidated = new List<BaseData>();
|
||||
Consolidator = new TickConsolidator(resolution.ToTimeSpan());
|
||||
Consolidator.DataConsolidated += (sender, consolidated) =>
|
||||
{
|
||||
Consolidated.Add(consolidated as TradeBar);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Use <see cref="OpenInterestConsolidator"/> to consolidate open interest ticks into a specified resolution
|
||||
/// </summary>
|
||||
public class OpenInterestTickAggregator : TickAggregator
|
||||
{
|
||||
public OpenInterestTickAggregator(Resolution resolution)
|
||||
: base(resolution, TickType.OpenInterest)
|
||||
{
|
||||
Consolidated = new List<BaseData>();
|
||||
Consolidator = new OpenInterestConsolidator(resolution.ToTimeSpan());
|
||||
Consolidator.DataConsolidated += (sender, consolidated) =>
|
||||
{
|
||||
Consolidated.Add(consolidated as OpenInterest);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Use <see cref="IdentityDataConsolidator{T}"/> to yield ticks unmodified into the consolidated data collection
|
||||
/// </summary>
|
||||
public class IdentityTickAggregator : TickAggregator
|
||||
{
|
||||
public IdentityTickAggregator(TickType tickType)
|
||||
: base(Resolution.Tick, tickType)
|
||||
{
|
||||
Consolidated = new List<BaseData>();
|
||||
Consolidator = FilteredIdentityDataConsolidator.ForTickType(tickType);
|
||||
Consolidator.DataConsolidated += (sender, consolidated) =>
|
||||
{
|
||||
Consolidated.Add(consolidated as Tick);
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Ionic.Zip;
|
||||
|
||||
namespace QuantConnect.ToolBox
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides an implementation of <see cref="IStreamProvider"/> that opens zip files
|
||||
/// </summary>
|
||||
public class ZipStreamProvider : IStreamProvider
|
||||
{
|
||||
private readonly object _sync = new object();
|
||||
private readonly Dictionary<string, ZipFile> _zipFiles = new Dictionary<string, ZipFile>();
|
||||
|
||||
/// <summary>
|
||||
/// Opens the specified source as read to be consumed stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be opened</param>
|
||||
/// <returns>The stream representing the specified source</returns>
|
||||
public IEnumerable<Stream> Open(string source)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
var archive = new ZipFile(source);
|
||||
_zipFiles.Add(source, archive);
|
||||
foreach (var entry in archive)
|
||||
{
|
||||
yield return entry.OpenReader();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Closes the specified source file stream
|
||||
/// </summary>
|
||||
/// <param name="source">The source file to be closed</param>
|
||||
public void Close(string source)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
ZipFile archive;
|
||||
if (_zipFiles.TryGetValue(source, out archive))
|
||||
{
|
||||
_zipFiles.Remove(source);
|
||||
archive.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
foreach (var zipFile in _zipFiles.Values)
|
||||
{
|
||||
zipFile.Dispose();
|
||||
}
|
||||
_zipFiles.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user