chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,209 @@
|
||||
/*
|
||||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
||||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using QuantConnect.Interfaces;
|
||||
using QuantConnect.Logging;
|
||||
using QuantConnect.Util;
|
||||
|
||||
namespace QuantConnect.Lean.Engine.DataFeeds.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a stream reader capabable of downloading a remote file and then
|
||||
/// reading it from disk
|
||||
/// </summary>
|
||||
public class RemoteFileSubscriptionStreamReader : IStreamReader
|
||||
{
|
||||
private readonly IStreamReader _streamReader;
|
||||
private static IDownloadProvider _downloader;
|
||||
// lock for multi thread scenarios where we are sharing the same cached file
|
||||
private static readonly object _fileSystemLock = new object();
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether or not this stream reader should be rate limited
|
||||
/// </summary>
|
||||
public bool ShouldBeRateLimited => false;
|
||||
|
||||
/// <summary>
|
||||
/// Direct access to the StreamReader instance
|
||||
/// </summary>
|
||||
public StreamReader StreamReader => _streamReader.StreamReader;
|
||||
|
||||
/// <summary>
|
||||
/// The local file name of the downloaded file
|
||||
/// </summary>
|
||||
public string LocalFileName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="RemoteFileSubscriptionStreamReader"/> class.
|
||||
/// </summary>
|
||||
/// <param name="dataCacheProvider">The <see cref="IDataCacheProvider"/> used to retrieve a stream of data</param>
|
||||
/// <param name="source">The remote url to be downloaded via web client</param>
|
||||
/// <param name="downloadDirectory">The local directory and destination of the download</param>
|
||||
/// <param name="headers">Defines header values to add to the request</param>
|
||||
public RemoteFileSubscriptionStreamReader(IDataCacheProvider dataCacheProvider, string source, string downloadDirectory, IEnumerable<KeyValuePair<string, string>> headers)
|
||||
{
|
||||
// don't use cache if data is ephemeral
|
||||
// will be false for live history requests and live subscriptions
|
||||
var useCache = !dataCacheProvider.IsDataEphemeral;
|
||||
|
||||
// create a hash for a new filename
|
||||
string baseFileName = string.Empty;
|
||||
string extension = string.Empty;
|
||||
string entryName = string.Empty;
|
||||
try
|
||||
{
|
||||
var uri = new Uri(source);
|
||||
baseFileName = uri.OriginalString;
|
||||
if (!string.IsNullOrEmpty(uri.Fragment))
|
||||
{
|
||||
baseFileName = baseFileName.Replace(uri.Fragment, "", StringComparison.InvariantCulture);
|
||||
}
|
||||
extension = uri.AbsolutePath.GetExtension();
|
||||
entryName = uri.Fragment;
|
||||
}
|
||||
catch
|
||||
{
|
||||
LeanData.ParseKey(source, out baseFileName, out entryName);
|
||||
extension = Path.GetExtension(baseFileName);
|
||||
}
|
||||
|
||||
var cacheFileName = (useCache ? baseFileName.ToMD5() : Guid.NewGuid().ToString()) + extension;
|
||||
LocalFileName = Path.Combine(downloadDirectory, cacheFileName);
|
||||
|
||||
byte[] bytes = null;
|
||||
if (useCache)
|
||||
{
|
||||
lock (_fileSystemLock)
|
||||
{
|
||||
if (!File.Exists(LocalFileName))
|
||||
{
|
||||
bytes = DownloadBytesWithRetry(source, headers);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
bytes = DownloadBytesWithRetry(source, headers);
|
||||
}
|
||||
|
||||
// Only persist a non-empty download. The remote source can intermittently answer with an empty body
|
||||
// (e.g. a transient HTTP 200 with no content); writing/caching that empty response would make the
|
||||
// whole subscription silently yield no data, and with caching enabled the empty file would be reused.
|
||||
if (bytes != null && bytes.Length > 0)
|
||||
{
|
||||
File.WriteAllBytes(LocalFileName, bytes);
|
||||
|
||||
// Send the file to the dataCacheProvider so it is available when the streamReader asks for it
|
||||
dataCacheProvider.Store(LocalFileName, bytes);
|
||||
}
|
||||
|
||||
// now we can just use the local file reader.
|
||||
// add the entry name to the local file name so the correct entry is read
|
||||
var fileNameWithEntry = LocalFileName;
|
||||
if (!string.IsNullOrEmpty(entryName))
|
||||
{
|
||||
fileNameWithEntry += entryName;
|
||||
}
|
||||
_streamReader = new LocalFileSubscriptionStreamReader(dataCacheProvider, fileNameWithEntry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets <see cref="SubscriptionTransportMedium.RemoteFile"/>
|
||||
/// </summary>
|
||||
public SubscriptionTransportMedium TransportMedium
|
||||
{
|
||||
get { return SubscriptionTransportMedium.RemoteFile; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether or not there's more data to be read in the stream
|
||||
/// </summary>
|
||||
public bool EndOfStream
|
||||
{
|
||||
get { return _streamReader.EndOfStream; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the next line/batch of content from the stream
|
||||
/// </summary>
|
||||
public string ReadLine()
|
||||
{
|
||||
return _streamReader.ReadLine();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes of the stream
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_streamReader.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Save reference to the download system.
|
||||
/// </summary>
|
||||
/// <param name="downloader">Downloader provider for the remote file fetching.</param>
|
||||
public static void SetDownloadProvider(IDownloadProvider downloader)
|
||||
{
|
||||
_downloader = downloader;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads the given source, retrying a few times on a transient failure. The remote endpoint can
|
||||
/// intermittently throw or answer with an empty body; a bounded retry lets a single hiccup recover
|
||||
/// instead of failing the whole subscription with no data.
|
||||
/// </summary>
|
||||
private static byte[] DownloadBytesWithRetry(string source, IEnumerable<KeyValuePair<string, string>> headers)
|
||||
{
|
||||
const int maxAttempts = 3;
|
||||
for (var attempt = 1; attempt <= maxAttempts; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
var bytes = _downloader.DownloadBytes(source, headers, null, null);
|
||||
if (bytes != null && bytes.Length > 0)
|
||||
{
|
||||
return bytes;
|
||||
}
|
||||
|
||||
// a successful but empty response is transient for these sources; retry before giving up
|
||||
Log.Trace($"RemoteFileSubscriptionStreamReader.DownloadBytesWithRetry(): empty response for {source} " +
|
||||
$"(attempt {attempt}/{maxAttempts})");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
if (attempt == maxAttempts)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
Log.Trace($"RemoteFileSubscriptionStreamReader.DownloadBytesWithRetry(): failed to download {source} " +
|
||||
$"(attempt {attempt}/{maxAttempts}): {exception.Message}");
|
||||
}
|
||||
|
||||
if (attempt < maxAttempts)
|
||||
{
|
||||
Thread.Sleep(2000 + 1000 * attempt);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user