wehub-resource-sync
124 lines
4.5 KiB
C#
124 lines
4.5 KiB
C#
/*
|
|
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
|
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using QuantConnect.Logging;
|
|
|
|
namespace QuantConnect.ToolBox
|
|
{
|
|
/// <summary>
|
|
/// Processing harness used to read files in, parse them, and process them.
|
|
/// </summary>
|
|
public class RawFileProcessor : IDisposable
|
|
{
|
|
private DateTime? _start;
|
|
private readonly IStreamProvider _streamProvider;
|
|
private readonly IStreamParser _parser;
|
|
private readonly IDataProcessor[] _processors;
|
|
|
|
/// <summary>
|
|
/// Gets or sets a name used for logging
|
|
/// </summary>
|
|
public string Name { get; set; }
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of the <see cref="RawFileProcessor"/> class
|
|
/// </summary>
|
|
public RawFileProcessor(IStreamProvider streamProvider, IStreamParser parser, params IDataProcessor[] processors)
|
|
{
|
|
_streamProvider = streamProvider;
|
|
_parser = parser;
|
|
_processors = processors;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Runs the raw file processor on the specified files
|
|
/// </summary>
|
|
/// <param name="name">A name for the processor used for logging</param>
|
|
/// <param name="sources">The raw files to be processed</param>
|
|
/// <param name="streamProvider">Instance capable of reading the sources into a stream</param>
|
|
/// <param name="streamParser">Instance capable of parsing the provided stream</param>
|
|
/// <param name="processors">The data processors to process the parsed data</param>
|
|
/// <returns>True if the operation completed without error, otherwise false</returns>
|
|
public static bool Run(string name, IEnumerable<string> sources, IStreamProvider streamProvider, IStreamParser streamParser, params IDataProcessor[] processors)
|
|
{
|
|
using (var processor = new RawFileProcessor(streamProvider, streamParser, processors) { Name = name })
|
|
{
|
|
foreach (var zip in sources)
|
|
{
|
|
try
|
|
{
|
|
processor.Process(zip);
|
|
}
|
|
catch (Exception err)
|
|
{
|
|
Log.Error(err);
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Perform processing on the specified source file
|
|
/// </summary>
|
|
/// <param name="source">The source file to be processed</param>
|
|
public void Process(string source)
|
|
{
|
|
_start = _start ?? DateTime.UtcNow;
|
|
|
|
// process the source file
|
|
foreach (var stream in _streamProvider.Open(source))
|
|
{
|
|
using (stream)
|
|
{
|
|
foreach (var data in _parser.Parse(source, stream))
|
|
{
|
|
foreach (var processor in _processors)
|
|
{
|
|
processor.Process(data);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Log.Trace("RawFileProcessor.Process({0}): Finished.", source);
|
|
_streamProvider.Close(source);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
|
|
/// </summary>
|
|
public void Dispose()
|
|
{
|
|
_streamProvider.Dispose();
|
|
_parser.Dispose();
|
|
foreach (var processor in _processors)
|
|
{
|
|
processor.Dispose();
|
|
}
|
|
|
|
if (_start.HasValue)
|
|
{
|
|
var stop = DateTime.UtcNow;
|
|
Log.Trace("RawFileProcessor.Dispose({0}): Elapsed {1}", Name, stop - _start);
|
|
}
|
|
}
|
|
}
|
|
} |