wehub/quantconnect--lean
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:02:50 +08:00
commit 0fc60fdcb1
5008 changed files with 910633 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Engine/DataFeeds/WorkScheduling/BaseWorkScheduler.cs
+40
View File
@@ -0,0 +1,40 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
namespace QuantConnect.Lean.Engine.DataFeeds.WorkScheduling
{
/// <summary>
/// Base work scheduler abstraction
/// </summary>
public abstract class WorkScheduler
{
/// <summary>
/// The quantity of workers to be used
/// </summary>
public static int WorkersCount = Configuration.Config.GetInt("data-feed-workers-count", Environment.ProcessorCount);
/// <summary>
/// Add a new work item to the queue
/// </summary>
/// <param name="symbol">The symbol associated with this work</param>
/// <param name="workFunc">The work function to run</param>
/// <param name="weightFunc">The weight function.
/// Work will be sorted in ascending order based on this weight</param>
public abstract void QueueWork(Symbol symbol, Func<int, bool> workFunc, Func<int> weightFunc);
}
}
Engine/DataFeeds/WorkScheduling/WeightedWorkQueue.cs
+185
View File
@@ -0,0 +1,185 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.Threading;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Runtime.CompilerServices;
namespace QuantConnect.Lean.Engine.DataFeeds.WorkScheduling
{
internal class WeightedWorkQueue
{
private int _pointer;
private bool _removed;
private Action _singleCallWork;
private readonly List<WorkItem> _workQueue;
/// <summary>
/// Event used to notify there is work ready to execute in this queue
/// </summary>
private AutoResetEvent _workAvailableEvent;
/// <summary>
/// Returns the thread priority to use for this work queue
/// </summary>
public ThreadPriority ThreadPriority => ThreadPriority.Lowest;
/// <summary>
/// Creates a new instance
/// </summary>
public WeightedWorkQueue()
{
_workQueue = new List<WorkItem>();
_workAvailableEvent = new AutoResetEvent(false);
}
/// <summary>
/// This is the worker thread loop.
/// It will first try to take a work item from the new work queue else will check his own queue.
/// </summary>
public void WorkerThread(ConcurrentQueue<WorkItem> newWork, AutoResetEvent newWorkEvent)
{
var waitHandles = new WaitHandle[] { _workAvailableEvent, newWorkEvent };
var waitedPreviousLoop = 0;
while (true)
{
WorkItem workItem;
if (!newWork.TryDequeue(out workItem))
{
workItem = Get();
if (workItem == null)
{
if(_singleCallWork != null)
{
try
{
_singleCallWork();
}
catch (Exception exception)
{
// this shouldn't happen but just in case
Logging.Log.Error(exception);
}
// we execute this once only and clear it's reference
_singleCallWork = null;
}
// no work to do, lets sleep and try again
WaitHandle.WaitAny(waitHandles, Math.Min(1 + (waitedPreviousLoop * 10), 250));
waitedPreviousLoop++;
continue;
}
}
else
{
Add(workItem);
}
try
{
waitedPreviousLoop = 0;
if (!workItem.Work(WeightedWorkScheduler.WorkBatchSize))
{
Remove(workItem);
}
}
catch (Exception exception)
{
Remove(workItem);
Logging.Log.Error(exception);
}
}
}
/// <summary>
/// Adds a new item to this work queue
/// </summary>
/// <param name="work">The work to add</param>
private void Add(WorkItem work)
{
_workQueue.Add(work);
}
/// <summary>
/// Adds a new item to this work queue
/// </summary>
/// <param name="work">The work to add</param>
public void AddSingleCall(Action work)
{
_singleCallWork = work;
_workAvailableEvent.Set();
}
/// <summary>
/// Removes an item from the work queue
/// </summary>
/// <param name="workItem">The work item to remove</param>
private void Remove(WorkItem workItem)
{
_workQueue.Remove(workItem);
_removed = true;
}
/// <summary>
/// Gets the next work item to process
/// </summary>
/// <returns>The work item to process, null if none available</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected WorkItem Get()
{
var count = _workQueue.Count;
if (count == 0)
{
return null;
}
var countFactor = (10 + 10 / count) / 10;
if (_removed)
{
// if we removed an item don't really trust the pointer any more
_removed = false;
_pointer = Math.Min(_pointer, count - 1);
}
var initial = _pointer;
do
{
var item = _workQueue[_pointer++];
if (_pointer >= count)
{
_pointer = 0;
// this will only really make a difference if there are many work items
if (25 > count)
{
// if we looped around let's sort the queue leave the jobs with less points at the start
_workQueue.Sort(WorkItem.Compare);
}
}
if (item.UpdateWeight() < WeightedWorkScheduler.MaxWorkWeight * countFactor)
{
return item;
}
} while (initial != _pointer);
// no work item is ready, pointer still will keep it's same value
return null;
}
}
}
Engine/DataFeeds/WorkScheduling/WeightedWorkScheduler.cs
+114
View File
@@ -0,0 +1,114 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace QuantConnect.Lean.Engine.DataFeeds.WorkScheduling
{
/// <summary>
/// This singleton class will create a thread pool to processes work
/// that will be prioritized based on it's weight
/// </summary>
/// <remarks>The threads in the pool will take ownership of the
/// <see cref="WorkItem"/> and not share it with another thread.
/// This is required because the data enumerator stack yields, which state
/// depends on the thread id</remarks>
public class WeightedWorkScheduler : WorkScheduler
{
private static readonly Lazy<WeightedWorkScheduler> _instance = new Lazy<WeightedWorkScheduler>(() => new WeightedWorkScheduler());
/// <summary>
/// This is the size of each work sprint
/// </summary>
public const int WorkBatchSize = 50;
/// <summary>
/// This is the maximum size a work item can weigh,
/// if reached, it will be ignored and not executed until its less
/// </summary>
/// <remarks>This is useful to limit RAM and CPU usage</remarks>
public static int MaxWorkWeight;
private readonly ConcurrentQueue<WorkItem> _newWork;
private readonly AutoResetEvent _newWorkEvent;
private Task _initializationTask;
private readonly List<WeightedWorkQueue> _workerQueues;
/// <summary>
/// Singleton instance
/// </summary>
public static WeightedWorkScheduler Instance => _instance.Value;
private WeightedWorkScheduler()
{
_newWork = new ConcurrentQueue<WorkItem>();
_newWorkEvent = new AutoResetEvent(false);
_workerQueues = new List<WeightedWorkQueue>(WorkersCount);
_initializationTask = Task.Run(() =>
{
MaxWorkWeight = Configuration.Config.GetInt("data-feed-max-work-weight", 400);
Logging.Log.Trace($"WeightedWorkScheduler(): will use {WorkersCount} workers and MaxWorkWeight is {MaxWorkWeight}");
for (var i = 0; i < WorkersCount; i++)
{
var workQueue = new WeightedWorkQueue();
_workerQueues.Add(workQueue);
var thread = new Thread(() => workQueue.WorkerThread(_newWork, _newWorkEvent))
{
IsBackground = true,
Priority = workQueue.ThreadPriority,
Name = $"WeightedWorkThread{i}"
};
thread.Start();
}
});
}
/// <summary>
/// Add a new work item to the queue
/// </summary>
/// <param name="symbol">The symbol associated with this work</param>
/// <param name="workFunc">The work function to run</param>
/// <param name="weightFunc">The weight function.
/// Work will be sorted in ascending order based on this weight</param>
public override void QueueWork(Symbol symbol, Func<int, bool> workFunc, Func<int> weightFunc)
{
_newWork.Enqueue(new WorkItem(workFunc, weightFunc));
_newWorkEvent.Set();
}
/// <summary>
/// Execute the given action in all workers once
/// </summary>
public void AddSingleCallForAll(Action action)
{
if (!_initializationTask.Wait(TimeSpan.FromSeconds(10)))
{
throw new TimeoutException("Timeout waiting for worker threads to start");
}
for (var i = 0; i < _workerQueues.Count; i++)
{
_workerQueues[i].AddSingleCall(action);
}
}
}
}
Engine/DataFeeds/WorkScheduling/WorkItem.cs
+86
View File
@@ -0,0 +1,86 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.Runtime.CompilerServices;
namespace QuantConnect.Lean.Engine.DataFeeds.WorkScheduling
{
/// <summary>
/// Class to represent a work item
/// </summary>
public class WorkItem
{
/// <summary>
/// Function to determine weight of item
/// </summary>
private Func<int> _weightFunc;
/// <summary>
/// The current weight
/// </summary>
public int Weight { get; private set; }
/// <summary>
/// The work function to execute
/// </summary>
public Func<int, bool> Work { get; }
/// <summary>
/// Creates a new instance
/// </summary>
/// <param name="work">The work function, takes an int, the amount of work to do
/// and returns a bool, false if this work item is finished</param>
/// <param name="weightFunc">The function used to determine the current weight</param>
public WorkItem(Func<int, bool> work, Func<int> weightFunc)
{
Work = work;
_weightFunc = weightFunc;
}
/// <summary>
/// Updates the weight of this work item
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int UpdateWeight()
{
Weight = _weightFunc();
return Weight;
}
/// <summary>
/// Compares two work items based on their weights
/// </summary>
public static int Compare(WorkItem obj, WorkItem other)
{
if (ReferenceEquals(obj, other))
{
return 0;
}
// By definition, any object compares greater than null
if (ReferenceEquals(obj, null))
{
return -1;
}
if (ReferenceEquals(null, other))
{
return 1;
}
return other.Weight.CompareTo(obj.Weight);
}
}
}