/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <param name="lowerThreshold">The lower threshold for the worker task, for which the consumer will trigger the worker
/// if it has stopped <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param>
/// <param name="upperThreshold">The upper threshold for the worker task, after which it will stop producing until requested
/// by the consumer <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator,
int lowerThreshold,
int upperThreshold)
{
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
Action produce = () =>
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.Dispose();
return;
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours, subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if we have more data than the upper threshold in the enqueueable, we don't want to fill the ram
if (count > upperThreshold)
{
// we use local count for the outside if, for performance, and adjust here
count = enqueueable.Count;
if (count > upperThreshold)
{
// we will be re scheduled to run by the consumer, see EnqueueableEnumerator
return;
}
}
}
// we made it here because MoveNext returned false, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.Dispose();
};
enqueueable.SetProducer(produce, lowerThreshold);
return(subscription);
}
private void ScheduleEnumerator(Subscription subscription, IEnumerator <BaseData> enumerator, EnqueueableEnumerator <SubscriptionData> enqueueable,
int lowerThreshold, int upperThreshold, SecurityExchangeHours exchangeHours, int firstLoopCount = 5)
{
Action produce = () =>
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.Dispose();
return;
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours, subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if we have more data than the upper threshold in the enqueueable
if (count > upperThreshold)
{
// we use local count for the outside if, for performance, and adjust here
count = enqueueable.Count;
if (count > upperThreshold)
{
return;
}
}
}
// we made it here because MoveNext returned false, stop the enqueueable
enqueueable.Stop();
};
enqueueable.SetProducer(produce, lowerThreshold);
}
/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <param name="firstLoopLimit">The first loop data point count for which the worker will stop</param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator,
int firstLoopLimit = 50)
{
var upperThreshold = GetUpperThreshold(request.Configuration.Resolution);
var lowerThreshold = GetLowerThreshold(request.Configuration.Resolution);
if (request.Configuration.Type == typeof(CoarseFundamental))
{
// the lower threshold will be when we start the worker again, if he is stopped
lowerThreshold = 200;
// the upper threshold will stop the worker from loading more data. This is roughly 1 GB
upperThreshold = 500;
}
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
// The first loop of a backtest can load hundreds of subscription feeds, resulting in long delays while the thresholds
// for the buffer are reached. For the first loop start up with just 50 items in the buffer.
var firstLoop = Ref.Create(true);
Action produce = () =>
{
try
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.DisposeSafely();
return;
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours,
subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if we have more data than the upper threshold in the enqueueable, we don't want to fill the ram
if (count > upperThreshold || count > firstLoopLimit && firstLoop.Value)
{
// we use local count for the outside if, for performance, and adjust here
count = enqueueable.Count;
if (count > upperThreshold || firstLoop.Value)
{
firstLoop.Value = false;
// we will be re scheduled to run by the consumer, see EnqueueableEnumerator
// if the consumer is already waiting for us wake him up, he will rescheduled us if required
enqueueable.CancellationTokenSource.Cancel();
return;
}
}
}
}
catch (Exception exception)
{
Log.Error(exception, $"Subscription worker task exception {request.Configuration}.");
}
// we made it here because MoveNext returned false or we exploded, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.DisposeSafely();
};
enqueueable.SetProducer(produce, lowerThreshold);
return(subscription);
}