private static IEnumerator <BaseData> GetNextTicksEnumerator(IDataQueueHandler dataQueueHandler)
{
while (true)
{
int ticks = 0;
foreach (var data in dataQueueHandler.GetNextTicks())
{
ticks++;
yield return(data);
}
if (ticks == 0)
{
Thread.Sleep(1);
}
}
}
/// <summary>
/// Provides an <see cref="IEnumerator{BaseData}"/> that will continually dequeue data
/// from the data queue handler while we're not cancelled
/// </summary>
/// <returns></returns>
private IEnumerator <BaseData> GetNextTicksEnumerator()
{
while (!_cancellationTokenSource.IsCancellationRequested)
{
int ticks = 0;
foreach (var data in _dataQueueHandler.GetNextTicks())
{
ticks++;
yield return(data);
}
if (ticks == 0)
{
Thread.Sleep(1);
}
}
}
/// <summary>
/// Stream Store Consumer uses the GetNextTicks() function to get current ticks from a data source and
/// then uses the stream store to compile them into trade bars.
/// </summary>
public void StreamStoreConsumer()
{
//Initialize
var update = new Dictionary <int, DateTime>();
//Scan for the required time period to stream:
Log.Trace("LiveTradingDataFeed.Stream(): Waiting for updated market hours...", true);
//Awake:
Log.Trace("LiveTradingDataFeed.Stream(): Market open, starting stream for " + string.Join(",", _symbols));
//Micro-thread for polling for new data from data source:
var liveThreadTask = new Task(() =>
{
if (_isDynamicallyLoadedData.All(x => x))
{
// if we're all custom data data don't waste CPU cycle with this thread
return;
}
//Blocking ForEach - Should stay within this loop as long as there is a data-connection
while (true)
{
var dataCollection = _dataQueue.GetNextTicks();
int ticksCount = 0;
foreach (var point in dataCollection)
{
ticksCount++;
//Get the stream store with this symbol:
for (var i = 0; i < Subscriptions.Count; i++)
{
if (_subscriptions[i].Symbol != point.Symbol)
{
continue;
}
var tick = point as Tick;
if (tick != null)
{
if (_subscriptions[i].Resolution == Resolution.Tick)
{
// put ticks directly into the bridge
AddSingleItemToBridge(tick, i);
}
else
{
// Update our internal counter
_streamStores[i].Update(tick);
// Update the realtime price stream value
_realtimePrices[i] = point.Value;
}
}
else
{
// reset the start time so it goes in sync with other data
point.Time = DateTime.Now.RoundDown(_subscriptions[i].Increment);
//If its not a tick, inject directly into bridge for this symbol:
//Bridge[i].Enqueue(new List<BaseData> {point});
AddSingleItemToBridge(point, i);
}
}
}
if (_exitTriggered)
{
return;
}
if (ticksCount == 0)
{
Thread.Sleep(5);
}
}
});
// Micro-thread for custom data/feeds. This only supports polling at this time. todo: Custom data sockets
var customFeedsTask = new Task(() =>
{
// used to help prevent future data from entering the algorithm
// initial to all true, when we get future data, flip flag to false to prevent move next
var needsMoveNext = Enumerable.Range(0, Subscriptions.Count).Select(x => true).ToArray();
while (true)
{
for (var i = 0; i < Subscriptions.Count; i++)
{
if (_isDynamicallyLoadedData[i])
{
if (!update.ContainsKey(i))
{
update.Add(i, new DateTime());
}
if (DateTime.Now > update[i])
{
//Now Time has passed -> Trigger a refresh,
if (needsMoveNext[i])
{
// if we didn't emit the previous value it's because it was in
// the future, so don't call MoveNext, just perform the date range
// checks below again
// in live mode subscription reader will move next but return null for current since nothing is there
_subscriptionManagers[i].MoveNext();
// true success defined as if we got a non-null value
if (_subscriptionManagers[i].Current == null)
{
// we failed to get new data for this guy, try again next second
update[i] = DateTime.Now.Add(Time.OneSecond);
continue;
}
}
// if we didn't get anything keep going
var data = _subscriptionManagers[i].Current;
if (data == null)
{
// heuristically speaking this should already be true, but no harm in explicitly setting it
needsMoveNext[i] = true;
continue;
}
// check to see if the data is too far in the past
// this is useful when using custom remote files that may stretch far into the past,
// so this if block will cause us to fast forward the reader until its recent increment
if (data.EndTime < DateTime.Now.Subtract(_subscriptions[i].Increment.Add(Time.OneSecond)))
{
// repeat this subscription, we're in the past still
i--;
continue;
}
// don't emit data in the future
if (data.EndTime < DateTime.Now)
{
if (_subscriptions[i].Resolution == Resolution.Tick)
{
// put ticks directly into the bridge
AddSingleItemToBridge(data, i);
}
else
{
_streamStores[i].Update(data); //Update bar builder.
_realtimePrices[i] = data.Value; //Update realtime price value.
needsMoveNext[i] = true;
}
}
else
{
// since this data is in the future and we didn't emit it,
// don't call MoveNext again and we'll keep performing time checks
// until its end time has passed and we can emit it into the bridge
needsMoveNext[i] = false;
}
// REVIEW:: We may want to set update to 'just before' the time, I'm thinking of daily data, so we
// ask for it at midnight, let's assume it's there, did we get it into the stream store
// before we called trigger archive?? I hope so, otherwise the data is always a full day late
update[i] = DateTime.Now.Add(_subscriptions[i].Increment).RoundDown(_subscriptions[i].Increment);
}
}
}
if (_exitTriggered)
{
return;
}
Thread.Sleep(10);
}
});
//Wait for micro-threads to break before continuing
liveThreadTask.Start();
// define what tasks we're going to wait on, we use a task from result in place of the custom task, just in case we never start it
var tasks = new [] { liveThreadTask, Task.FromResult(1) };
// if we have any dynamically loaded data, start the custom thread
if (_isDynamicallyLoadedData.Any(x => x))
{
//Start task and set it as the second one we want to monitor:
customFeedsTask.Start();
tasks[1] = customFeedsTask;
}
Task.WaitAll(tasks);
//Once we're here the tasks have died, signal
if (!_exitTriggered)
{
_endOfBridges = true;
}
Log.Trace(string.Format("LiveTradingDataFeed.Stream(): Stream Task Completed. Exit Signal: {0}", _exitTriggered));
}
/// <summary>
/// Stream Store Consumer uses the GetNextTicks() function to get current ticks from a data source and
/// then uses the stream store to compile them into trade bars.
/// </summary>
public void StreamStoreConsumer()
{
//Scan for the required time period to stream:
Log.Trace("LiveTradingDataFeed.Stream(): Waiting for updated market hours...", true);
var symbols = (from security in _algorithm.Securities.Values
where !security.SubscriptionDataConfig.IsCustomData && (security.Type == SecurityType.Equity || security.Type == SecurityType.Forex)
select security.Symbol.Permtick).ToList <string>();
Log.Trace("LiveTradingDataFeed.Stream(): Market open, starting stream for " + string.Join(",", symbols));
//Micro-thread for polling for new data from data source:
var liveThreadTask = new Task(() =>
{
if (_subscriptions.All(x => x.Value.IsCustomData))
{
// if we're all custom data data don't waste CPU cycle with this thread
return;
}
//Blocking ForEach - Should stay within this loop as long as there is a data-connection
while (true)
{
var dataCollection = _dataQueue.GetNextTicks();
int ticksCount = 0;
foreach (var point in dataCollection)
{
ticksCount++;
foreach (var kvp in _subscriptions)
{
var subscription = kvp.Value;
if (subscription.Configuration.Symbol != point.Symbol)
{
continue;
}
var tick = point as Tick;
if (tick != null)
{
// Update the realtime price stream value
subscription.SetRealtimePrice(point.Value);
if (subscription.Configuration.Resolution == Resolution.Tick)
{
// put ticks directly into the bridge
AddSingleItemToBridge(subscription, tick);
}
else
{
// Update our internal counter
subscription.StreamStore.Update(tick);
}
}
else
{
// reset the start time so it goes in sync with other data
point.Time = DateTime.UtcNow.ConvertFromUtc(subscription.TimeZone).RoundDown(subscription.Configuration.Increment);
//If its not a tick, inject directly into bridge for this symbol:
//Bridge[i].Enqueue(new List<BaseData> {point});
AddSingleItemToBridge(subscription, point);
}
}
}
if (_cancellationTokenSource.IsCancellationRequested)
{
return;
}
if (ticksCount == 0)
{
Thread.Sleep(5);
}
}
}, TaskCreationOptions.LongRunning);
// Micro-thread for custom data/feeds. This only supports polling at this time. todo: Custom data sockets
var customFeedsTask = new Task(() =>
{
while (true)
{
foreach (var kvp in _subscriptions)
{
var subscription = kvp.Value;
// custom only thread
if (!subscription.IsCustomData)
{
continue;
}
// wait for when it's time to update
if (!subscription.NeedsUpdate)
{
continue;
}
var repeat = true;
BaseData data = null;
while (repeat && TryMoveNext(subscription, out data))
{
if (data == null)
{
break;
}
// check to see if the data is too far in the past
// this is useful when using custom remote files that may stretch far into the past,
// so this if block will cause us to fast forward the reader until its recent increment
var earliestExpectedFirstPoint = DateTime.UtcNow.Subtract(subscription.Configuration.Increment.Add(Time.OneSecond));
repeat = data.EndTime.ConvertToUtc(subscription.TimeZone) < earliestExpectedFirstPoint;
}
if (data == null)
{
continue;
}
// don't emit data in the future
// TODO : Move this concern into LiveSubscription, maybe a CustomLiveSubscription, end goal just enumerate the damn thing at it works
if (data.EndTime.ConvertToUtc(subscription.TimeZone) < DateTime.UtcNow)
{
if (subscription.Configuration.Resolution == Resolution.Tick)
{
// put ticks directly into the bridge
AddSingleItemToBridge(subscription, data);
}
else
{
Log.Trace("LiveTradingDataFeed.Custom(): Add to stream store.");
subscription.StreamStore.Update(data); //Update bar builder.
subscription.SetRealtimePrice(data.Value); //Update realtime price value.
subscription.NeedsMoveNext = true;
}
}
else
{
// since this data is in the future and we didn't emit it,
// don't call MoveNext again and we'll keep performing time checks
// until its end time has passed and we can emit it into the bridge
subscription.NeedsMoveNext = false;
}
}
if (_cancellationTokenSource.IsCancellationRequested)
{
return;
}
Thread.Sleep(10);
}
}, TaskCreationOptions.LongRunning);
//Wait for micro-threads to break before continuing
liveThreadTask.Start();
// define what tasks we're going to wait on, we use a task from result in place of the custom task, just in case we never start it
var tasks = new [] { liveThreadTask, Task.FromResult(1) };
// if we have any dynamically loaded data, start the custom thread
if (_subscriptions.Any(x => x.Value.IsCustomData))
{
//Start task and set it as the second one we want to monitor:
customFeedsTask.Start();
tasks[1] = customFeedsTask;
}
Task.WaitAll(tasks);
//Once we're here the tasks have died, signal
if (!_cancellationTokenSource.IsCancellationRequested)
{
_endOfBridges = true;
}
Log.Trace(string.Format("LiveTradingDataFeed.Stream(): Stream Task Completed. Exit Signal: {0}", _cancellationTokenSource.IsCancellationRequested));
}
private static IEnumerator<BaseData> GetNextTicksEnumerator(IDataQueueHandler dataQueueHandler)
{
while (true)
{
int ticks = 0;
foreach (var data in dataQueueHandler.GetNextTicks())
{
ticks++;
yield return data;
}
if (ticks == 0) Thread.Sleep(1);
}
}
/// <summary>
/// Gets the next ticks from the live trading feed
/// </summary>
/// <returns>The next ticks to be processed</returns>
public override IEnumerable <Tick> GetNextTicks()
{
return(_queue.GetNextTicks());
}
/// <summary>
/// Returns the next ticks from the data source. The data source itself is defined in the derived class's
/// implementation of this function. For example, if obtaining data from a brokerage, say IB, then the derived
/// implementation would ask the IB API for the next ticks
/// </summary>
/// <returns>The next ticks to be aggregated and sent to algoithm</returns>
public virtual IEnumerable <BaseData> GetNextTicks()
{
return(_dataQueue.GetNextTicks());
}
