public void PassesTicksStraightThrough()
{
var enumerator = new EnqueueableEnumerator<Tick>();
// add some ticks
var currentTime = new DateTime(2015, 10, 08);
// returns true even if no data present until stop is called
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) {Quantity = 10};
enumerator.Enqueue(tick1);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(tick1, enumerator.Current);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var tick2 = new Tick(currentTime, Symbols.SPY, 199.56m, 199.21m, 200.02m) {Quantity = 5};
enumerator.Enqueue(tick2);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(tick2, enumerator.Current);
enumerator.Stop();
Assert.IsFalse(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
}
public void PassesTicksStraightThrough()
{
var enumerator = new EnqueueableEnumerator<Tick>();
// add some ticks
var currentTime = new DateTime(2015, 10, 08);
// returns true even if no data present until stop is called
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) {Quantity = 10};
enumerator.Enqueue(tick1);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(tick1, enumerator.Current);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var tick2 = new Tick(currentTime, Symbols.SPY, 199.56m, 199.21m, 200.02m) {Quantity = 5};
enumerator.Enqueue(tick2);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(tick2, enumerator.Current);
enumerator.Stop();
Assert.IsFalse(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
}
public void RecordsInternalQueueCount()
{
var enumerator = new EnqueueableEnumerator<Tick>();
var currentTime = new DateTime(2015, 12, 01);
var tick = new Tick(currentTime, Symbols.SPY, 100, 101);
enumerator.Enqueue(tick);
Assert.AreEqual(1, enumerator.Count);
tick = new Tick(currentTime, Symbols.SPY, 100, 101);
enumerator.Enqueue(tick);
Assert.AreEqual(2, enumerator.Count);
enumerator.MoveNext();
Assert.AreEqual(1, enumerator.Count);
enumerator.MoveNext();
Assert.AreEqual(0, enumerator.Count);
}
public void RecordsInternalQueueCount()
{
var enumerator = new EnqueueableEnumerator<Tick>();
var currentTime = new DateTime(2015, 12, 01);
var tick = new Tick(currentTime, Symbols.SPY, 100, 101);
enumerator.Enqueue(tick);
Assert.AreEqual(1, enumerator.Count);
tick = new Tick(currentTime, Symbols.SPY, 100, 101);
enumerator.Enqueue(tick);
Assert.AreEqual(2, enumerator.Count);
enumerator.MoveNext();
Assert.AreEqual(1, enumerator.Count);
enumerator.MoveNext();
Assert.AreEqual(0, enumerator.Count);
}
/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator)
{
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
Func <int, bool> produce = (workBatchSize) =>
{
try
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.DisposeSafely();
return(false);
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours,
subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if added more data than the work batch size, we don't want to fill the ram
if (count > workBatchSize)
{
return(true);
}
}
}
catch (Exception exception)
{
Log.Error(exception, $"Subscription worker task exception {request.Configuration}.");
}
// we made it here because MoveNext returned false or we exploded, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.DisposeSafely();
return(false);
};
WeightedWorkScheduler.Instance.QueueWork(produce,
// if the subscription finished we return 0, so the work is prioritized and gets removed
() => enqueueable.HasFinished ? 0 : enqueueable.Count);
return(subscription);
}
public void RecordsMostRecentlyEnqueuedItem()
{
var enumerator = new EnqueueableEnumerator<Tick>();
var currentTime = new DateTime(2015, 12, 01);
var tick1 = new Tick(currentTime, Symbols.SPY, 100, 101);
enumerator.Enqueue(tick1);
Assert.AreEqual(null, enumerator.Current);
Assert.AreEqual(tick1, enumerator.LastEnqueued);
var tick2 = new Tick(currentTime, Symbols.SPY, 100, 101);
enumerator.Enqueue(tick2);
Assert.AreEqual(tick2, enumerator.LastEnqueued);
enumerator.MoveNext();
Assert.AreEqual(tick1, enumerator.Current);
enumerator.MoveNext();
Assert.AreEqual(tick2, enumerator.Current);
}
private void ScheduleEnumerator(Subscription subscription, IEnumerator <BaseData> enumerator, EnqueueableEnumerator <SubscriptionData> enqueueable,
int lowerThreshold, int upperThreshold, int firstLoopCount = 5)
{
// schedule the work on the controller
var security = subscription.Security;
var configuration = subscription.Configuration;
var firstLoop = true;
FuncParallelRunnerWorkItem workItem = null;
workItem = new FuncParallelRunnerWorkItem(() => enqueueable.Count < lowerThreshold, () =>
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.Dispose();
return;
}
var subscriptionData = SubscriptionData.Create(configuration, security.Exchange.Hours, subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// special behavior for first loop to spool up quickly
if (firstLoop && count > firstLoopCount)
{
// there's more data in the enumerator, reschedule to run again
firstLoop = false;
_controller.Schedule(workItem);
return;
}
// stop executing if we've dequeued more than the lower threshold or have
// more total that upper threshold in the enqueueable's queue
if (count > lowerThreshold || enqueueable.Count > upperThreshold)
{
// there's more data in the enumerator, reschedule to run again
_controller.Schedule(workItem);
return;
}
}
// we made it here because MoveNext returned false, stop the enqueueable and don't reschedule
enqueueable.Stop();
});
_controller.Schedule(workItem);
}
/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <param name="lowerThreshold">The lower threshold for the worker task, for which the consumer will trigger the worker
/// if it has stopped <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param>
/// <param name="upperThreshold">The upper threshold for the worker task, after which it will stop producing until requested
/// by the consumer <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator,
int lowerThreshold,
int upperThreshold)
{
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
Action produce = () =>
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.Dispose();
return;
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours, subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if we have more data than the upper threshold in the enqueueable, we don't want to fill the ram
if (count > upperThreshold)
{
// we use local count for the outside if, for performance, and adjust here
count = enqueueable.Count;
if (count > upperThreshold)
{
// we will be re scheduled to run by the consumer, see EnqueueableEnumerator
return;
}
}
}
// we made it here because MoveNext returned false, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.Dispose();
};
enqueueable.SetProducer(produce, lowerThreshold);
return(subscription);
}
public void MoveNextBlocks()
{
var finished = new ManualResetEvent(false);
var enumerator = new EnqueueableEnumerator <Tick>(true);
// producer
int count = 0;
Task.Run(() =>
{
while (!finished.WaitOne(TimeSpan.FromMilliseconds(50)))
{
enumerator.Enqueue(new Tick(DateTime.Now, Symbols.SPY, 100, 101));
count++;
// 5 data points is plenty
if (count > 5)
{
finished.Set();
enumerator.Stop();
}
}
});
// consumer
int dequeuedCount = 0;
bool encounteredError = false;
var consumerTaskFinished = new ManualResetEvent(false);
Task.Run(() =>
{
while (enumerator.MoveNext())
{
dequeuedCount++;
if (enumerator.Current == null)
{
encounteredError = true;
}
}
consumerTaskFinished.Set();
});
finished.WaitOne(Timeout.Infinite);
consumerTaskFinished.WaitOne(Timeout.Infinite);
Assert.IsFalse(enumerator.MoveNext());
Assert.IsFalse(encounteredError);
Assert.AreEqual(count, dequeuedCount);
enumerator.Dispose();
}
public void SetErrorHandlerExitsOnTrueReturn()
{
var enqueable = new EnqueueableEnumerator <BaseData>();
var exchange = new BaseDataExchange("test");
var cancellationToken = new CancellationTokenSource();
Task.Factory.StartNew(() =>
{
while (!cancellationToken.IsCancellationRequested)
{
Thread.Sleep(10);
enqueable.Enqueue(new Tick {
Symbol = Symbols.SPY, Time = DateTime.UtcNow
});
}
});
var first = true;
var errorCaught = new AutoResetEvent(true);
BaseData last = null;
exchange.SetErrorHandler(error =>
{
errorCaught.Set();
return(true);
});
exchange.AddEnumerator(Symbols.SPY, enqueable, handleData: spy =>
{
if (first)
{
first = false;
throw new Exception();
}
last = spy;
});
Task.Run(() => exchange.Start());
Assert.IsTrue(errorCaught.WaitOne(DefaultTimeout));
exchange.Stop();
Assert.IsNull(last);
enqueable.Dispose();
cancellationToken.Cancel();
}
public void EndsQueueConsumption()
{
var enqueable = new EnqueueableEnumerator <BaseData>();
var exchange = new BaseDataExchange("test");
var cancellationToken = new CancellationTokenSource();
Task.Run(() =>
{
while (!cancellationToken.IsCancellationRequested)
{
Thread.Sleep(10);
enqueable.Enqueue(new Tick {
Symbol = Symbols.SPY, Time = DateTime.UtcNow
});
}
});
BaseData last = null;
var lastUpdated = new AutoResetEvent(false);
exchange.AddEnumerator(Symbols.SPY, enqueable, handleData: spy =>
{
last = spy;
lastUpdated.Set();
});
var finishedRunning = new AutoResetEvent(false);
Task.Run(() => { exchange.Start(); finishedRunning.Set(); });
Assert.IsTrue(lastUpdated.WaitOne(DefaultTimeout));
exchange.Stop();
cancellationToken.Cancel();
Assert.IsTrue(finishedRunning.WaitOne(DefaultTimeout));
var endTime = DateTime.UtcNow;
Assert.IsNotNull(last);
Assert.IsTrue(last.Time <= endTime);
enqueable.Dispose();
}
public void Step1D(decimal min, decimal max, decimal step)
{
var param = new OptimizationStepParameter("ema-fast", min, max, step);
var set = new HashSet <OptimizationParameter>()
{
param
};
_strategy.Initialize(new Target("Profit", new Maximization(), null), new List <Constraint>(), set, new StepBaseOptimizationStrategySettings());
var counter = 0;
using (var enumerator = new EnqueueableEnumerator <ParameterSet>())
{
_strategy.NewParameterSet += (s, parameterSet) =>
{
enumerator.Enqueue(parameterSet);
};
_strategy.PushNewResults(OptimizationResult.Initial);
using (var paramEnumerator = new OptimizationStepParameterEnumerator(param))
{
while (paramEnumerator.MoveNext())
{
var value = paramEnumerator.Current;
counter++;
Assert.IsTrue(enumerator.MoveNext());
var suggestion = enumerator.Current;
Assert.IsNotNull(suggestion);
Assert.IsTrue(suggestion.Value.All(s => set.Any(arg => arg.Name == s.Key)));
Assert.AreEqual(1, suggestion.Value.Count);
Assert.AreEqual(value, suggestion.Value["ema-fast"]);
}
}
Assert.AreEqual(0, enumerator.Count);
}
Assert.Greater(counter, 0);
Assert.AreEqual(Math.Floor((param.MaxValue - param.MinValue) / param.Step.Value) + 1, counter);
}
public void DefaultErrorHandlerDoesNotStopQueueConsumption()
{
var enqueable = new EnqueueableEnumerator <BaseData>();
var exchange = new BaseDataExchange("test");
var cancellationToken = new CancellationTokenSource();
Task.Run(() =>
{
while (!cancellationToken.IsCancellationRequested)
{
Thread.Sleep(10);
enqueable.Enqueue(new Tick {
Symbol = Symbols.SPY, Time = DateTime.UtcNow
});
}
});
var first = true;
BaseData last = null;
var lastUpdated = new AutoResetEvent(false);
exchange.AddEnumerator(Symbols.SPY, enqueable, handleData: spy =>
{
if (first)
{
first = false;
throw new Exception("This exception should be swalloed by the exchange!");
}
last = spy;
lastUpdated.Set();
});
Task.Run(() => exchange.Start());
Assert.IsTrue(lastUpdated.WaitOne(DefaultTimeout));
exchange.Stop();
cancellationToken.Cancel();
enqueable.Dispose();
}
private void ScheduleEnumerator(IEnumerator <BaseData> enumerator, EnqueueableEnumerator <BaseData> enqueueable, int lowerThreshold, int upperThreshold, int firstLoopCount = 5)
{
// schedule the work on the controller
var firstLoop = true;
FuncParallelRunnerWorkItem workItem = null;
workItem = new FuncParallelRunnerWorkItem(() => enqueueable.Count < lowerThreshold, () =>
{
var count = 0;
while (enumerator.MoveNext())
{
// drop the data into the back of the enqueueable
enqueueable.Enqueue(enumerator.Current);
count++;
// special behavior for first loop to spool up quickly
if (firstLoop && count > firstLoopCount)
{
// there's more data in the enumerator, reschedule to run again
firstLoop = false;
_controller.Schedule(workItem);
return;
}
// stop executing if we've dequeued more than the lower threshold or have
// more total that upper threshold in the enqueueable's queue
if (count > lowerThreshold || enqueueable.Count > upperThreshold)
{
// there's more data in the enumerator, reschedule to run again
_controller.Schedule(workItem);
return;
}
}
// we made it here because MoveNext returned false, stop the enqueueable and don't reschedule
enqueueable.Stop();
});
_controller.Schedule(workItem);
}
private void ScheduleEnumerator(Subscription subscription, IEnumerator <BaseData> enumerator, EnqueueableEnumerator <SubscriptionData> enqueueable,
int lowerThreshold, int upperThreshold, SecurityExchangeHours exchangeHours, int firstLoopCount = 5)
{
Action produce = () =>
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.Dispose();
return;
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours, subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if we have more data than the upper threshold in the enqueueable
if (count > upperThreshold)
{
// we use local count for the outside if, for performance, and adjust here
count = enqueueable.Count;
if (count > upperThreshold)
{
return;
}
}
}
// we made it here because MoveNext returned false, stop the enqueueable
enqueueable.Stop();
};
enqueueable.SetProducer(produce, lowerThreshold);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
if (timeTriggered is UserDefinedUniverse)
{
var userDefined = (UserDefinedUniverse)timeTriggered;
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
var currentFrontierUtcTime = _frontierTimeProvider.GetUtcNow();
if (items == null || currentFrontierUtcTime == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(currentFrontierUtcTime, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, currentFrontierUtcTime, collection);
_algorithm.OnSecuritiesChanged(changes);
subscription.OnNewDataAvailable();
};
}
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// we subscribe using a normalized symbol, without a random GUID,
// since the ticker plant will send the coarse data using this symbol
var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
// We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data
_exchange.AddDataHandler(normalizedSymbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
if (!USEnergyAPI.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
if (!FredApi.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
});
enumerator = enqueable;
}
else
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.Type.Name)
{
case nameof(QuoteBar):
var quoteBarAggregator = new QuoteBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.Quote && !tick.Suspicious)
{
quoteBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
});
enumerator = quoteBarAggregator;
break;
case nameof(TradeBar):
var tradeBarAggregator = new TradeBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
var auxDataEnumerator = new LiveAuxiliaryDataEnumerator(
request.Security.Exchange.TimeZone,
_timeProvider);
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
if (data.DataType == MarketDataType.Auxiliary)
{
auxDataEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
else
{
var tick = data as Tick;
if (tick?.TickType == TickType.Trade && !tick.Suspicious)
{
tradeBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
});
enumerator = request.Configuration.SecurityType == SecurityType.Equity
? (IEnumerator <BaseData>) new LiveEquityDataSynchronizingEnumerator(_frontierTimeProvider, request.Security.Exchange.TimeZone, auxDataEnumerator, tradeBarAggregator)
: tradeBarAggregator;
break;
case nameof(OpenInterest):
var oiAggregator = new OpenInterestEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.OpenInterest && !tick.Suspicious)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
case nameof(Tick):
default:
// tick or streaming custom data subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
var tick = data as Tick;
if (tick != null)
{
if (tick.TickType == request.Configuration.TickType)
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
if (tick.TickType != TickType.OpenInterest)
{
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
}
else
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
});
enumerator = tickEnumerator;
break;
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
IEnumerator <BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(dt => new Tick {
Time = dt
}).GetEnumerator();
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(universe, security, config, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="universe"></param>
/// <param name="security">The security to create a subscription for</param>
/// <param name="config">The subscription config to be added</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime utcStartTime, DateTime utcEndTime)
{
Subscription subscription = null;
try
{
var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);
IEnumerator <BaseData> enumerator;
if (config.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseData>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
var factoryReadEnumerator = factory.Read(source).GetEnumerator();
var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
return(new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(config.Symbol, rateLimit);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
if (config.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
public void LiveFillForwardEnumeratorDoesNotStall()
{
var now = DateTime.UtcNow;
var timeProvider = new ManualTimeProvider(new DateTime(2020, 5, 21, 9, 40, 0, 100), TimeZones.NewYork);
var enqueueableEnumerator = new EnqueueableEnumerator <BaseData>();
var fillForwardEnumerator = new LiveFillForwardEnumerator(
timeProvider,
enqueueableEnumerator,
new SecurityExchange(MarketHoursDatabase.FromDataFolder()
.ExchangeHoursListing
.First(kvp => kvp.Key.Market == Market.USA && kvp.Key.SecurityType == SecurityType.Equity)
.Value
.ExchangeHours),
Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()),
false,
now,
Resolution.Minute.ToTimeSpan(),
TimeZones.NewYork,
new DateTime(2020, 5, 21)
);
var openingBar = new TradeBar
{
Open = 0.01m,
High = 0.01m,
Low = 0.01m,
Close = 0.01m,
Volume = 1,
EndTime = new DateTime(2020, 5, 21, 9, 40, 0),
Symbol = Symbols.AAPL
};
var secondBar = new TradeBar
{
Open = 1m,
High = 2m,
Low = 1m,
Close = 2m,
Volume = 100,
EndTime = new DateTime(2020, 5, 21, 9, 42, 0),
Symbol = Symbols.AAPL
};
// Enqueue the first point, which will be emitted ASAP.
enqueueableEnumerator.Enqueue(openingBar);
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.NotNull(fillForwardEnumerator.Current);
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
// Advance the time, we expect a fill-forward bar.
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 41, 0, 100));
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
Assert.AreEqual(openingBar.EndTime.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
// Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 42, 0, 100));
enqueueableEnumerator.Enqueue(secondBar);
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var userDefined = request.Universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionDataSourceReader.ForSource(source, _dataFileProvider, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental) || config.Type == typeof(ETFConstituentData))
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating {config.Type.Name} universe: {config.Symbol.ID}");
// Will try to pull data from the data folder every 10min, file with yesterdays date.
// If lean is started today it will trigger initial coarse universe selection
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
// we adjust time to the previous tradable date
time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone),
TimeSpan.FromMinutes(10)
);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
// aggregates each coarse data point into a single BaseDataCollection
var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, true);
_customExchange.AddEnumerator(config.Symbol, aggregator);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID);
Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) =>
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration);
var input = Subscribe(subRequest.Configuration, (sender, args) => subscription.OnNewDataAvailable());
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID);
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
// send the subscription for the new symbol through to the data queuehandler
if (_channelProvider.ShouldStreamSubscription(subscription.Configuration))
{
Subscribe(request.Configuration, (sender, args) => subscription.OnNewDataAvailable());
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator<BaseData> enumerator;
var userDefined = request.Universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue) return;
var symbol = items.OfType<Symbol>().FirstOrDefault();
if (symbol == null) return;
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
else if (config.Type == typeof (CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol});
var enqueable = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return subscription;
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
IEnumerator<BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(dt => new Tick { Time = dt }).GetEnumerator();
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof (CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] {security.Symbol});
var enqueable = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);
return subscription;
}
public void RecordsMostRecentlyEnqueuedItem()
{
var enumerator = new EnqueueableEnumerator<Tick>();
var currentTime = new DateTime(2015, 12, 01);
var tick1 = new Tick(currentTime, Symbols.SPY, 100, 101);
enumerator.Enqueue(tick1);
Assert.AreEqual(null, enumerator.Current);
Assert.AreEqual(tick1, enumerator.LastEnqueued);
var tick2 = new Tick(currentTime, Symbols.SPY, 100, 101);
enumerator.Enqueue(tick2);
Assert.AreEqual(tick2, enumerator.LastEnqueued);
enumerator.MoveNext();
Assert.AreEqual(tick1, enumerator.Current);
enumerator.MoveNext();
Assert.AreEqual(tick2, enumerator.Current);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(_job, request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
if (!USEnergyAPI.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
if (!FredApi.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
if (!TradingEconomicsCalendar.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
TradingEconomicsCalendar.SetAuthCode(Config.Get("trading-economics-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
EventHandler handler = (sender, args) => subscription?.OnNewDataAvailable();
enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler);
if (request.Configuration.SecurityType == SecurityType.Equity && CorporateEventEnumeratorFactory.ShouldEmitAuxiliaryBaseData(request.Configuration))
{
var dividends = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler);
var splits = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler);
enumerator = new LiveEquityDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, dividends, splits);
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
private void ScheduleEnumerator(IEnumerator<BaseData> enumerator, EnqueueableEnumerator<BaseData> enqueueable, int lowerThreshold, int upperThreshold, int firstLoopCount = 5)
{
// schedule the work on the controller
var firstLoop = true;
FuncParallelRunnerWorkItem workItem = null;
workItem = new FuncParallelRunnerWorkItem(() => enqueueable.Count < lowerThreshold, () =>
{
var count = 0;
while (enumerator.MoveNext())
{
// drop the data into the back of the enqueueable
enqueueable.Enqueue(enumerator.Current);
count++;
// special behavior for first loop to spool up quickly
if (firstLoop && count > firstLoopCount)
{
// there's more data in the enumerator, reschedule to run again
firstLoop = false;
_controller.Schedule(workItem);
return;
}
// stop executing if we've dequeued more than the lower threshold or have
// more total that upper threshold in the enqueueable's queue
if (count > lowerThreshold || enqueueable.Count > upperThreshold)
{
// there's more data in the enumerator, reschedule to run again
_controller.Schedule(workItem);
return;
}
}
// we made it here because MoveNext returned false, stop the enqueueable and don't reschedule
enqueueable.Stop();
});
_controller.Schedule(workItem);
}
/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <param name="factorFileProvider">The factor file provider</param>
/// <param name="enablePriceScale">Enables price factoring</param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator,
IFactorFileProvider factorFileProvider,
bool enablePriceScale)
{
var factorFile = GetFactorFileToUse(request.Configuration, factorFileProvider);
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
var config = subscription.Configuration;
var lastTradableDate = DateTime.MinValue;
decimal?currentScale = null;
Func <int, bool> produce = (workBatchSize) =>
{
try
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.DisposeSafely();
return(false);
}
var data = enumerator.Current;
var requestMode = config.DataNormalizationMode;
var mode = requestMode != DataNormalizationMode.Raw
? requestMode
: DataNormalizationMode.Adjusted;
if (enablePriceScale && data?.Time.Date > lastTradableDate)
{
lastTradableDate = data.Time.Date;
currentScale = GetScaleFactor(factorFile, mode, data.Time.Date);
}
SubscriptionData subscriptionData = SubscriptionData.Create(
config,
exchangeHours,
subscription.OffsetProvider,
data,
mode,
enablePriceScale ? currentScale : null);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if added more data than the work batch size, we don't want to fill the ram
if (count > workBatchSize)
{
return(true);
}
}
}
catch (Exception exception)
{
Log.Error(exception, $"Subscription worker task exception {request.Configuration}.");
}
// we made it here because MoveNext returned false or we exploded, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.DisposeSafely();
return(false);
};
WeightedWorkScheduler.Instance.QueueWork(produce,
// if the subscription finished we return 0, so the work is prioritized and gets removed
() =>
{
if (enqueueable.HasFinished)
{
return(0);
}
var count = enqueueable.Count;
return(count > WeightedWorkScheduler.MaxWorkWeight ? WeightedWorkScheduler.MaxWorkWeight : count);
}
);
return(subscription);
}
public void Step3D()
{
var args = new HashSet <OptimizationParameter>()
{
new OptimizationStepParameter("ema-fast", 10, 100, 1),
new OptimizationStepParameter("ema-slow", 20, 200, 4),
new OptimizationStepParameter("ema-custom", 30, 300, 2)
};
_strategy.Initialize(new Target("Profit", new Maximization(), null), null, args, new StepBaseOptimizationStrategySettings());
var counter = 0;
using (var enumerator = new EnqueueableEnumerator <ParameterSet>())
{
_strategy.NewParameterSet += (s, parameterSet) =>
{
enumerator.Enqueue(parameterSet);
};
_strategy.PushNewResults(OptimizationResult.Initial);
var fastParam = args.First(arg => arg.Name == "ema-fast") as OptimizationStepParameter;
var slowParam = args.First(arg => arg.Name == "ema-slow") as OptimizationStepParameter;
var customParam = args.First(arg => arg.Name == "ema-custom") as OptimizationStepParameter;
using (var fastEnumerator = new OptimizationStepParameterEnumerator(fastParam))
{
using (var slowEnumerator = new OptimizationStepParameterEnumerator(slowParam))
{
using (var customEnumerator = new OptimizationStepParameterEnumerator(customParam))
{
while (fastEnumerator.MoveNext())
{
var fast = fastEnumerator.Current;
slowEnumerator.Reset();
while (slowEnumerator.MoveNext())
{
var slow = slowEnumerator.Current;
customEnumerator.Reset();
while (customEnumerator.MoveNext())
{
var custom = customEnumerator.Current;
counter++;
Assert.IsTrue(enumerator.MoveNext());
var suggestion = enumerator.Current;
Assert.IsNotNull(suggestion);
Assert.IsTrue(suggestion.Value.All(s =>
args.Any(arg => arg.Name == s.Key)));
Assert.AreEqual(3, suggestion.Value.Count());
Assert.AreEqual(fast, suggestion.Value["ema-fast"]);
Assert.AreEqual(slow, suggestion.Value["ema-slow"]);
Assert.AreEqual(custom, suggestion.Value["ema-custom"]);
}
}
}
}
}
}
Assert.AreEqual(0, enumerator.Count);
}
Assert.Greater(counter, 0);
var total = 1m;
foreach (var arg in args.Cast <OptimizationStepParameter>())
{
total *= (arg.MaxValue - arg.MinValue) / arg.Step.Value + 1;
}
Assert.AreEqual(total, counter);
}
public void MoveNextBlocks()
{
var finished = new ManualResetEvent(false);
var enumerator = new EnqueueableEnumerator<Tick>(true);
// producer
int count = 0;
Task.Run(() =>
{
while (!finished.WaitOne(TimeSpan.FromMilliseconds(50)))
{
enumerator.Enqueue(new Tick(DateTime.Now, Symbols.SPY, 100, 101));
count++;
// 5 data points is plenty
if (count > 5)
{
finished.Set();
enumerator.Stop();
}
}
});
// consumer
int dequeuedCount = 0;
bool encounteredError = false;
var consumerTaskFinished = new ManualResetEvent(false);
Task.Run(() =>
{
while (enumerator.MoveNext())
{
dequeuedCount++;
if (enumerator.Current == null)
{
encounteredError = true;
}
}
consumerTaskFinished.Set();
});
finished.WaitOne(Timeout.Infinite);
consumerTaskFinished.WaitOne(Timeout.Infinite);
Assert.IsFalse(enumerator.MoveNext());
Assert.IsFalse(encounteredError);
Assert.AreEqual(count, dequeuedCount);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
if (timeTriggered is UserDefinedUniverse)
{
var userDefined = (UserDefinedUniverse)timeTriggered;
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, tzOffsetProvider, enumerator);
var subscription = new Subscription(request.Universe, request.Security, config, subscriptionDataEnumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator<BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataCollectionSubscriptionEnumeratorFactory(r => new [] {dateInDataTimeZone});
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, frontierAware);
var enqueable = new EnqueueableEnumerator<BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.TickType)
{
case TickType.Quote:
var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Quote)
{
quoteBarAggregator.ProcessData(tick);
if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = quoteBarAggregator;
break;
case TickType.Trade:
default:
var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Trade)
{
tradeBarAggregator.ProcessData(tick);
if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = tradeBarAggregator;
break;
case TickType.OpenInterest:
var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.OpenInterest)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
}
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, subscriptionDataEnumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
var auxEnumerators = new List <IEnumerator <BaseData> >();
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
{
auxEnumerators.Add(auxDataEnumator);
}
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
enumerator = Subscribe(request.Configuration, handler);
if (request.Configuration.EmitSplitsAndDividends())
{
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
}
if (auxEnumerators.Count > 0)
{
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <param name="firstLoopLimit">The first loop data point count for which the worker will stop</param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator,
int firstLoopLimit = 50)
{
var upperThreshold = GetUpperThreshold(request.Configuration.Resolution);
var lowerThreshold = GetLowerThreshold(request.Configuration.Resolution);
if (request.Configuration.Type == typeof(CoarseFundamental))
{
// the lower threshold will be when we start the worker again, if he is stopped
lowerThreshold = 200;
// the upper threshold will stop the worker from loading more data. This is roughly 1 GB
upperThreshold = 500;
}
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
// The first loop of a backtest can load hundreds of subscription feeds, resulting in long delays while the thresholds
// for the buffer are reached. For the first loop start up with just 50 items in the buffer.
var firstLoop = Ref.Create(true);
Action produce = () =>
{
try
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.DisposeSafely();
return;
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours,
subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if we have more data than the upper threshold in the enqueueable, we don't want to fill the ram
if (count > upperThreshold || count > firstLoopLimit && firstLoop.Value)
{
// we use local count for the outside if, for performance, and adjust here
count = enqueueable.Count;
if (count > upperThreshold || firstLoop.Value)
{
firstLoop.Value = false;
// we will be re scheduled to run by the consumer, see EnqueueableEnumerator
// if the consumer is already waiting for us wake him up, he will rescheduled us if required
enqueueable.CancellationTokenSource.Cancel();
return;
}
}
}
}
catch (Exception exception)
{
Log.Error(exception, $"Subscription worker task exception {request.Configuration}.");
}
// we made it here because MoveNext returned false or we exploded, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.DisposeSafely();
};
enqueueable.SetProducer(produce, lowerThreshold);
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
return(new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider));
});
// rate limit the refreshing of the stack to the requested interval
// At Tick resolution, it will refresh at full speed
// At Second and Minute resolution, it will refresh every second and minute respectively
// At Hour and Daily resolutions, it will refresh every 30 minutes
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.TickType)
{
case TickType.Quote:
var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Quote)
{
quoteBarAggregator.ProcessData(tick);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = quoteBarAggregator;
break;
case TickType.Trade:
default:
var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Trade)
{
tradeBarAggregator.ProcessData(tick);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = tradeBarAggregator;
break;
case TickType.OpenInterest:
var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.OpenInterest)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
}
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <param name="factorFileProvider">The factor file provider</param>
/// <param name="enablePriceScale">Enables price factoring</param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator,
IFactorFileProvider factorFileProvider,
bool enablePriceScale)
{
var factorFile = GetFactorFileToUse(request.Configuration, factorFileProvider);
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
var config = subscription.Configuration;
var lastTradableDate = DateTime.MinValue;
decimal?currentScale = null;
Func <int, bool> produce = (workBatchSize) =>
{
try
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.DisposeSafely();
return(false);
}
var data = enumerator.Current;
// Use our config filter to see if we should emit this
// This currently catches Auxiliary data that we don't want to emit
if (data != null && !config.ShouldEmitData(data))
{
continue;
}
// In the event we have "Raw" configuration, we will force our subscription data
// to precalculate adjusted data. The data will still be emitted as raw, but
// if the config is changed at any point it can emit adjusted data as well
// See SubscriptionData.Create() and PrecalculatedSubscriptionData for more
var requestMode = config.DataNormalizationMode;
var mode = requestMode != DataNormalizationMode.Raw
? requestMode
: DataNormalizationMode.Adjusted;
// We update our price scale factor when the date changes for non fill forward bars or if we haven't initialized yet.
// We don't take into account auxiliary data because we don't scale it and because the underlying price data could be fill forwarded
if (enablePriceScale && data?.Time.Date > lastTradableDate && data.DataType != MarketDataType.Auxiliary && (!data.IsFillForward || lastTradableDate == DateTime.MinValue))
{
lastTradableDate = data.Time.Date;
currentScale = GetScaleFactor(factorFile, mode, data.Time.Date);
}
SubscriptionData subscriptionData = SubscriptionData.Create(
config,
exchangeHours,
subscription.OffsetProvider,
data,
mode,
enablePriceScale ? currentScale : null);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if added more data than the work batch size, we don't want to fill the ram
if (count > workBatchSize)
{
return(true);
}
}
}
catch (Exception exception)
{
Log.Error(exception, $"Subscription worker task exception {request.Configuration}.");
}
// we made it here because MoveNext returned false or we exploded, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.DisposeSafely();
return(false);
};
WeightedWorkScheduler.Instance.QueueWork(config.Symbol, produce,
// if the subscription finished we return 0, so the work is prioritized and gets removed
() =>
{
if (enqueueable.HasFinished)
{
return(0);
}
var count = enqueueable.Count;
return(count > WeightedWorkScheduler.MaxWorkWeight ? WeightedWorkScheduler.MaxWorkWeight : count);
}
);
return(subscription);
}
/// <summary>
/// Enqueues the data
/// </summary>
/// <param name="data">The data to be handled</param>
public override void HandleData(BaseData data)
{
_enqueueable.Enqueue(data);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="universe"></param>
/// <param name="security">The security to create a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
{
Subscription subscription = null;
try
{
var config = security.SubscriptionDataConfig;
var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);
IEnumerator<BaseData> enumerator;
if (config.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseData>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
var source = sourceProvider.GetSource(config, currentLocalDate, true);
var factoryReadEnumerator = factory.Read(source).GetEnumerator();
var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(rateLimit);
var enqueable = new EnqueueableEnumerator<BaseData>();
_customExchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
// TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
// finally, make our subscriptions aware of the frontier of the data feed, this will help
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}