private Subscription CreateSubscription(Resolution resolution, string symbol = "AAPL", bool isInternalFeed = false,
SecurityType type = SecurityType.Equity, TickType tickType = TickType.Trade)
{
var start = DateTime.UtcNow;
var end = start.AddSeconds(10);
Security security;
Symbol _symbol;
if (type == SecurityType.Equity)
{
_symbol = new Symbol(SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA), symbol);
security = new Equity(
_symbol,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
);
}
else if (type == SecurityType.Option)
{
_symbol = new Symbol(SecurityIdentifier.GenerateOption(DateTime.Now,
SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA),
Market.USA, 0.0m, OptionRight.Call, OptionStyle.American), symbol);
security = new Option(
_symbol,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
);
}
else if (type == SecurityType.Future)
{
_symbol = new Symbol(SecurityIdentifier.GenerateFuture(DateTime.Now, symbol, Market.USA), symbol);
security = new Future(
_symbol,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
);
}
else
{
throw new Exception("SecurityType not implemented");
}
var config = new SubscriptionDataConfig(typeof(TradeBar), _symbol, resolution, DateTimeZone.Utc, DateTimeZone.Utc, true, false, isInternalFeed, false, tickType);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, start, end);
var enumerator = new EnqueueableEnumerator <BaseData>();
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
/// <summary>
/// Creates a subscription to process the history request
/// </summary>
private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history)
{
// data reader expects these values in local times
var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var reader = history.GetEnumerator();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone, start);
}
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
/// <summary>
/// Creates a new <see cref="Subscription"/> which will directly consume the provided enumerator
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription Create(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator)
{
var exchangeHours = request.Security.Exchange.Hours;
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var dataEnumerator = new SubscriptionDataEnumerator(
request.Configuration,
exchangeHours,
timeZoneOffsetProvider,
enumerator
);
return(new Subscription(request, dataEnumerator, timeZoneOffsetProvider));
}
public void EnumeratorEmitsAuxData(Type typeOfConfig, bool shouldReceiveAuxData)
{
var config = CreateConfig(Resolution.Hour, typeOfConfig);
var security = GetSecurity(config);
var time = new DateTime(2010, 1, 1);
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, time, time.AddDays(1));
// Make a aux data stream; for this testing case we will just use delisting data points
var totalPoints = 8;
var stream = Enumerable.Range(0, totalPoints).Select(x => new Delisting {
Time = time.AddHours(x)
}).GetEnumerator();
var enumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, tzOffsetProvider, stream, false);
// Test our SubscriptionDataEnumerator to see if it emits the aux data
int dataReceivedCount = 0;
while (enumerator.MoveNext())
{
dataReceivedCount++;
if (enumerator.Current != null && enumerator.Current.Data.DataType == MarketDataType.Auxiliary)
{
Assert.IsTrue(shouldReceiveAuxData);
}
}
// If it should receive aux data it should have emitted all points
// otherwise none should have been emitted
if (shouldReceiveAuxData)
{
Assert.AreEqual(totalPoints, dataReceivedCount);
}
else
{
Assert.AreEqual(0, dataReceivedCount);
}
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental) || config.Type == typeof(ETFConstituentData))
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating {config.Type.Name} universe: {config.Symbol.ID}");
// Will try to pull data from the data folder every 10min, file with yesterdays date.
// If lean is started today it will trigger initial coarse universe selection
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
// we adjust time to the previous tradable date
time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone),
TimeSpan.FromMinutes(10)
);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
// aggregates each coarse data point into a single BaseDataCollection
var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, true);
_customExchange.AddEnumerator(config.Symbol, aggregator);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID);
Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) =>
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration);
var input = Subscribe(subRequest.Configuration, (sender, args) => subscription.OnNewDataAvailable());
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID);
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
// send the subscription for the new symbol through to the data queuehandler
if (_channelProvider.ShouldStreamSubscription(subscription.Configuration))
{
Subscribe(request.Configuration, (sender, args) => subscription.OnNewDataAvailable());
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
var auxEnumerators = new List <IEnumerator <BaseData> >();
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
{
auxEnumerators.Add(auxDataEnumator);
}
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
enumerator = Subscribe(request.Configuration, handler);
if (request.Configuration.EmitSplitsAndDividends())
{
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
}
if (auxEnumerators.Count > 0)
{
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
if (timeTriggered is UserDefinedUniverse)
{
var userDefined = (UserDefinedUniverse)timeTriggered;
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
var currentFrontierUtcTime = _frontierTimeProvider.GetUtcNow();
if (items == null || currentFrontierUtcTime == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(currentFrontierUtcTime, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, currentFrontierUtcTime, collection);
_algorithm.OnSecuritiesChanged(changes);
subscription.OnNewDataAvailable();
};
}
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// we subscribe using a normalized symbol, without a random GUID,
// since the ticker plant will send the coarse data using this symbol
var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
// We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data
_exchange.AddDataHandler(normalizedSymbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
if (!USEnergyAPI.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
if (!FredApi.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
});
enumerator = enqueable;
}
else
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.Type.Name)
{
case nameof(QuoteBar):
var quoteBarAggregator = new QuoteBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.Quote && !tick.Suspicious)
{
quoteBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
});
enumerator = quoteBarAggregator;
break;
case nameof(TradeBar):
var tradeBarAggregator = new TradeBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
var auxDataEnumerator = new LiveAuxiliaryDataEnumerator(
request.Security.Exchange.TimeZone,
_timeProvider);
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
if (data.DataType == MarketDataType.Auxiliary)
{
auxDataEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
else
{
var tick = data as Tick;
if (tick?.TickType == TickType.Trade && !tick.Suspicious)
{
tradeBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
});
enumerator = request.Configuration.SecurityType == SecurityType.Equity
? (IEnumerator <BaseData>) new LiveEquityDataSynchronizingEnumerator(_frontierTimeProvider, request.Security.Exchange.TimeZone, auxDataEnumerator, tradeBarAggregator)
: tradeBarAggregator;
break;
case nameof(OpenInterest):
var oiAggregator = new OpenInterestEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.OpenInterest && !tick.Suspicious)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
case nameof(Tick):
default:
// tick or streaming custom data subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
var tick = data as Tick;
if (tick != null)
{
if (tick.TickType == request.Configuration.TickType)
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
if (tick.TickType != TickType.OpenInterest)
{
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
}
else
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
});
enumerator = tickEnumerator;
break;
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(_job, request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
if (!USEnergyAPI.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
if (!FredApi.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
if (!TradingEconomicsCalendar.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
TradingEconomicsCalendar.SetAuthCode(Config.Get("trading-economics-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
EventHandler handler = (sender, args) => subscription?.OnNewDataAvailable();
enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler);
if (request.Configuration.SecurityType == SecurityType.Equity && CorporateEventEnumeratorFactory.ShouldEmitAuxiliaryBaseData(request.Configuration))
{
var dividends = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler);
var splits = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler);
enumerator = new LiveEquityDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, dividends, splits);
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
);
var mapFileResolver = MapFileResolver.Empty;
if (config.TickerShouldBeMapped())
{
mapFileResolver = _mapFileProvider.Get(config.Market);
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
config.MappedSymbol = mapFile.GetMappedSymbol(start, config.MappedSymbol);
}
var dataReader = new SubscriptionDataReader(config,
start,
end,
mapFileResolver,
_factorFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
_dataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); };
var reader = CorporateEventEnumeratorFactory.CreateEnumerators(
dataReader,
config,
_factorFileProvider,
dataReader,
mapFileResolver,
false);
// has to be initialized after adding all the enumerators since it will execute a MoveNext
dataReader.Initialize();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone, start);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
public void EnumerationWhileUpdatingDoesNotThrow()
{
var cts = new CancellationTokenSource();
var subscriptions = new SubscriptionCollection();
var start = DateTime.UtcNow;
var end = start.AddSeconds(10);
var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Minute, DateTimeZone.Utc, DateTimeZone.Utc, true, false, false);
var security = new Equity(
Symbols.SPY,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, start, end);
var enumerator = new EnqueueableEnumerator <BaseData>();
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, enumerator, false);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
var subscription = new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider);
var addTask = Task.Factory.StartNew(() =>
{
Log.Trace("Add task started");
while (DateTime.UtcNow < end)
{
if (!subscriptions.Contains(config))
{
subscriptions.TryAdd(subscription);
}
Thread.Sleep(1);
}
Log.Trace("Add task ended");
}, cts.Token);
var removeTask = Task.Factory.StartNew(() =>
{
Log.Trace("Remove task started");
while (DateTime.UtcNow < end)
{
Subscription removed;
subscriptions.TryRemove(config, out removed);
Thread.Sleep(1);
}
Log.Trace("Remove task ended");
}, cts.Token);
var readTask = Task.Factory.StartNew(() =>
{
Log.Trace("Read task started");
while (DateTime.UtcNow < end)
{
foreach (var sub in subscriptions)
{
}
Thread.Sleep(1);
}
Log.Trace("Read task ended");
}, cts.Token);
Task.WaitAll(addTask, removeTask, readTask);
subscription.Dispose();
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
private Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
// during warmup we might get requested to add some asset which has already expired in which case the live enumerator will be empty
if (!IsExpired(request.Configuration))
{
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack, handleData: data =>
{
enqueable.Enqueue(data);
subscription?.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
var auxEnumerators = new List <IEnumerator <BaseData> >();
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
{
auxEnumerators.Add(auxDataEnumator);
}
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
enumerator = Subscribe(request.Configuration, handler);
if (request.Configuration.EmitSplitsAndDividends())
{
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
}
if (auxEnumerators.Count > 0)
{
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
}
}
// scale prices before 'SubscriptionFilterEnumerator' since it updates securities realtime price
// and before fill forwarding so we don't happen to apply twice the factor
if (request.Configuration.PricesShouldBeScaled(liveMode: true))
{
enumerator = new PriceScaleFactorEnumerator(
enumerator,
request.Configuration,
_factorFileProvider,
liveMode: true);
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
// define market hours and user filters to incoming data after the frontier enumerator so during warmup we avoid any realtime data making it's way into the securities
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
}
}
else
{
enumerator = Enumerable.Empty <BaseData>().GetEnumerator();
}
enumerator = GetWarmupEnumerator(request, enumerator);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
