public void FiltersOutPastData()
{
var start = new DateTime(2015, 10, 10, 13, 0, 0);
var data = new List <Tick>
{
new Tick {
Time = start.AddMinutes(-1)
},
new Tick {
Time = start.AddSeconds(-1)
},
new Tick {
Time = start.AddSeconds(1)
},
new Tick {
Time = start.AddSeconds(0)
},
new Tick {
Time = start.AddSeconds(2)
}
};
var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(0.5));
Assert.IsTrue(fastForward.MoveNext());
Assert.AreEqual(start.AddSeconds(1), fastForward.Current.Time);
Assert.IsTrue(fastForward.MoveNext());
Assert.AreEqual(start.AddSeconds(2), fastForward.Current.Time);
}
/// <summary>
/// Creates an enumerator to read the specified request.
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var config = request.Configuration;
// frontier value used to prevent emitting duplicate time stamps between refreshed enumerators
// also provides some immediate fast-forward to handle spooling through remote files quickly
var frontier = Ref.Create(DateTime.MinValue);
var lastSourceRefreshTime = DateTime.MinValue;
var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
// this is refreshing the enumerator stack for each new source
var refresher = new RefreshEnumerator <BaseData>(() =>
{
// rate limit the refresh of this enumerator stack
var utcNow = _timeProvider.GetUtcNow();
var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment);
if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls)
{
return(Enumerable.Empty <BaseData>().GetEnumerator());
}
lastSourceRefreshTime = utcNow;
var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date;
var source = sourceFactory.GetSource(config, localDate, true);
// fetch the new source and enumerate the data source reader
var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate);
if (SourceRequiresFastForward(source))
{
// apply fast forward logic for file transport mediums
var maximumDataAge = GetMaximumDataAge(config.Increment);
enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge);
}
else
{
// rate limit calls to this enumerator stack
enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls);
}
if (source.Format == FileFormat.Collection)
{
// unroll collections into individual data points after fast forward/rate limiting applied
enumerator = enumerator.SelectMany(data =>
{
var collection = data as BaseDataCollection;
return(collection?.Data.GetEnumerator() ?? new List <BaseData> {
data
}.GetEnumerator());
});
}
return(enumerator);
});
// prevent calls to the enumerator stack if current is in the future
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
return(new FrontierAwareEnumerator(refresher, _timeProvider, timeZoneOffsetProvider));
}
public void FastForwardsOldDataAllowsEquals()
{
var start = new DateTime(2015, 10, 10, 13, 0, 0);
var data = new List<Tick>
{
new Tick {Time = start.AddMinutes(-1)},
new Tick {Time = start.AddSeconds(-1)},
new Tick {Time = start.AddSeconds(0)},
new Tick {Time = start.AddSeconds(1)},
};
var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(1));
Assert.IsTrue(fastForward.MoveNext());
Assert.AreEqual(start.AddSeconds(-1), fastForward.Current.Time);
}
public void CurrentIsNullWhenEnumeratorReturnsFalse()
{
var start = new DateTime(2015, 10, 10, 13, 0, 0);
var data = new List <Tick>
{
new Tick {
Time = start.AddSeconds(0)
}
};
var timeProvider = new ManualTimeProvider(start, TimeZones.Utc);
var fastForward = new FastForwardEnumerator(data.GetEnumerator(), timeProvider, TimeZones.Utc, TimeSpan.FromSeconds(0.5));
Assert.IsTrue(fastForward.MoveNext());
Assert.IsNotNull(fastForward.Current);
Assert.IsFalse(fastForward.MoveNext());
Assert.IsNull(fastForward.Current);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var userDefined = request.Universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionDataSourceReader.ForSource(source, _dataFileProvider, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
return(new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider));
});
// rate limit the refreshing of the stack to the requested interval
// At Tick resolution, it will refresh at full speed
// At Second and Minute resolution, it will refresh every second and minute respectively
// At Hour and Daily resolutions, it will refresh every 30 minutes
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.TickType)
{
case TickType.Quote:
var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Quote)
{
quoteBarAggregator.ProcessData(tick);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = quoteBarAggregator;
break;
case TickType.Trade:
default:
var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Trade)
{
tradeBarAggregator.ProcessData(tick);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = tradeBarAggregator;
break;
case TickType.OpenInterest:
var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.OpenInterest)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
}
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator<BaseData> enumerator;
var userDefined = request.Universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue) return;
var symbol = items.OfType<Symbol>().FirstOrDefault();
if (symbol == null) return;
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
else if (config.Type == typeof (CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol});
var enqueable = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return subscription;
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);
var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder().GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
IEnumerator <BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
.Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
.Select(dt => new Tick {
Time = dt
}).GetEnumerator();
}
else if (config.Type == typeof(CoarseFundamental))
{
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new Dictionary <SecurityType, List <string> >
{
{ config.SecurityType, new List <string> {
config.Symbol
} }
});
var enqueable = new EnqueableEnumerator <BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
});
enumerator = enqueable;
}
else
{
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(config.TimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
var source = sourceProvider.GetSource(config, currentLocalDate, true);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, config.TimeZone, config.Increment);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(config.TimeZone, startTimeUtc, endTimeUtc);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider);
return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(rateLimit);
var enqueable = new EnqueableEnumerator <BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
else
{
enqueable.Enqueue(data);
}
});
enumerator = enqueable;
}
// create the subscription
var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc, true);
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="universe"></param>
/// <param name="security">The security to create a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
{
Subscription subscription = null;
try
{
var config = security.SubscriptionDataConfig;
var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone);
var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime);
IEnumerator <BaseData> enumerator;
if (config.IsCustomData)
{
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseData>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(config.TimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
var source = sourceProvider.GetSource(config, currentLocalDate, true);
var factoryReadEnumerator = factory.Read(source).GetEnumerator();
var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, config.TimeZone, maximumDataAge);
return(new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(rateLimit);
var enqueable = new EnqueableEnumerator <BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider);
_exchange.SetHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueableEnumerator <BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
// TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
// finally, make our subscriptions aware of the frontier of the data feed, this will help
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
IEnumerator <BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(dt => new Tick {
Time = dt
}).GetEnumerator();
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(universe, security, config, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);
return(subscription);
}
/// <summary>
/// Creates an enumerator to read the specified request.
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var config = request.Configuration;
// frontier value used to prevent emitting duplicate time stamps between refreshed enumerators
// also provides some immediate fast-forward to handle spooling through remote files quickly
var frontier = Ref.Create(request.StartTimeLocal);
var lastSourceRefreshTime = DateTime.MinValue;
var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
// this is refreshing the enumerator stack for each new source
var refresher = new RefreshEnumerator <BaseData>(() =>
{
// rate limit the refresh of this enumerator stack
var utcNow = _timeProvider.GetUtcNow();
var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment);
if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls)
{
return(Enumerable.Empty <BaseData>().GetEnumerator());
}
lastSourceRefreshTime = utcNow;
var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date;
var source = sourceFactory.GetSource(config, localDate, true);
// fetch the new source and enumerate the data source reader
var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate);
if (SourceRequiresFastForward(source))
{
// The FastForwardEnumerator implements these two features:
// (1) make sure we never emit past data
// (2) data filtering based on a maximum data age
// For custom data we don't want feature (2) because we would reject data points emitted later
// (e.g. Quandl daily data after a weekend), so we disable it using a huge maximum data age.
// apply fast forward logic for file transport mediums
var maximumDataAge = GetMaximumDataAge(Time.MaxTimeSpan);
enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge);
}
else
{
// rate limit calls to this enumerator stack
enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls);
}
if (source.Format == FileFormat.Collection)
{
// unroll collections into individual data points after fast forward/rate limiting applied
enumerator = enumerator.SelectMany(data =>
{
var collection = data as BaseDataCollection;
return(collection?.Data.GetEnumerator() ?? new List <BaseData> {
data
}.GetEnumerator());
});
}
return(enumerator);
});
return(refresher);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="security">The security to create a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription)
{
Subscription subscription = null;
try
{
var config = security.SubscriptionDataConfig;
var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone);
var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone);
IEnumerator<BaseData> enumerator;
if (config.IsCustomData)
{
// custom data uses backtest readers
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false);
// apply fast forwarding, this is especially important for RemoteFile types that
// can send in large chunks of old, irrelevant data
var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment);
// apply rate limits (1x per increment, max 30 minutes between calls)
// TODO : Pull limits from config file?
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
// add the enumerator to the exchange
_customExchange.AddEnumerator(rateLimit);
// this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider);
_exchange.SetHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
// TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
// finally, make our subscriptions aware of the frontier of the data feed, this will help
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(universe.Market, config.Symbol, universe.SecurityType);
var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
IEnumerator<BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
.Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
.Select(dt => new Tick { Time = dt }).GetEnumerator();
}
else if (config.Type == typeof (CoarseFundamental))
{
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>>
{
{config.SecurityType, new List<Symbol>{config.Symbol}}
});
var enqueable = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
});
enumerator = enqueable;
}
else
{
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
var source = sourceProvider.GetSource(config, currentLocalDate, true);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(rateLimit);
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
else
{
enqueable.Enqueue(data);
}
});
enumerator = enqueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
return subscription;
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="universe"></param>
/// <param name="security">The security to create a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
{
Subscription subscription = null;
try
{
var config = security.SubscriptionDataConfig;
var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);
IEnumerator<BaseData> enumerator;
if (config.IsCustomData)
{
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseData>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
var source = sourceProvider.GetSource(config, currentLocalDate, true);
var factoryReadEnumerator = factory.Read(source).GetEnumerator();
var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(rateLimit);
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
_exchange.SetHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
// TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
// finally, make our subscriptions aware of the frontier of the data feed, this will help
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
IEnumerator<BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(dt => new Tick { Time = dt }).GetEnumerator();
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof (CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] {security.Symbol});
var enqueable = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);
return subscription;
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator<BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
return new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider);
});
// rate limit the refreshing of the stack to the requested interval
// At Tick resolution, it will refresh at full speed
// At Second and Minute resolution, it will refresh every second and minute respectively
// At Hour and Daily resolutions, it will refresh every 30 minutes
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);
var enqueable = new EnqueueableEnumerator<BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
