private Subscription CreateDataSubscription(SubscriptionRequest request)
{
// ReSharper disable once PossibleMultipleEnumeration
if (!request.TradableDays.Any())
{
_algorithm.Error(
$"No data loaded for {request.Security.Symbol} because there were no tradeable dates for this security."
);
return(null);
}
// ReSharper disable once PossibleMultipleEnumeration
var enumeratorFactory = GetEnumeratorFactory(request);
var enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = ConfigureEnumerator(request, false, enumerator);
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
// add this enumerator to our exchange
ScheduleEnumerator(subscription,
enumerator,
enqueueable,
GetLowerThreshold(request.Configuration.Resolution),
GetUpperThreshold(request.Configuration.Resolution),
request.Security.Exchange.Hours);
return(subscription);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// grab the relevant exchange hours
var config = request.Configuration;
// define our data enumerator
var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request, _dataProvider);
var firstLoopCount = 5;
var lowerThreshold = GetLowerThreshold(config.Resolution);
var upperThreshold = GetUpperThreshold(config.Resolution);
if (config.Type == typeof(CoarseFundamental))
{
firstLoopCount = 2;
// the lower threshold will be when we start the worker again, if he is stopped
lowerThreshold = 200;
// the upper threshold will stop the worker from loading more data. This is roughly 1 GB
upperThreshold = 500;
}
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
// add this enumerator to our exchange
ScheduleEnumerator(subscription, enumerator, enqueueable, lowerThreshold, upperThreshold, request.Security.Exchange.Hours, firstLoopCount);
return(subscription);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// grab the relevant exchange hours
var config = request.Configuration;
// define our data enumerator
var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request, _dataProvider);
var firstLoopCount = 5;
var lowerThreshold = GetLowerThreshold(config.Resolution);
var upperThreshold = GetUpperThreshold(config.Resolution);
if (config.Type == typeof(CoarseFundamental))
{
firstLoopCount = 2;
lowerThreshold = 5;
upperThreshold = 100000;
}
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request.Universe, request.Security, config, enqueueable, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
// add this enumerator to our exchange
ScheduleEnumerator(subscription, enumerator, enqueueable, lowerThreshold, upperThreshold, firstLoopCount);
return(subscription);
}
private Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime startTimeUtc, DateTime endTimeUtc)
{
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return(null);
}
// get the map file resolver for this market
var mapFileResolver = MapFileResolver.Empty;
if (config.SecurityType == SecurityType.Equity)
{
mapFileResolver = _mapFileProvider.Get(config.Market);
}
// ReSharper disable once PossibleMultipleEnumeration
var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates);
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return(subscription);
}
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef <TimeSpan> fillForwardResolution)
{
var config = security.SubscriptionDataConfig;
var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return(null);
}
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return(subscription);
}
public void YieldsFutureDataAtCorrectTime()
{
var currentTime = new DateTime(2015, 10, 13);
var timeProvider = new ManualTimeProvider(currentTime);
var underlying = new List<Tick>
{
new Tick {Time = currentTime.AddSeconds(10)}
};
var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
for (int i = 0; i < 10; i++)
{
timeProvider.AdvanceSeconds(1);
Assert.IsTrue(frontierAware.MoveNext());
if (i < 9)
{
Assert.IsNull(frontierAware.Current);
}
else
{
Assert.IsNotNull(frontierAware.Current);
Assert.AreEqual(underlying[0], frontierAware.Current);
}
}
}
public bool UpdateRealTimePrice(
Subscription subscription,
TimeZoneOffsetProvider timeZoneOffsetProvider,
SecurityExchangeHours exchangeHours)
{
return(UpdateSubscriptionRealTimePrice(subscription, timeZoneOffsetProvider, exchangeHours, new Tick()));
}
/// <summary>
/// Creates a subscription to process the history request
/// </summary>
private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history)
{
// data reader expects these values in local times
var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.TimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
null,
true,
request.DataNormalizationMode
);
var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency));
var reader = history.GetEnumerator();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment);
}
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
return(new Subscription(null, security, config, reader, timeZoneOffsetProvider, start, end, false));
}
public void CreatedSubscriptionDoesNotRoundDownForPeriodLessData()
{
var data = new MyCustomData
{
Time = new DateTime(2020, 5, 21, 8, 9, 0),
Symbol = Symbols.SPY
};
var config = new SubscriptionDataConfig(
typeof(TradeBar),
Symbols.SPY,
Resolution.Hour,
TimeZones.Utc,
TimeZones.Utc,
false,
false,
false
);
var exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.Utc);
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.Utc, new DateTime(2020, 5, 21), new DateTime(2020, 5, 22));
var subscription = SubscriptionData.Create(config, exchangeHours, offsetProvider, data);
Assert.AreEqual(new DateTime(2020, 5, 21, 8, 9, 0), subscription.Data.Time);
Assert.AreEqual(new DateTime(2020, 5, 21, 8, 9, 0), subscription.Data.EndTime);
}
/// <summary>
/// Creates an enumerator to read the specified request.
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var config = request.Configuration;
// frontier value used to prevent emitting duplicate time stamps between refreshed enumerators
// also provides some immediate fast-forward to handle spooling through remote files quickly
var frontier = Ref.Create(DateTime.MinValue);
var lastSourceRefreshTime = DateTime.MinValue;
var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
// this is refreshing the enumerator stack for each new source
var refresher = new RefreshEnumerator <BaseData>(() =>
{
// rate limit the refresh of this enumerator stack
var utcNow = _timeProvider.GetUtcNow();
var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment);
if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls)
{
return(Enumerable.Empty <BaseData>().GetEnumerator());
}
lastSourceRefreshTime = utcNow;
var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date;
var source = sourceFactory.GetSource(config, localDate, true);
// fetch the new source and enumerate the data source reader
var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate);
if (SourceRequiresFastForward(source))
{
// apply fast forward logic for file transport mediums
var maximumDataAge = GetMaximumDataAge(config.Increment);
enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge);
}
else
{
// rate limit calls to this enumerator stack
enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls);
}
if (source.Format == FileFormat.Collection)
{
// unroll collections into individual data points after fast forward/rate limiting applied
enumerator = enumerator.SelectMany(data =>
{
var collection = data as BaseDataCollection;
return(collection?.Data.GetEnumerator() ?? new List <BaseData> {
data
}.GetEnumerator());
});
}
return(enumerator);
});
// prevent calls to the enumerator stack if current is in the future
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
return(new FrontierAwareEnumerator(refresher, _timeProvider, timeZoneOffsetProvider));
}
public void CreatedSubscriptionRoundsTimeDownForDataWithPeriod()
{
var tb = new TradeBar
{
Time = new DateTime(2020, 5, 21, 8, 9, 0),
Period = TimeSpan.FromHours(1),
Symbol = Symbols.SPY
};
var config = new SubscriptionDataConfig(
typeof(TradeBar),
Symbols.SPY,
Resolution.Hour,
TimeZones.Utc,
TimeZones.Utc,
false,
false,
false
);
var exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.Utc);
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.Utc, new DateTime(2020, 5, 21), new DateTime(2020, 5, 22));
var subscription = SubscriptionData.Create(config, exchangeHours, offsetProvider, tb);
Assert.AreEqual(new DateTime(2020, 5, 21, 8, 0, 0), subscription.Data.Time);
Assert.AreEqual(new DateTime(2020, 5, 21, 9, 0, 0), subscription.Data.EndTime);
}
public void ReturnsCurrentOffset()
{
var utcDate = new DateTime(2015, 07, 07);
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1));
var currentOffset = offsetProvider.GetOffsetTicks(utcDate);
Assert.AreEqual(-TimeSpan.FromHours(4).TotalHours, TimeSpan.FromTicks(currentOffset).TotalHours);
}
public void YieldsFutureDataAtCorrectTime()
{
var currentTime = new DateTime(2015, 10, 13);
var timeProvider = new ManualTimeProvider(currentTime);
var underlying = new List <Tick>
{
new Tick {
Time = currentTime.AddSeconds(10)
}
};
var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
for (int i = 0; i < 10; i++)
{
timeProvider.AdvanceSeconds(1);
Assert.IsTrue(frontierAware.MoveNext());
if (i < 9)
{
Assert.IsNull(frontierAware.Current);
}
else
{
Assert.IsNotNull(frontierAware.Current);
Assert.AreEqual(underlying[0], frontierAware.Current);
}
}
}
public Subscription CreateSubscription(SubscriptionRequest request)
{
var offsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone,
request.StartTimeUtc,
request.EndTimeUtc);
return(new Subscription(request, _dummyData.GetEnumerator(), offsetProvider));
}
public void ReturnsCurrentOffset()
{
var utcDate = new DateTime(2015, 07, 07);
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1));
var currentOffset = offsetProvider.GetOffsetTicks(utcDate);
Assert.AreEqual(-TimeSpan.FromHours(4).TotalHours, TimeSpan.FromTicks(currentOffset).TotalHours);
}
public void ReturnsCorrectOffsetAfterDST()
{
// the exact instant DST goes into affect
var utcDate = new DateTime(2015, 03, 08, 2, 0, 0).AddHours(5);
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1));
var currentOffset = offsetProvider.GetOffsetTicks(utcDate);
Assert.AreEqual(-TimeSpan.FromHours(4).TotalHours, TimeSpan.FromTicks(currentOffset).TotalHours);
}
private Subscription CreateSubscription(Resolution resolution, string symbol = "AAPL", bool isInternalFeed = false,
SecurityType type = SecurityType.Equity, TickType tickType = TickType.Trade)
{
var start = DateTime.UtcNow;
var end = start.AddSeconds(10);
Security security;
Symbol _symbol;
if (type == SecurityType.Equity)
{
_symbol = new Symbol(SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA), symbol);
security = new Equity(
_symbol,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
);
}
else if (type == SecurityType.Option)
{
_symbol = new Symbol(SecurityIdentifier.GenerateOption(DateTime.Now,
SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA),
Market.USA, 0.0m, OptionRight.Call, OptionStyle.American), symbol);
security = new Option(
_symbol,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
);
}
else if (type == SecurityType.Future)
{
_symbol = new Symbol(SecurityIdentifier.GenerateFuture(DateTime.Now, symbol, Market.USA), symbol);
security = new Future(
_symbol,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
);
}
else
{
throw new Exception("SecurityType not implemented");
}
var config = new SubscriptionDataConfig(typeof(TradeBar), _symbol, resolution, DateTimeZone.Utc, DateTimeZone.Utc, true, false, isInternalFeed, false, tickType);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, start, end);
var enumerator = new EnqueueableEnumerator <BaseData>();
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
/// <summary>
/// Initializes a new instance of the <see cref="Subscription"/> class
/// </summary>
/// <param name="security">The security this subscription is for</param>
/// <param name="enumerator">The subscription's data source</param>
/// <param name="utcStartTime">The start time of the subscription</param>
/// <param name="utcEndTime">The end time of the subscription</param>
/// <param name="isUserDefined">True if the user explicitly defined this subscription, false otherwise</param>
/// <param name="isFundamentalSubscription">True if this subscription is used to define the times to perform universe selection
/// for a specific market, false for all other subscriptions</param>
public Subscription(Security security, IEnumerator <BaseData> enumerator, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefined, bool isFundamentalSubscription)
{
Security = security;
_enumerator = enumerator;
IsUserDefined = isUserDefined;
IsFundamentalSubscription = isFundamentalSubscription;
Configuration = security.SubscriptionDataConfig;
OffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime);
}
public void ReturnsCorrectOffsetAfterDST()
{
// the exact instant DST goes into affect
var utcDate = new DateTime(2015, 03, 08, 2, 0, 0).AddHours(5);
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1));
var currentOffset = offsetProvider.GetOffsetTicks(utcDate);
Assert.AreEqual(-TimeSpan.FromHours(4).TotalHours, TimeSpan.FromTicks(currentOffset).TotalHours);
}
/// <summary>
/// Will wrap the provided enumerator with a <see cref="FrontierAwareEnumerator"/>
/// using a <see cref="PredicateTimeProvider"/> that will advance time based on the provided
/// function
/// </summary>
/// <remarks>Won't advance time if now.Hour is bigger or equal than 23pm, less or equal than 5am or Saturday.
/// This is done to prevent universe selection occurring in those hours so that the subscription changes
/// are handled correctly.</remarks>
private IEnumerator <BaseData> GetConfiguredFrontierAwareEnumerator(
IEnumerator <BaseData> enumerator,
TimeZoneOffsetProvider tzOffsetProvider,
Func <DateTime, bool> customStepEvaluator)
{
var stepTimeProvider = new PredicateTimeProvider(_frontierTimeProvider, customStepEvaluator);
return(new FrontierAwareEnumerator(enumerator, stepTimeProvider, tzOffsetProvider));
}
public void ChangeDataNormalizationMode()
{
var tb = new TradeBar
{
Time = new DateTime(2020, 5, 21, 8, 9, 0),
Period = TimeSpan.FromHours(1),
Symbol = Symbols.SPY,
Open = 100,
High = 200,
Low = 300,
Close = 400
};
var factor = 0.5m;
var sumOfDividends = 100m;
var adjustedTb = tb.Clone(tb.IsFillForward).Adjust(factor);
var exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.Utc);
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.Utc, new DateTime(2020, 5, 21), new DateTime(2020, 5, 22));
var emitTimeUtc = offsetProvider.ConvertToUtc(tb.EndTime);
_config.SumOfDividends = sumOfDividends;
var subscriptionData = new PrecalculatedSubscriptionData(
_config,
tb,
adjustedTb,
DataNormalizationMode.Adjusted,
emitTimeUtc);
_config.DataNormalizationMode = DataNormalizationMode.Raw;
Assert.AreEqual(tb.Open, (subscriptionData.Data as TradeBar).Open);
Assert.AreEqual(tb.High, (subscriptionData.Data as TradeBar).High);
Assert.AreEqual(tb.Low, (subscriptionData.Data as TradeBar).Low);
Assert.AreEqual(tb.Close, (subscriptionData.Data as TradeBar).Close);
_config.DataNormalizationMode = DataNormalizationMode.Adjusted;
Assert.AreEqual(tb.Open * factor, (subscriptionData.Data as TradeBar).Open);
Assert.AreEqual(tb.High * factor, (subscriptionData.Data as TradeBar).High);
Assert.AreEqual(tb.Low * factor, (subscriptionData.Data as TradeBar).Low);
Assert.AreEqual(tb.Close * factor, (subscriptionData.Data as TradeBar).Close);
_config.DataNormalizationMode = DataNormalizationMode.TotalReturn;
Assert.Throws <ArgumentException>(() =>
{
var data = subscriptionData.Data;
}
);
_config.DataNormalizationMode = DataNormalizationMode.SplitAdjusted;
Assert.Throws <ArgumentException>(() =>
{
var data = subscriptionData.Data;
}
);
}
/// <summary>
/// Will wrap the provided enumerator with a <see cref="FrontierAwareEnumerator"/>
/// using a <see cref="PredicateTimeProvider"/> that will advance time based on the provided
/// function
/// </summary>
/// <remarks>Won't advance time if now.Hour is bigger or equal than 23pm, less or equal than 5am or Saturday.
/// This is done to prevent universe selection occurring in those hours so that the subscription changes
/// are handled correctly.</remarks>
private IEnumerator <BaseData> GetConfiguredFrontierAwareEnumerator(IEnumerator <BaseData> enumerator,
TimeZoneOffsetProvider tzOffsetProvider)
{
var stepTimeProvider = new PredicateTimeProvider(_frontierTimeProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
return(new FrontierAwareEnumerator(enumerator, stepTimeProvider, tzOffsetProvider));
}
/// <summary>
/// Creates a new instance
/// </summary>
/// <param name="configuration">The subscription's configuration</param>
/// <param name="exchangeHours">The security's exchange hours</param>
/// <param name="offsetProvider">The subscription's time zone offset provider</param>
/// <param name="enumerator">The underlying data enumerator</param>
/// <returns>A subscription data enumerator</returns>
public SubscriptionDataEnumerator(SubscriptionDataConfig configuration,
SecurityExchangeHours exchangeHours,
TimeZoneOffsetProvider offsetProvider,
IEnumerator <BaseData> enumerator)
{
_enumerator = enumerator;
_offsetProvider = offsetProvider;
_exchangeHours = exchangeHours;
_configuration = configuration;
}
/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator)
{
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
Func <int, bool> produce = (workBatchSize) =>
{
try
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.DisposeSafely();
return(false);
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours,
subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if added more data than the work batch size, we don't want to fill the ram
if (count > workBatchSize)
{
return(true);
}
}
}
catch (Exception exception)
{
Log.Error(exception, $"Subscription worker task exception {request.Configuration}.");
}
// we made it here because MoveNext returned false or we exploded, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.DisposeSafely();
return(false);
};
WeightedWorkScheduler.Instance.QueueWork(produce,
// if the subscription finished we return 0, so the work is prioritized and gets removed
() => enqueueable.HasFinished ? 0 : enqueueable.Count);
return(subscription);
}
private Subscription CreateSubscription(QCAlgorithm algorithm, Security security, DateTime startTimeUtc, DateTime endTimeUtc)
{
var universe = algorithm.UniverseManager.Values.OfType <UserDefinedUniverse>()
.Single(u => u.SelectSymbols(default(DateTime), null).Contains(security.Symbol));
var config = security.Subscriptions.First();
var enumerator = DataTradeBarEnumerator(algorithm.StartDate, algorithm.EndDate, Time.OneSecond);
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, startTimeUtc, endTimeUtc);
return(new Subscription(universe, security, config, enumerator, offsetProvider, endTimeUtc, endTimeUtc, false));
}
public void ConvertToUtcAfterDST()
{
// the exact instant DST goes into affect
var tzDate = new DateTime(2015, 03, 08, 2, 0, 0);
var utcDate = tzDate.AddHours(5);
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1));
var result = offsetProvider.ConvertToUtc(tzDate);
// We substract an hour due to the effect of DST
Assert.AreEqual(utcDate - TimeSpan.FromHours(1), result);
}
/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <param name="lowerThreshold">The lower threshold for the worker task, for which the consumer will trigger the worker
/// if it has stopped <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param>
/// <param name="upperThreshold">The upper threshold for the worker task, after which it will stop producing until requested
/// by the consumer <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator,
int lowerThreshold,
int upperThreshold)
{
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
Action produce = () =>
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.Dispose();
return;
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours, subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if we have more data than the upper threshold in the enqueueable, we don't want to fill the ram
if (count > upperThreshold)
{
// we use local count for the outside if, for performance, and adjust here
count = enqueueable.Count;
if (count > upperThreshold)
{
// we will be re scheduled to run by the consumer, see EnqueueableEnumerator
return;
}
}
}
// we made it here because MoveNext returned false, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.Dispose();
};
enqueueable.SetProducer(produce, lowerThreshold);
return(subscription);
}
private TimeZoneOffsetProvider GetTimeZoneOffsetProvider(Symbol symbol)
{
TimeZoneOffsetProvider offsetProvider;
if (!_symbolExchangeTimeZones.TryGetValue(symbol, out offsetProvider))
{
// read the exchange time zone from market-hours-database
var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
_symbolExchangeTimeZones[symbol] = offsetProvider = new TimeZoneOffsetProvider(exchangeTimeZone, TimeProvider.GetUtcNow(), Time.EndOfTime);
}
return(offsetProvider);
}
/// <summary>
/// Creates a subscription to process the history request
/// </summary>
private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history)
{
// data reader expects these values in local times
var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var reader = history.GetEnumerator();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone, start);
}
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
public void ReturnsTrueWhenNextDataIsAheadOfFrontier()
{
var currentTime = new DateTime(2015, 10, 13);
var timeProvider = new ManualTimeProvider(currentTime);
var underlying = new List<Tick>
{
new Tick {Time = currentTime.AddSeconds(1)}
};
var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
Assert.IsTrue(frontierAware.MoveNext());
Assert.IsNull(frontierAware.Current);
}
/// <summary>
/// Creates a new <see cref="Subscription"/> which will directly consume the provided enumerator
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription Create(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator)
{
var exchangeHours = request.Security.Exchange.Hours;
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var dataEnumerator = new SubscriptionDataEnumerator(
request.Configuration,
exchangeHours,
timeZoneOffsetProvider,
enumerator
);
return(new Subscription(request, dataEnumerator, timeZoneOffsetProvider));
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder().GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);
// define our data enumerator
IEnumerator <BaseData> enumerator;
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
.Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
.Select(dt => new Tick {
Time = dt
}).GetEnumerator();
}
else
{
// normal reader for all others
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, false);
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
// only message the user if it's one of their universe types
var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental);
PrimeSubscriptionPump(subscription, messageUser);
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
}
/// <summary>
/// Updates the subscription RealTimePrice if the exchange is open
/// </summary>
/// <param name="subscription">The <see cref="Subscription"/></param>
/// <param name="timeZoneOffsetProvider">The <see cref="TimeZoneOffsetProvider"/> used to convert now into the timezone of the exchange</param>
/// <param name="exchangeHours">The <see cref="SecurityExchangeHours"/> used to determine
/// if the exchange is open and we should update</param>
/// <param name="data">The <see cref="BaseData"/> used to update the real time price</param>
/// <returns>True if the real time price was updated</returns>
protected bool UpdateSubscriptionRealTimePrice(
Subscription subscription,
TimeZoneOffsetProvider timeZoneOffsetProvider,
SecurityExchangeHours exchangeHours,
BaseData data)
{
if (subscription != null &&
exchangeHours.IsOpen(
timeZoneOffsetProvider.ConvertFromUtc(_timeProvider.GetUtcNow()),
subscription.Configuration.ExtendedMarketHours))
{
subscription.RealtimePrice = data.Value;
return(true);
}
return(false);
}
private Subscription CreateSubscription(QCAlgorithm algorithm, Security security, DateTime startTimeUtc, DateTime endTimeUtc, out int dataPointCount)
{
var universe = algorithm.UniverseManager.Values.OfType <UserDefinedUniverse>()
.Single(u => u.SelectSymbols(default(DateTime), null).Contains(security.Symbol));
var config = security.Subscriptions.First();
var data = LinqExtensions.Range(algorithm.StartDate, algorithm.EndDate, c => c + config.Increment).Select(time => new DataPoint
{
Time = time,
EndTime = time + config.Increment
})
.ToList();
var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, startTimeUtc, endTimeUtc);
dataPointCount = data.Count;
return(new Subscription(universe, security, config, data.GetEnumerator(), offsetProvider, endTimeUtc, endTimeUtc, false));
}
public void YieldsDataWhenFrontierPasses()
{
var currentTime = new DateTime(2015, 10, 13);
var timeProvider = new ManualTimeProvider(currentTime);
var underlying = new List<Tick>
{
new Tick {Time = currentTime.AddSeconds(1)}
};
var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1));
var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider);
timeProvider.AdvanceSeconds(1);
Assert.IsTrue(frontierAware.MoveNext());
Assert.IsNotNull(frontierAware.Current);
Assert.AreEqual(underlying[0], frontierAware.Current);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="security">The security to create a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription)
{
Subscription subscription = null;
try
{
var config = security.SubscriptionDataConfig;
var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone);
var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone);
IEnumerator<BaseData> enumerator;
if (config.IsCustomData)
{
// custom data uses backtest readers
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false);
// apply fast forwarding, this is especially important for RemoteFile types that
// can send in large chunks of old, irrelevant data
var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment);
// apply rate limits (1x per increment, max 30 minutes between calls)
// TODO : Pull limits from config file?
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
// add the enumerator to the exchange
_customExchange.AddEnumerator(rateLimit);
// this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider);
_exchange.SetHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
// TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
// finally, make our subscriptions aware of the frontier of the data feed, this will help
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
/// <summary>
/// Main routine for datafeed analysis.
/// </summary>
/// <remarks>This is a hot-thread and should be kept extremely lean. Modify with caution.</remarks>
public void Run()
{
var frontier = DateTime.MaxValue;
try
{
// get a listing of time zones and also the initial frontier in utc
var offsetProviders = new TimeZoneOffsetProvider[Subscriptions.Count];
for (int i = 0; i < Subscriptions.Count; i++)
{
// skip the already finished readers
if (SubscriptionReaders[i].Current == null)
{
continue;
}
offsetProviders[i] = new TimeZoneOffsetProvider(Subscriptions[i].TimeZone, _algorithm.StartDate.AddDays(-1), _algorithm.EndDate.AddDays(1));
// compute the initial frontier time
var currentEndTimeUtc = SubscriptionReaders[i].Current.EndTime.ConvertToUtc(Subscriptions[i].TimeZone);
var endTime = SubscriptionReaders[i].Current.EndTime.Ticks - offsetProviders[i].GetOffsetTicks(currentEndTimeUtc);
if (endTime < frontier.Ticks)
{
frontier = new DateTime(endTime);
}
}
Log.Trace(string.Format("FileSystemDataFeed.Run(): Begin: {0} UTC", frontier));
// continue to loop over each subscription, enqueuing data in time order
while (!_cancellationTokenSource.IsCancellationRequested)
{
var earlyBirdTicks = long.MaxValue;
var data = new Dictionary<int, List<BaseData>>();
for (int i = 0; i < _subscriptions; i++)
{
if (_endOfBridge[i])
{
// skip subscriptions that are finished
continue;
}
var cache = new List<BaseData>();
data[i] = cache;
var enumerator = SubscriptionReaders[i];
var currentOffsetTicks = offsetProviders[i].GetOffsetTicks(frontier);
while (enumerator.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks)
{
// we want bars rounded using their subscription times, we make a clone
// so we don't interfere with the enumerator's internal logic
var clone = enumerator.Current.Clone(enumerator.Current.IsFillForward);
clone.Time = clone.Time.RoundDown(Subscriptions[i].Increment);
cache.Add(clone);
if (!enumerator.MoveNext())
{
Log.Trace("FileSystemDataFeed.Run(): Finished subscription: " + Subscriptions[i].Symbol);
_endOfBridge[i] = true;
break;
}
}
earlyBirdTicks = Math.Min(earlyBirdTicks, enumerator.Current.EndTime.Ticks - currentOffsetTicks);
}
if (earlyBirdTicks == long.MaxValue)
{
// there's no more data to pull off, we're done
break;
}
// enqueue our next time slice and set the frontier for the next
Bridge.Add(new TimeSlice(frontier, data), _cancellationTokenSource.Token);
// never go backwards in time, so take the max between early birds and the current frontier
frontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc);
}
if (!_cancellationTokenSource.IsCancellationRequested)
{
Bridge.CompleteAdding();
}
}
catch (Exception err)
{
Log.Error("FileSystemDataFeed.Run(): Encountered an error: " + err.Message);
}
finally
{
Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", frontier));
LoadingComplete = true;
//Close up all streams:
for (var i = 0; i < Subscriptions.Count; i++)
{
SubscriptionReaders[i].Dispose();
}
Log.Trace("FileSystemDataFeed.Run(): Ending Thread... ");
IsActive = false;
}
}
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution)
{
var config = security.SubscriptionDataConfig;
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return null;
}
// get the map file resolver for this market
var mapFileResolver = MapFileResolver.Empty;
if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return subscription;
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
IEnumerator<BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(dt => new Tick { Time = dt }).GetEnumerator();
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof (CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] {security.Symbol});
var enqueable = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);
return subscription;
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator<BaseData> enumerator;
var userDefined = request.Universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue) return;
var symbol = items.OfType<Symbol>().FirstOrDefault();
if (symbol == null) return;
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
else if (config.Type == typeof (CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol});
var enqueable = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return subscription;
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(universe.Market, config.Symbol, universe.SecurityType);
var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
IEnumerator<BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
.Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
.Select(dt => new Tick { Time = dt }).GetEnumerator();
}
else if (config.Type == typeof (CoarseFundamental))
{
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>>
{
{config.SecurityType, new List<Symbol>{config.Symbol}}
});
var enqueable = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
});
enumerator = enqueable;
}
else
{
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
var source = sourceProvider.GetSource(config, currentLocalDate, true);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(rateLimit);
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
else
{
enqueable.Enqueue(data);
}
});
enumerator = enqueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
return subscription;
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="universe"></param>
/// <param name="security">The security to create a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
{
Subscription subscription = null;
try
{
var config = security.SubscriptionDataConfig;
var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);
IEnumerator<BaseData> enumerator;
if (config.IsCustomData)
{
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseData>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
var source = sourceProvider.GetSource(config, currentLocalDate, true);
var factoryReadEnumerator = factory.Read(source).GetEnumerator();
var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(rateLimit);
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
_exchange.SetHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
// TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
// finally, make our subscriptions aware of the frontier of the data feed, this will help
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator<BaseData> enumerator;
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
.Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
.Select(dt => new Tick {Time = dt}).GetEnumerator();
}
else
{
// normal reader for all others
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, tradeableDates, false);
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
// only message the user if it's one of their universe types
var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental);
PrimeSubscriptionPump(subscription, messageUser);
_subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
}
private Subscription CreateSubscription(SubscriptionRequest request)
{
// ReSharper disable once PossibleMultipleEnumeration
if (!request.TradableDays.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", request.Security.Symbol));
return null;
}
// ReSharper disable once PossibleMultipleEnumeration
var enumeratorFactory = GetEnumeratorFactory(request);
var enumerator = enumeratorFactory.CreateEnumerator(request);
enumerator = ConfigureEnumerator(request, false, enumerator);
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(request.Configuration.Resolution), GetUpperThreshold(request.Configuration.Resolution));
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request.Universe, request.Security, request.Configuration, enqueueable, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
return subscription;
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator<BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataCollectionSubscriptionEnumeratorFactory(r => new [] {dateInDataTimeZone});
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, frontierAware);
var enqueable = new EnqueueableEnumerator<BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// grab the relevant exchange hours
var config = request.Configuration;
// define our data enumerator
var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request);
var firstLoopCount = 5;
var lowerThreshold = GetLowerThreshold(config.Resolution);
var upperThreshold = GetUpperThreshold(config.Resolution);
if (config.Type == typeof (CoarseFundamental))
{
firstLoopCount = 2;
lowerThreshold = 5;
upperThreshold = 100000;
}
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
ScheduleEnumerator(enumerator, enqueueable, lowerThreshold, upperThreshold, firstLoopCount);
enumerator = enqueueable;
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
return new Subscription(request.Universe, request.Security, config, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config);
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator<BaseData> enumerator;
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator();
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else if (config.Type == typeof (CoarseFundamental))
{
var cf = new CoarseFundamental();
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// load coarse data day by day
var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate)
let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date
let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false)
let source = cf.GetSource(config, dateInDataTimeZone, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate);
ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2);
enumerator = enqueueable;
}
else
{
// normal reader for all others
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false);
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
_subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
}
private Subscription CreateSubscription(Universe universe, Security security, DateTime startTimeUtc, DateTime endTimeUtc)
{
var config = security.SubscriptionDataConfig;
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return null;
}
// get the map file resolver for this market
var mapFileResolver = MapFileResolver.Empty;
if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);
// ReSharper disable once PossibleMultipleEnumeration
var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates);
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return subscription;
}
