private Subscription CreateDataSubscription(SubscriptionRequest request) { // ReSharper disable once PossibleMultipleEnumeration if (!request.TradableDays.Any()) { _algorithm.Error( $"No data loaded for {request.Security.Symbol} because there were no tradeable dates for this security." ); return(null); } // ReSharper disable once PossibleMultipleEnumeration var enumeratorFactory = GetEnumeratorFactory(request); var enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = ConfigureEnumerator(request, false, enumerator); var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider); // add this enumerator to our exchange ScheduleEnumerator(subscription, enumerator, enqueueable, GetLowerThreshold(request.Configuration.Resolution), GetUpperThreshold(request.Configuration.Resolution), request.Security.Exchange.Hours); return(subscription); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // grab the relevant exchange hours var config = request.Configuration; // define our data enumerator var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request, _dataProvider); var firstLoopCount = 5; var lowerThreshold = GetLowerThreshold(config.Resolution); var upperThreshold = GetUpperThreshold(config.Resolution); if (config.Type == typeof(CoarseFundamental)) { firstLoopCount = 2; // the lower threshold will be when we start the worker again, if he is stopped lowerThreshold = 200; // the upper threshold will stop the worker from loading more data. This is roughly 1 GB upperThreshold = 500; } var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider); // add this enumerator to our exchange ScheduleEnumerator(subscription, enumerator, enqueueable, lowerThreshold, upperThreshold, request.Security.Exchange.Hours, firstLoopCount); return(subscription); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // grab the relevant exchange hours var config = request.Configuration; // define our data enumerator var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request, _dataProvider); var firstLoopCount = 5; var lowerThreshold = GetLowerThreshold(config.Resolution); var upperThreshold = GetUpperThreshold(config.Resolution); if (config.Type == typeof(CoarseFundamental)) { firstLoopCount = 2; lowerThreshold = 5; upperThreshold = 100000; } var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); var subscription = new Subscription(request.Universe, request.Security, config, enqueueable, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); // add this enumerator to our exchange ScheduleEnumerator(subscription, enumerator, enqueueable, lowerThreshold, upperThreshold, firstLoopCount); return(subscription); }
private Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime startTimeUtc, DateTime endTimeUtc) { var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return(null); } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) { mapFileResolver = _mapFileProvider.Get(config.Market); } // ReSharper disable once PossibleMultipleEnumeration var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates); var enqueueable = new EnqueueableEnumerator <BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return(subscription); }
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef <TimeSpan> fillForwardResolution) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return(null); } // ReSharper disable once PossibleMultipleEnumeration IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return(subscription); }
public void YieldsFutureDataAtCorrectTime() { var currentTime = new DateTime(2015, 10, 13); var timeProvider = new ManualTimeProvider(currentTime); var underlying = new List<Tick> { new Tick {Time = currentTime.AddSeconds(10)} }; var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1)); var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider); for (int i = 0; i < 10; i++) { timeProvider.AdvanceSeconds(1); Assert.IsTrue(frontierAware.MoveNext()); if (i < 9) { Assert.IsNull(frontierAware.Current); } else { Assert.IsNotNull(frontierAware.Current); Assert.AreEqual(underlying[0], frontierAware.Current); } } }
public bool UpdateRealTimePrice( Subscription subscription, TimeZoneOffsetProvider timeZoneOffsetProvider, SecurityExchangeHours exchangeHours) { return(UpdateSubscriptionRealTimePrice(subscription, timeZoneOffsetProvider, exchangeHours, new Tick())); }
/// <summary> /// Creates a subscription to process the history request /// </summary> private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history) { // data reader expects these values in local times var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.TimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, null, true, request.DataNormalizationMode ); var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency)); var reader = history.GetEnumerator(); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment); } var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end); return(new Subscription(null, security, config, reader, timeZoneOffsetProvider, start, end, false)); }
public void CreatedSubscriptionDoesNotRoundDownForPeriodLessData() { var data = new MyCustomData { Time = new DateTime(2020, 5, 21, 8, 9, 0), Symbol = Symbols.SPY }; var config = new SubscriptionDataConfig( typeof(TradeBar), Symbols.SPY, Resolution.Hour, TimeZones.Utc, TimeZones.Utc, false, false, false ); var exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.Utc); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.Utc, new DateTime(2020, 5, 21), new DateTime(2020, 5, 22)); var subscription = SubscriptionData.Create(config, exchangeHours, offsetProvider, data); Assert.AreEqual(new DateTime(2020, 5, 21, 8, 9, 0), subscription.Data.Time); Assert.AreEqual(new DateTime(2020, 5, 21, 8, 9, 0), subscription.Data.EndTime); }
/// <summary> /// Creates an enumerator to read the specified request. /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var config = request.Configuration; // frontier value used to prevent emitting duplicate time stamps between refreshed enumerators // also provides some immediate fast-forward to handle spooling through remote files quickly var frontier = Ref.Create(DateTime.MinValue); var lastSourceRefreshTime = DateTime.MinValue; var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type }); // this is refreshing the enumerator stack for each new source var refresher = new RefreshEnumerator <BaseData>(() => { // rate limit the refresh of this enumerator stack var utcNow = _timeProvider.GetUtcNow(); var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment); if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls) { return(Enumerable.Empty <BaseData>().GetEnumerator()); } lastSourceRefreshTime = utcNow; var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date; var source = sourceFactory.GetSource(config, localDate, true); // fetch the new source and enumerate the data source reader var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate); if (SourceRequiresFastForward(source)) { // apply fast forward logic for file transport mediums var maximumDataAge = GetMaximumDataAge(config.Increment); enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge); } else { // rate limit calls to this enumerator stack enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls); } if (source.Format == FileFormat.Collection) { // unroll collections into individual data points after fast forward/rate limiting applied enumerator = enumerator.SelectMany(data => { var collection = data as BaseDataCollection; return(collection?.Data.GetEnumerator() ?? new List <BaseData> { data }.GetEnumerator()); }); } return(enumerator); }); // prevent calls to the enumerator stack if current is in the future var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); return(new FrontierAwareEnumerator(refresher, _timeProvider, timeZoneOffsetProvider)); }
public void CreatedSubscriptionRoundsTimeDownForDataWithPeriod() { var tb = new TradeBar { Time = new DateTime(2020, 5, 21, 8, 9, 0), Period = TimeSpan.FromHours(1), Symbol = Symbols.SPY }; var config = new SubscriptionDataConfig( typeof(TradeBar), Symbols.SPY, Resolution.Hour, TimeZones.Utc, TimeZones.Utc, false, false, false ); var exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.Utc); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.Utc, new DateTime(2020, 5, 21), new DateTime(2020, 5, 22)); var subscription = SubscriptionData.Create(config, exchangeHours, offsetProvider, tb); Assert.AreEqual(new DateTime(2020, 5, 21, 8, 0, 0), subscription.Data.Time); Assert.AreEqual(new DateTime(2020, 5, 21, 9, 0, 0), subscription.Data.EndTime); }
public void ReturnsCurrentOffset() { var utcDate = new DateTime(2015, 07, 07); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1)); var currentOffset = offsetProvider.GetOffsetTicks(utcDate); Assert.AreEqual(-TimeSpan.FromHours(4).TotalHours, TimeSpan.FromTicks(currentOffset).TotalHours); }
public void YieldsFutureDataAtCorrectTime() { var currentTime = new DateTime(2015, 10, 13); var timeProvider = new ManualTimeProvider(currentTime); var underlying = new List <Tick> { new Tick { Time = currentTime.AddSeconds(10) } }; var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1)); var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider); for (int i = 0; i < 10; i++) { timeProvider.AdvanceSeconds(1); Assert.IsTrue(frontierAware.MoveNext()); if (i < 9) { Assert.IsNull(frontierAware.Current); } else { Assert.IsNotNull(frontierAware.Current); Assert.AreEqual(underlying[0], frontierAware.Current); } } }
public Subscription CreateSubscription(SubscriptionRequest request) { var offsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc); return(new Subscription(request, _dummyData.GetEnumerator(), offsetProvider)); }
public void ReturnsCurrentOffset() { var utcDate = new DateTime(2015, 07, 07); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1)); var currentOffset = offsetProvider.GetOffsetTicks(utcDate); Assert.AreEqual(-TimeSpan.FromHours(4).TotalHours, TimeSpan.FromTicks(currentOffset).TotalHours); }
public void ReturnsCorrectOffsetAfterDST() { // the exact instant DST goes into affect var utcDate = new DateTime(2015, 03, 08, 2, 0, 0).AddHours(5); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1)); var currentOffset = offsetProvider.GetOffsetTicks(utcDate); Assert.AreEqual(-TimeSpan.FromHours(4).TotalHours, TimeSpan.FromTicks(currentOffset).TotalHours); }
private Subscription CreateSubscription(Resolution resolution, string symbol = "AAPL", bool isInternalFeed = false, SecurityType type = SecurityType.Equity, TickType tickType = TickType.Trade) { var start = DateTime.UtcNow; var end = start.AddSeconds(10); Security security; Symbol _symbol; if (type == SecurityType.Equity) { _symbol = new Symbol(SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA), symbol); security = new Equity( _symbol, SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc), new Cash(Currencies.USD, 0, 1), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null ); } else if (type == SecurityType.Option) { _symbol = new Symbol(SecurityIdentifier.GenerateOption(DateTime.Now, SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA), Market.USA, 0.0m, OptionRight.Call, OptionStyle.American), symbol); security = new Option( _symbol, SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc), new Cash(Currencies.USD, 0, 1), new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null ); } else if (type == SecurityType.Future) { _symbol = new Symbol(SecurityIdentifier.GenerateFuture(DateTime.Now, symbol, Market.USA), symbol); security = new Future( _symbol, SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc), new Cash(Currencies.USD, 0, 1), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null ); } else { throw new Exception("SecurityType not implemented"); } var config = new SubscriptionDataConfig(typeof(TradeBar), _symbol, resolution, DateTimeZone.Utc, DateTimeZone.Utc, true, false, isInternalFeed, false, tickType); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, start, end); var enumerator = new EnqueueableEnumerator <BaseData>(); var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, enumerator); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end); return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider)); }
/// <summary> /// Initializes a new instance of the <see cref="Subscription"/> class /// </summary> /// <param name="security">The security this subscription is for</param> /// <param name="enumerator">The subscription's data source</param> /// <param name="utcStartTime">The start time of the subscription</param> /// <param name="utcEndTime">The end time of the subscription</param> /// <param name="isUserDefined">True if the user explicitly defined this subscription, false otherwise</param> /// <param name="isFundamentalSubscription">True if this subscription is used to define the times to perform universe selection /// for a specific market, false for all other subscriptions</param> public Subscription(Security security, IEnumerator <BaseData> enumerator, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefined, bool isFundamentalSubscription) { Security = security; _enumerator = enumerator; IsUserDefined = isUserDefined; IsFundamentalSubscription = isFundamentalSubscription; Configuration = security.SubscriptionDataConfig; OffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime); }
public void ReturnsCorrectOffsetAfterDST() { // the exact instant DST goes into affect var utcDate = new DateTime(2015, 03, 08, 2, 0, 0).AddHours(5); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1)); var currentOffset = offsetProvider.GetOffsetTicks(utcDate); Assert.AreEqual(-TimeSpan.FromHours(4).TotalHours, TimeSpan.FromTicks(currentOffset).TotalHours); }
/// <summary> /// Will wrap the provided enumerator with a <see cref="FrontierAwareEnumerator"/> /// using a <see cref="PredicateTimeProvider"/> that will advance time based on the provided /// function /// </summary> /// <remarks>Won't advance time if now.Hour is bigger or equal than 23pm, less or equal than 5am or Saturday. /// This is done to prevent universe selection occurring in those hours so that the subscription changes /// are handled correctly.</remarks> private IEnumerator <BaseData> GetConfiguredFrontierAwareEnumerator( IEnumerator <BaseData> enumerator, TimeZoneOffsetProvider tzOffsetProvider, Func <DateTime, bool> customStepEvaluator) { var stepTimeProvider = new PredicateTimeProvider(_frontierTimeProvider, customStepEvaluator); return(new FrontierAwareEnumerator(enumerator, stepTimeProvider, tzOffsetProvider)); }
public void ChangeDataNormalizationMode() { var tb = new TradeBar { Time = new DateTime(2020, 5, 21, 8, 9, 0), Period = TimeSpan.FromHours(1), Symbol = Symbols.SPY, Open = 100, High = 200, Low = 300, Close = 400 }; var factor = 0.5m; var sumOfDividends = 100m; var adjustedTb = tb.Clone(tb.IsFillForward).Adjust(factor); var exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.Utc); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.Utc, new DateTime(2020, 5, 21), new DateTime(2020, 5, 22)); var emitTimeUtc = offsetProvider.ConvertToUtc(tb.EndTime); _config.SumOfDividends = sumOfDividends; var subscriptionData = new PrecalculatedSubscriptionData( _config, tb, adjustedTb, DataNormalizationMode.Adjusted, emitTimeUtc); _config.DataNormalizationMode = DataNormalizationMode.Raw; Assert.AreEqual(tb.Open, (subscriptionData.Data as TradeBar).Open); Assert.AreEqual(tb.High, (subscriptionData.Data as TradeBar).High); Assert.AreEqual(tb.Low, (subscriptionData.Data as TradeBar).Low); Assert.AreEqual(tb.Close, (subscriptionData.Data as TradeBar).Close); _config.DataNormalizationMode = DataNormalizationMode.Adjusted; Assert.AreEqual(tb.Open * factor, (subscriptionData.Data as TradeBar).Open); Assert.AreEqual(tb.High * factor, (subscriptionData.Data as TradeBar).High); Assert.AreEqual(tb.Low * factor, (subscriptionData.Data as TradeBar).Low); Assert.AreEqual(tb.Close * factor, (subscriptionData.Data as TradeBar).Close); _config.DataNormalizationMode = DataNormalizationMode.TotalReturn; Assert.Throws <ArgumentException>(() => { var data = subscriptionData.Data; } ); _config.DataNormalizationMode = DataNormalizationMode.SplitAdjusted; Assert.Throws <ArgumentException>(() => { var data = subscriptionData.Data; } ); }
/// <summary> /// Will wrap the provided enumerator with a <see cref="FrontierAwareEnumerator"/> /// using a <see cref="PredicateTimeProvider"/> that will advance time based on the provided /// function /// </summary> /// <remarks>Won't advance time if now.Hour is bigger or equal than 23pm, less or equal than 5am or Saturday. /// This is done to prevent universe selection occurring in those hours so that the subscription changes /// are handled correctly.</remarks> private IEnumerator <BaseData> GetConfiguredFrontierAwareEnumerator(IEnumerator <BaseData> enumerator, TimeZoneOffsetProvider tzOffsetProvider) { var stepTimeProvider = new PredicateTimeProvider(_frontierTimeProvider, // advance time if before 23pm or after 5am and not on Saturdays time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday); return(new FrontierAwareEnumerator(enumerator, stepTimeProvider, tzOffsetProvider)); }
/// <summary> /// Creates a new instance /// </summary> /// <param name="configuration">The subscription's configuration</param> /// <param name="exchangeHours">The security's exchange hours</param> /// <param name="offsetProvider">The subscription's time zone offset provider</param> /// <param name="enumerator">The underlying data enumerator</param> /// <returns>A subscription data enumerator</returns> public SubscriptionDataEnumerator(SubscriptionDataConfig configuration, SecurityExchangeHours exchangeHours, TimeZoneOffsetProvider offsetProvider, IEnumerator <BaseData> enumerator) { _enumerator = enumerator; _offsetProvider = offsetProvider; _exchangeHours = exchangeHours; _configuration = configuration; }
/// <summary> /// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/> /// that will be feed by a worker task /// </summary> /// <param name="request">The subscription data request</param> /// <param name="enumerator">The data enumerator stack</param> /// <returns>A new subscription instance ready to consume</returns> public static Subscription CreateAndScheduleWorker( SubscriptionRequest request, IEnumerator <BaseData> enumerator) { var exchangeHours = request.Security.Exchange.Hours; var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider); Func <int, bool> produce = (workBatchSize) => { try { var count = 0; while (enumerator.MoveNext()) { // subscription has been removed, no need to continue enumerating if (enqueueable.HasFinished) { enumerator.DisposeSafely(); return(false); } var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours, subscription.OffsetProvider, enumerator.Current); // drop the data into the back of the enqueueable enqueueable.Enqueue(subscriptionData); count++; // stop executing if added more data than the work batch size, we don't want to fill the ram if (count > workBatchSize) { return(true); } } } catch (Exception exception) { Log.Error(exception, $"Subscription worker task exception {request.Configuration}."); } // we made it here because MoveNext returned false or we exploded, stop the enqueueable enqueueable.Stop(); // we have to dispose of the enumerator enumerator.DisposeSafely(); return(false); }; WeightedWorkScheduler.Instance.QueueWork(produce, // if the subscription finished we return 0, so the work is prioritized and gets removed () => enqueueable.HasFinished ? 0 : enqueueable.Count); return(subscription); }
private Subscription CreateSubscription(QCAlgorithm algorithm, Security security, DateTime startTimeUtc, DateTime endTimeUtc) { var universe = algorithm.UniverseManager.Values.OfType <UserDefinedUniverse>() .Single(u => u.SelectSymbols(default(DateTime), null).Contains(security.Symbol)); var config = security.Subscriptions.First(); var enumerator = DataTradeBarEnumerator(algorithm.StartDate, algorithm.EndDate, Time.OneSecond); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, startTimeUtc, endTimeUtc); return(new Subscription(universe, security, config, enumerator, offsetProvider, endTimeUtc, endTimeUtc, false)); }
public void ConvertToUtcAfterDST() { // the exact instant DST goes into affect var tzDate = new DateTime(2015, 03, 08, 2, 0, 0); var utcDate = tzDate.AddHours(5); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, utcDate, utcDate.AddDays(1)); var result = offsetProvider.ConvertToUtc(tzDate); // We substract an hour due to the effect of DST Assert.AreEqual(utcDate - TimeSpan.FromHours(1), result); }
/// <summary> /// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/> /// that will be feed by a worker task /// </summary> /// <param name="request">The subscription data request</param> /// <param name="enumerator">The data enumerator stack</param> /// <param name="lowerThreshold">The lower threshold for the worker task, for which the consumer will trigger the worker /// if it has stopped <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param> /// <param name="upperThreshold">The upper threshold for the worker task, after which it will stop producing until requested /// by the consumer <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param> /// <returns>A new subscription instance ready to consume</returns> public static Subscription CreateAndScheduleWorker( SubscriptionRequest request, IEnumerator <BaseData> enumerator, int lowerThreshold, int upperThreshold) { var exchangeHours = request.Security.Exchange.Hours; var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider); Action produce = () => { var count = 0; while (enumerator.MoveNext()) { // subscription has been removed, no need to continue enumerating if (enqueueable.HasFinished) { enumerator.Dispose(); return; } var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours, subscription.OffsetProvider, enumerator.Current); // drop the data into the back of the enqueueable enqueueable.Enqueue(subscriptionData); count++; // stop executing if we have more data than the upper threshold in the enqueueable, we don't want to fill the ram if (count > upperThreshold) { // we use local count for the outside if, for performance, and adjust here count = enqueueable.Count; if (count > upperThreshold) { // we will be re scheduled to run by the consumer, see EnqueueableEnumerator return; } } } // we made it here because MoveNext returned false, stop the enqueueable enqueueable.Stop(); // we have to dispose of the enumerator enumerator.Dispose(); }; enqueueable.SetProducer(produce, lowerThreshold); return(subscription); }
private TimeZoneOffsetProvider GetTimeZoneOffsetProvider(Symbol symbol) { TimeZoneOffsetProvider offsetProvider; if (!_symbolExchangeTimeZones.TryGetValue(symbol, out offsetProvider)) { // read the exchange time zone from market-hours-database var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; _symbolExchangeTimeZones[symbol] = offsetProvider = new TimeZoneOffsetProvider(exchangeTimeZone, TimeProvider.GetUtcNow(), Time.EndOfTime); } return(offsetProvider); }
/// <summary> /// Creates a subscription to process the history request /// </summary> private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history) { // data reader expects these values in local times var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var reader = history.GetEnumerator(); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone, start); } var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end); var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, reader); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end); return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider)); }
public void ReturnsTrueWhenNextDataIsAheadOfFrontier() { var currentTime = new DateTime(2015, 10, 13); var timeProvider = new ManualTimeProvider(currentTime); var underlying = new List<Tick> { new Tick {Time = currentTime.AddSeconds(1)} }; var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1)); var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider); Assert.IsTrue(frontierAware.MoveNext()); Assert.IsNull(frontierAware.Current); }
/// <summary> /// Creates a new <see cref="Subscription"/> which will directly consume the provided enumerator /// </summary> /// <param name="request">The subscription data request</param> /// <param name="enumerator">The data enumerator stack</param> /// <returns>A new subscription instance ready to consume</returns> public static Subscription Create( SubscriptionRequest request, IEnumerator <BaseData> enumerator) { var exchangeHours = request.Security.Exchange.Hours; var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); var dataEnumerator = new SubscriptionDataEnumerator( request.Configuration, exchangeHours, timeZoneOffsetProvider, enumerator ); return(new Subscription(request, dataEnumerator, timeZoneOffsetProvider)); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder().GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); // define our data enumerator IEnumerator <BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick { Time = dt }).GetEnumerator(); } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, false); } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); // only message the user if it's one of their universe types var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental); PrimeSubscriptionPump(subscription, messageUser); _subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription); }
/// <summary> /// Updates the subscription RealTimePrice if the exchange is open /// </summary> /// <param name="subscription">The <see cref="Subscription"/></param> /// <param name="timeZoneOffsetProvider">The <see cref="TimeZoneOffsetProvider"/> used to convert now into the timezone of the exchange</param> /// <param name="exchangeHours">The <see cref="SecurityExchangeHours"/> used to determine /// if the exchange is open and we should update</param> /// <param name="data">The <see cref="BaseData"/> used to update the real time price</param> /// <returns>True if the real time price was updated</returns> protected bool UpdateSubscriptionRealTimePrice( Subscription subscription, TimeZoneOffsetProvider timeZoneOffsetProvider, SecurityExchangeHours exchangeHours, BaseData data) { if (subscription != null && exchangeHours.IsOpen( timeZoneOffsetProvider.ConvertFromUtc(_timeProvider.GetUtcNow()), subscription.Configuration.ExtendedMarketHours)) { subscription.RealtimePrice = data.Value; return(true); } return(false); }
private Subscription CreateSubscription(QCAlgorithm algorithm, Security security, DateTime startTimeUtc, DateTime endTimeUtc, out int dataPointCount) { var universe = algorithm.UniverseManager.Values.OfType <UserDefinedUniverse>() .Single(u => u.SelectSymbols(default(DateTime), null).Contains(security.Symbol)); var config = security.Subscriptions.First(); var data = LinqExtensions.Range(algorithm.StartDate, algorithm.EndDate, c => c + config.Increment).Select(time => new DataPoint { Time = time, EndTime = time + config.Increment }) .ToList(); var offsetProvider = new TimeZoneOffsetProvider(TimeZones.NewYork, startTimeUtc, endTimeUtc); dataPointCount = data.Count; return(new Subscription(universe, security, config, data.GetEnumerator(), offsetProvider, endTimeUtc, endTimeUtc, false)); }
public void YieldsDataWhenFrontierPasses() { var currentTime = new DateTime(2015, 10, 13); var timeProvider = new ManualTimeProvider(currentTime); var underlying = new List<Tick> { new Tick {Time = currentTime.AddSeconds(1)} }; var offsetProvider = new TimeZoneOffsetProvider(DateTimeZone.Utc, new DateTime(2015, 1, 1), new DateTime(2016, 1, 1)); var frontierAware = new FrontierAwareEnumerator(underlying.GetEnumerator(), timeProvider, offsetProvider); timeProvider.AdvanceSeconds(1); Assert.IsTrue(frontierAware.MoveNext()); Assert.IsNotNull(frontierAware.Current); Assert.AreEqual(underlying[0], frontierAware.Current); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone); var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // custom data uses backtest readers var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false); // apply fast forwarding, this is especially important for RemoteFile types that // can send in large chunks of old, irrelevant data var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment); // apply rate limits (1x per increment, max 30 minutes between calls) // TODO : Pull limits from config file? var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); // add the enumerator to the exchange _customExchange.AddEnumerator(rateLimit); // this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Main routine for datafeed analysis. /// </summary> /// <remarks>This is a hot-thread and should be kept extremely lean. Modify with caution.</remarks> public void Run() { var frontier = DateTime.MaxValue; try { // get a listing of time zones and also the initial frontier in utc var offsetProviders = new TimeZoneOffsetProvider[Subscriptions.Count]; for (int i = 0; i < Subscriptions.Count; i++) { // skip the already finished readers if (SubscriptionReaders[i].Current == null) { continue; } offsetProviders[i] = new TimeZoneOffsetProvider(Subscriptions[i].TimeZone, _algorithm.StartDate.AddDays(-1), _algorithm.EndDate.AddDays(1)); // compute the initial frontier time var currentEndTimeUtc = SubscriptionReaders[i].Current.EndTime.ConvertToUtc(Subscriptions[i].TimeZone); var endTime = SubscriptionReaders[i].Current.EndTime.Ticks - offsetProviders[i].GetOffsetTicks(currentEndTimeUtc); if (endTime < frontier.Ticks) { frontier = new DateTime(endTime); } } Log.Trace(string.Format("FileSystemDataFeed.Run(): Begin: {0} UTC", frontier)); // continue to loop over each subscription, enqueuing data in time order while (!_cancellationTokenSource.IsCancellationRequested) { var earlyBirdTicks = long.MaxValue; var data = new Dictionary<int, List<BaseData>>(); for (int i = 0; i < _subscriptions; i++) { if (_endOfBridge[i]) { // skip subscriptions that are finished continue; } var cache = new List<BaseData>(); data[i] = cache; var enumerator = SubscriptionReaders[i]; var currentOffsetTicks = offsetProviders[i].GetOffsetTicks(frontier); while (enumerator.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks) { // we want bars rounded using their subscription times, we make a clone // so we don't interfere with the enumerator's internal logic var clone = enumerator.Current.Clone(enumerator.Current.IsFillForward); clone.Time = clone.Time.RoundDown(Subscriptions[i].Increment); cache.Add(clone); if (!enumerator.MoveNext()) { Log.Trace("FileSystemDataFeed.Run(): Finished subscription: " + Subscriptions[i].Symbol); _endOfBridge[i] = true; break; } } earlyBirdTicks = Math.Min(earlyBirdTicks, enumerator.Current.EndTime.Ticks - currentOffsetTicks); } if (earlyBirdTicks == long.MaxValue) { // there's no more data to pull off, we're done break; } // enqueue our next time slice and set the frontier for the next Bridge.Add(new TimeSlice(frontier, data), _cancellationTokenSource.Token); // never go backwards in time, so take the max between early birds and the current frontier frontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc); } if (!_cancellationTokenSource.IsCancellationRequested) { Bridge.CompleteAdding(); } } catch (Exception err) { Log.Error("FileSystemDataFeed.Run(): Encountered an error: " + err.Message); } finally { Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", frontier)); LoadingComplete = true; //Close up all streams: for (var i = 0; i < Subscriptions.Count; i++) { SubscriptionReaders[i].Dispose(); } Log.Trace("FileSystemDataFeed.Run(): Ending Thread... "); IsActive = false; } }
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return null; } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market); // ReSharper disable once PossibleMultipleEnumeration IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return subscription; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); IEnumerator<BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(dt => new Tick { Time = dt }).GetEnumerator(); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } else if (config.Type == typeof (CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] {security.Symbol}); var enqueable = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true); return subscription; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator<BaseData> enumerator; var userDefined = request.Universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) return; var symbol = items.OfType<Symbol>().FirstOrDefault(); if (symbol == null) return; var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; } else if (config.Type == typeof (CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol}); var enqueable = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); return subscription; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(universe.Market, config.Symbol, universe.SecurityType); var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator<BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick { Time = dt }).GetEnumerator(); } else if (config.Type == typeof (CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>> { {config.SecurityType, new List<Symbol>{config.Symbol}} }); var enqueable = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); return subscription; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge); return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator<BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick {Time = dt}).GetEnumerator(); } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, tradeableDates, false); } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); // only message the user if it's one of their universe types var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental); PrimeSubscriptionPump(subscription, messageUser); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); }
private Subscription CreateSubscription(SubscriptionRequest request) { // ReSharper disable once PossibleMultipleEnumeration if (!request.TradableDays.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", request.Security.Symbol)); return null; } // ReSharper disable once PossibleMultipleEnumeration var enumeratorFactory = GetEnumeratorFactory(request); var enumerator = enumeratorFactory.CreateEnumerator(request); enumerator = ConfigureEnumerator(request, false, enumerator); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(request.Configuration.Resolution), GetUpperThreshold(request.Configuration.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); var subscription = new Subscription(request.Universe, request.Security, request.Configuration, enqueueable, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false); return subscription; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator<BaseData> enumerator; if (request.Configuration.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date; var enumeratorFactory = new BaseDataCollectionSubscriptionEnumeratorFactory(r => new [] {dateInDataTimeZone}); var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request); var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider); _customExchange.AddEnumerator(request.Configuration.Symbol, frontierAware); var enqueable = new EnqueueableEnumerator<BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (request.Configuration.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(request.Configuration.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (request.Configuration.FillDataForward) { enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // grab the relevant exchange hours var config = request.Configuration; // define our data enumerator var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request); var firstLoopCount = 5; var lowerThreshold = GetLowerThreshold(config.Resolution); var upperThreshold = GetUpperThreshold(config.Resolution); if (config.Type == typeof (CoarseFundamental)) { firstLoopCount = 2; lowerThreshold = 5; upperThreshold = 100000; } var enqueueable = new EnqueueableEnumerator<BaseData>(true); ScheduleEnumerator(enumerator, enqueueable, lowerThreshold, upperThreshold, firstLoopCount); enumerator = enqueueable; // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); return new Subscription(request.Universe, request.Security, config, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config); var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator<BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator(); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } else if (config.Type == typeof (CoarseFundamental)) { var cf = new CoarseFundamental(); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // load coarse data day by day var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate) let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false) let source = cf.GetSource(config, dateInDataTimeZone, false) let coarseFundamentalForDate = factory.Read(source) select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate); ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2); enumerator = enqueueable; } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); }
private Subscription CreateSubscription(Universe universe, Security security, DateTime startTimeUtc, DateTime endTimeUtc) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return null; } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market); // ReSharper disable once PossibleMultipleEnumeration var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return subscription; }