public void ThrowingEnumeratorStackDisposesOfSubscription() { var enumerator = new TestDataEnumerator { MoveNextTrueCount = 10, ThrowException = true }; var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator); var count = 0; while (enumerator.MoveNextTrueCount != 9) { if (count++ > 100) { Assert.Fail("Timeout waiting for producer"); } Thread.Sleep(1); } Assert.IsFalse(subscription.MoveNext()); Assert.IsTrue(subscription.EndOfStream); Assert.IsTrue(enumerator.Disposed); }
// This unit tests reproduces GH 3885 where the consumer hanged forever public void ConsumerDoesNotHang() { for (var i = 0; i < 10000; i++) { var dataPoints = 10; var enumerator = new TestDataEnumerator { MoveNextTrueCount = dataPoints }; var factorFileProfider = new Mock <IFactorFileProvider>(); factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market)); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator, factorFileProfider.Object, false); for (var j = 0; j < dataPoints; j++) { Assert.IsTrue(subscription.MoveNext()); } Assert.IsFalse(subscription.MoveNext()); subscription.DisposeSafely(); } }
public void SubscriptionIsDisposed() { var dataPoints = 10; var enumerator = new TestDataEnumerator { MoveNextTrueCount = dataPoints }; var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator); var count = 0; while (enumerator.MoveNextTrueCount > 8) { if (count++ > 100) { Assert.Fail($"Timeout waiting for producer. {enumerator.MoveNextTrueCount}"); } Thread.Sleep(1); } subscription.DisposeSafely(); Assert.IsFalse(subscription.MoveNext()); }
// This unit tests reproduces GH 3885 where the consumer hanged forever public void ConsumerDoesNotHang() { for (var i = 0; i < 10000; i++) { var dataPoints = 10; var enumerator = new TestDataEnumerator { MoveNextTrueCount = dataPoints }; var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator); for (var j = 0; j < dataPoints; j++) { Assert.IsTrue(subscription.MoveNext()); } Assert.IsFalse(subscription.MoveNext()); subscription.DisposeSafely(); } }
public void PriceScaleDoesNotUpdateForFillForwardBar() { var referenceTime = new DateTime(2020, 08, 06); var point = new Tick(referenceTime, Symbols.SPY, 1, 2); var point2 = point.Clone(true); point2.Time = referenceTime.AddDays(1); var point3 = point.Clone(false); point3.Time = referenceTime.AddDays(2); ; var enumerator = new List <BaseData> { point, point2, point3 }.GetEnumerator(); var factorFileProfider = new Mock <IFactorFileProvider>(); var factorFile = new FactorFile(_security.Symbol.Value, new[] { new FactorFileRow(referenceTime, 0.5m, 1), new FactorFileRow(referenceTime.AddDays(1), 1m, 1) }, referenceTime); factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(factorFile); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, referenceTime, Time.EndOfTime ), enumerator, factorFileProfider.Object, true); Assert.IsTrue(subscription.MoveNext()); Assert.AreEqual(1, (subscription.Current.Data as Tick).AskPrice); Assert.IsFalse((subscription.Current.Data as Tick).IsFillForward); Assert.IsTrue(subscription.MoveNext()); Assert.AreEqual(1, (subscription.Current.Data as Tick).AskPrice); Assert.IsTrue((subscription.Current.Data as Tick).IsFillForward); Assert.IsTrue(subscription.MoveNext()); Assert.AreEqual(2, (subscription.Current.Data as Tick).AskPrice); Assert.IsFalse((subscription.Current.Data as Tick).IsFillForward); subscription.DisposeSafely(); }
public void SubscriptionEmitsAuxData(Type typeOfConfig, bool shouldReceiveAuxData) { var factorFileProvider = new Mock <IFactorFileProvider>(); var config = new SubscriptionDataConfig(typeOfConfig, _security.Symbol, Resolution.Hour, TimeZones.NewYork, TimeZones.NewYork, true, true, false); factorFileProvider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, config.Market)); var totalPoints = 8; var time = new DateTime(2010, 1, 1); var enumerator = Enumerable.Range(0, totalPoints).Select(x => new Delisting { Time = time.AddHours(x) }).GetEnumerator(); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, config, DateTime.UtcNow, Time.EndOfTime ), enumerator, factorFileProvider.Object, false); // Test our subscription stream to see if it emits the aux data it should be filtered // by the SubscriptionUtils produce function if the config isn't for a TradeBar int dataReceivedCount = 0; while (subscription.MoveNext()) { dataReceivedCount++; if (subscription.Current != null && subscription.Current.Data.DataType == MarketDataType.Auxiliary) { Assert.IsTrue(shouldReceiveAuxData); } } // If it should receive aux data it should have emitted all points // otherwise none should have been emitted if (shouldReceiveAuxData) { Assert.AreEqual(totalPoints, dataReceivedCount); } else { Assert.AreEqual(0, dataReceivedCount); } }
public void ThrowingEnumeratorStackDisposesOfSubscription() { var enumerator = new TestDataEnumerator { MoveNextTrueCount = 10, ThrowException = true }; var factorFileProfider = new Mock <IFactorFileProvider>(); factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market)); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator, factorFileProfider.Object, false); var count = 0; while (enumerator.MoveNextTrueCount != 9) { if (count++ > 100) { Assert.Fail("Timeout waiting for producer"); } Thread.Sleep(1); } Assert.IsFalse(subscription.MoveNext()); Assert.IsTrue(subscription.EndOfStream); // enumerator is disposed by the producer count = 0; while (!enumerator.Disposed) { if (count++ > 100) { Assert.Fail("Timeout waiting for producer"); } Thread.Sleep(1); } }
public void FirstLoopLimit(int firstLoopLimit) { var dataPoints = 10; var enumerator = new TestDataEnumerator { MoveNextTrueCount = dataPoints }; var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator, firstLoopLimit: firstLoopLimit); var count = 0; var expectedValue = dataPoints - firstLoopLimit - 1; expectedValue = expectedValue > 0 ? expectedValue : -1; while (enumerator.MoveNextTrueCount != expectedValue) { if (count++ > 100) { Assert.Fail("Timeout waiting for producer"); } Thread.Sleep(1); } // producer should only have produced 'firstLoopLimit' data points, lets assert remaining to produce Assert.AreEqual(expectedValue, enumerator.MoveNextTrueCount); for (var j = 0; j < dataPoints; j++) { Assert.IsTrue(subscription.MoveNext()); } Assert.IsFalse(subscription.MoveNext()); subscription.DisposeSafely(); Assert.IsTrue(enumerator.Disposed); }
public void SubscriptionIsDisposed() { var dataPoints = 10; var enumerator = new TestDataEnumerator { MoveNextTrueCount = dataPoints }; var factorFileProfider = new Mock <IFactorFileProvider>(); factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market)); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator, factorFileProfider.Object, false); var count = 0; while (enumerator.MoveNextTrueCount > 8) { if (count++ > 100) { Assert.Fail($"Timeout waiting for producer. {enumerator.MoveNextTrueCount}"); } Thread.Sleep(1); } subscription.DisposeSafely(); Assert.IsFalse(subscription.MoveNext()); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc) { // data reader expects these values in local times var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); _dataPermissionManager.AssertConfiguration(config); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var mapFileResolver = MapFileResolver.Empty; if (config.TickerShouldBeMapped()) { mapFileResolver = _mapFileProvider.Get(config.Market); var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol); } // Tradable dates are defined with the data time zone to access the right source var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours); var dataReader = new SubscriptionDataReader(config, startTimeLocal, endTimeLocal, mapFileResolver, _factorFileProvider, tradableDates, false, _dataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, mapFileResolver, false, startTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > endTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > startTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request) { var config = request.ToSubscriptionDataConfig(); DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var dataReader = new SubscriptionDataReader(config, request, _mapFileProvider, _factorFileProvider, _dataCacheProvider, _dataProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, _mapFileProvider, request.StartTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out all aux data if (data.DataType == MarketDataType.Auxiliary) { return(false); } // filter out future data if (data.EndTime > request.EndTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > request.StartTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }