public void EmptyNullEnumerators(bool skipsBasedOnEndTime) { var time = new DateTime(2020, 1, 1); // empty enumerators var enumerator1 = new List <BaseData>().GetEnumerator(); var enumerator2 = new List <BaseData>().GetEnumerator(); var enumerator3 = new List <BaseData> { new Tick(time, Symbols.SPY, 10, 10), new Tick(time.AddSeconds(-1), Symbols.SPY, 20, 20), new Tick(time.AddSeconds(1), Symbols.SPY, 30, 30) }.GetEnumerator(); var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, null, enumerator2, enumerator3); Assert.IsTrue(concat.MoveNext()); Assert.AreEqual(10, (concat.Current as Tick).AskPrice); Assert.IsTrue(concat.MoveNext()); Assert.AreEqual(20, (concat.Current as Tick).AskPrice); Assert.IsTrue(concat.MoveNext()); Assert.AreEqual(30, (concat.Current as Tick).AskPrice); Assert.IsFalse(concat.MoveNext()); Assert.IsNull(concat.Current); concat.Dispose(); }
public void SkipsBasedOnEndTime(bool skipsBasedOnEndTime) { var time = new DateTime(2020, 1, 1); var enumerator1 = new List <BaseData> { new Tick(time, Symbols.SPY, 10, 10) }.GetEnumerator(); var enumerator2 = new List <BaseData> { new Tick(time.AddSeconds(-1), Symbols.SPY, 20, 20), //should be skipped because end time is before previous tick new Tick(time.AddSeconds(1), Symbols.SPY, 30, 30) }.GetEnumerator(); var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, enumerator2); Assert.IsTrue(concat.MoveNext()); Assert.AreEqual(10, (concat.Current as Tick).AskPrice); if (!skipsBasedOnEndTime) { Assert.IsTrue(concat.MoveNext()); Assert.AreEqual(20, (concat.Current as Tick).AskPrice); } Assert.IsTrue(concat.MoveNext()); Assert.AreEqual(30, (concat.Current as Tick).AskPrice); Assert.IsFalse(concat.MoveNext()); Assert.IsNull(concat.Current); concat.Dispose(); }
/// <summary> /// Build and apply the warmup enumerators when required /// </summary> private IEnumerator <BaseData> GetWarmupEnumerator(SubscriptionRequest request, IEnumerator <BaseData> liveEnumerator) { if (_algorithm.IsWarmingUp) { var warmupRequest = new SubscriptionRequest(request, endTimeUtc: _timeProvider.GetUtcNow(), // we will not fill forward each warmup enumerators separately but concatenated bellow configuration: new SubscriptionDataConfig(request.Configuration, fillForward: false)); if (warmupRequest.TradableDays.Any()) { // since we will source data locally and from the history provider, let's limit the history request size // by setting a start date respecting the 'MaximumWarmupHistoryDaysLookBack' var historyWarmup = warmupRequest; var warmupHistoryStartDate = warmupRequest.EndTimeUtc.AddDays(-MaximumWarmupHistoryDaysLookBack); if (warmupHistoryStartDate > warmupRequest.StartTimeUtc) { historyWarmup = new SubscriptionRequest(warmupRequest, startTimeUtc: warmupHistoryStartDate); } var synchronizedWarmupEnumerator = TryAddFillForwardEnumerator(warmupRequest, // we concatenate the file based and history based warmup enumerators, dropping duplicate time stamps new ConcatEnumerator(true, GetFileBasedWarmupEnumerator(warmupRequest), GetHistoryWarmupEnumerator(historyWarmup)) { CanEmitNull = false }, // if required by the original request, we will fill forward the Synced warmup data request.Configuration.FillDataForward); // the order here is important, concat enumerator will keep the last enumerator given and dispose of the rest liveEnumerator = new ConcatEnumerator(true, synchronizedWarmupEnumerator, liveEnumerator); } } return(liveEnumerator); }
protected void FixKeyMap() { _keyToEntry.Clear(); IEnumerator ie = new ConcatEnumerator(_entries.GetEnumerator(), new ConcatEnumerator(_fixedEntries.GetEnumerator(), _macroEntries.GetEnumerator())); while (ie.MoveNext()) { Entry e = (Entry)ie.Current; if (e.Key != Keys.None) { Keys k = e.Key | e.Modifiers; if (!_keyToEntry.ContainsKey(k)) { _keyToEntry.Add(k, e); } } } _keyMapIsDirty = false; }
/// <summary> /// Returns the content of the collection as <see cref="ConcatEnumerator"/>. /// </summary> public IEnumerator <T> Get() { if (this.getters == null) { Debug.LogWarning("Tried getting value event with no getter!"); return(default(IEnumerator <T>)); } // Setup enumerator ConcatEnumerator enumerator = new ConcatEnumerator(); enumerator.Init(); for (int i = 0; i < this.getters.Count; i++) { enumerator.AddCollection(this.getters[i]()); } return(enumerator); }
public void DropsEnumeratorsReturningNullAndTrue(bool skipsBasedOnEndTime) { var enumerator1 = new TestEnumerator(); var enumerator2 = new TestEnumerator(); var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, null, enumerator2); Assert.IsTrue(concat.MoveNext()); Assert.IsNull(concat.Current); Assert.IsTrue(enumerator1.Disposed); Assert.IsFalse(enumerator2.Disposed); Assert.AreEqual(1, enumerator2.MoveNextCount); Assert.IsTrue(concat.MoveNext()); // we assert it just keeps the last enumerator and drops the rest Assert.IsTrue(enumerator1.Disposed); Assert.IsFalse(enumerator2.Disposed); Assert.IsNull(concat.Current); Assert.AreEqual(2, enumerator2.MoveNextCount); concat.Dispose(); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc) { // data reader expects these values in local times var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); _dataPermissionManager.AssertConfiguration(config); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var mapFileResolver = MapFileResolver.Empty; if (config.TickerShouldBeMapped()) { mapFileResolver = _mapFileProvider.Get(config.Market); var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol); } // Tradable dates are defined with the data time zone to access the right source var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours); var dataReader = new SubscriptionDataReader(config, startTimeLocal, endTimeLocal, mapFileResolver, _factorFileProvider, tradableDates, false, _dataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, mapFileResolver, false, startTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > endTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > startTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request) { var config = request.ToSubscriptionDataConfig(); DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var dataReader = new SubscriptionDataReader(config, request, _mapFileProvider, _factorFileProvider, _dataCacheProvider, _dataProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, _mapFileProvider, request.StartTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out all aux data if (data.DataType == MarketDataType.Auxiliary) { return(false); } // filter out future data if (data.EndTime > request.EndTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > request.StartTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }