public void FillForwardHoursAtEndOfDayByHalfHour() { var dataResolution = Time.OneHour; var reference = new DateTime(2015, 6, 25, 14, 0, 0); var data = new BaseData[] { // thurs 6/25 new TradeBar { Value = 0, Time = reference, Period = dataResolution }, // fri 6/26 new TradeBar { Value = 1, Time = reference.Date.AddDays(1), Period = dataResolution }, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var ffResolution = TimeSpan.FromMinutes(30); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(ffResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution); // 3:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); // 3:30 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(1.5), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); // 4:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); // 12:00am Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(data.Last().EndTime, fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
public void FillsForwardGapBeforeEndOfSubscription() { var dataResolution = Time.OneMinute; var reference = new DateTime(2015, 6, 25, 15, 57, 0); var data = new[] { new TradeBar { Time = reference, Value = 0, Period = dataResolution, Volume = 100 } }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); var isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromMinutes(1)), isExtendedMarketHours, reference.Date.AddHours(16), dataResolution, exchange.TimeZone, data.First().EndTime); // 3:58 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume); // 3:39 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); // 4:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); Assert.IsFalse(fillForwardEnumerator.MoveNext()); fillForwardEnumerator.Dispose(); }
public void HandlesDaylightSavingTimeChange() { var dailyBarsEmitted = 0; var fillForwardBars = new List <BaseData>(); // 3 QuoteBars as they would be read from the EURUSD oanda daily file by QuoteBar.Reader() // The conversion from dataTimeZone to exchangeTimeZone has been done by hand // dataTimeZone == UTC /* * 20180311 00:00,1.2308,1.2308,1.2308,1.2308,0,1.23096,1.23096,1.23096,1.23096,0 * 20180312 00:00,1.23082,1.23449,1.22898,1.23382,0,1.23097,1.23463,1.22911,1.23396,0 */ var data = new BaseData[] { // Sunday 3/11 new QuoteBar { Value = 0, Time = new DateTime(2018, 3, 10, 19, 0, 0), Period = Time.OneDay }, // Monday 3/12 new QuoteBar { Value = 1, Time = new DateTime(2018, 3, 11, 20, 0, 0), Period = Time.OneDay }, }.ToList(); var enumerator = data.GetEnumerator(); var market = Market.Oanda; var symbol = Symbol.Create("EURUSD", SecurityType.Forex, market); var marketHours = MarketHoursDatabase.FromDataFolder(); var exchange = new ForexExchange(marketHours.GetExchangeHours(market, symbol, SecurityType.Forex)); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromDays(1)), false, data.Last().EndTime, Time.OneDay, TimeZones.Utc, data.First().EndTime); while (fillForwardEnumerator.MoveNext()) { fillForwardBars.Add(fillForwardEnumerator.Current); Console.WriteLine(fillForwardEnumerator.Current.Time.DayOfWeek + " " + fillForwardEnumerator.Current.Time + " - " + fillForwardEnumerator.Current.EndTime.DayOfWeek + " " + fillForwardEnumerator.Current.EndTime); dailyBarsEmitted++; } Assert.AreEqual(2, dailyBarsEmitted); Assert.AreEqual(new DateTime(2018, 3, 10, 19, 0, 0), fillForwardBars[0].Time); Assert.AreEqual(new DateTime(2018, 3, 11, 20, 0, 0), fillForwardBars[1].Time); fillForwardEnumerator.Dispose(); }
public void FillsForwardHourlyOnMinutesBeginningOfDay() { var dataResolution = Time.OneHour; var reference = new DateTime(2015, 6, 25); var data = new BaseData[] { // thurs 6/25 new TradeBar { Value = 0, Time = reference, Period = dataResolution, Volume = 100 }, // fri 6/26 new TradeBar { Value = 1, Time = reference.Date.AddHours(9), Period = dataResolution, Volume = 200 }, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var ffResolution = TimeSpan.FromMinutes(15); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(ffResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime); // 12:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume); // 9:45 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(9.75), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); // 10:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(10), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume); Assert.IsFalse(fillForwardEnumerator.MoveNext()); fillForwardEnumerator.Dispose(); }
/// <summary> /// Creates an enumerator for the specified security/configuration /// </summary> private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security, SubscriptionDataConfig config, DateTime localStartTime, DateTime localEndTime, MapFileResolver mapFileResolver, IEnumerable <DateTime> tradeableDates, bool useSubscriptionDataReader, bool aggregate) { IEnumerator <BaseData> enumerator; if (useSubscriptionDataReader) { enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); } else { var sourceFactory = (BaseData)Activator.CreateInstance(config.Type); enumerator = (from date in tradeableDates let source = sourceFactory.GetSource(config, date, false) let factory = SubscriptionFactory.ForSource(source, config, date, false) let entriesForDate = factory.Read(source) from entry in entriesForDate select entry).GetEnumerator(); } if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, config.Symbol); } // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (config.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime); } return(enumerator); }
public void FillsForwardMissingDaysOnFillForwardResolutionOfAnHour() { var dataResolution = Time.OneDay; var reference = new DateTime(2015, 6, 23); var data = new BaseData[] { // tues 6/23 new TradeBar { Value = 0, Time = reference, Period = dataResolution, Volume = 100 }, // wed 7/1 new TradeBar { Value = 1, Time = reference.AddDays(8), Period = dataResolution, Volume = 200 }, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var ffResolution = TimeSpan.FromHours(1); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(ffResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime); int dailyBars = 0; int hourlyBars = 0; while (fillForwardEnumerator.MoveNext()) { Console.WriteLine(fillForwardEnumerator.Current.EndTime); if (fillForwardEnumerator.Current.Time.TimeOfDay == TimeSpan.Zero) { dailyBars++; } else { hourlyBars++; Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); } } // we expect 7 daily bars here, beginning tues, wed, thurs, fri, mon, tues, wed Assert.AreEqual(7, dailyBars); // we expect 6 days worth of ff hourly bars at 7 bars a day Assert.AreEqual(42, hourlyBars); fillForwardEnumerator.Dispose(); }
public void FillForwardHoursAtEndOfDayByHalfHour() { var dataResolution = Time.OneHour; var reference = new DateTime(2015, 6, 25, 14, 0, 0); var data = new BaseData[] { // thurs 6/25 new TradeBar{Value = 0, Time = reference, Period = dataResolution}, // fri 6/26 new TradeBar{Value = 1, Time = reference.Date.AddDays(1), Period = dataResolution}, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var ffResolution = TimeSpan.FromMinutes(30); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, ffResolution, isExtendedMarketHours, data.Last().EndTime, dataResolution); // 3:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); // 3:30 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(1.5), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); // 4:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); // 12:00am Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(data.Last().EndTime, fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
public void FillsForwardMidDay() { var dataResolution = Time.OneMinute; var reference = new DateTime(2015, 6, 25, 9, 30, 0); var data = Enumerable.Range(0, 2).Select(x => new TradeBar { Time = reference.AddMinutes(x * 2), Value = x, Period = dataResolution, Volume = (x + 1) * 100 }).ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); var isExtendedMarketHours = false; var fillForwardResolution = TimeSpan.FromMinutes(1); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(fillForwardResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime); // 9:31 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume); // 9:32 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); // 9:33 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume); Assert.IsFalse(fillForwardEnumerator.MoveNext()); fillForwardEnumerator.Dispose(); }
public void FillsForwardRestOfDay() { var dataResolution = Time.OneMinute; var reference = new DateTime(2015, 6, 25, 15, 57, 0); var data = Enumerable.Range(0, 1).Select(x => new TradeBar { Time = reference.AddMinutes(x * 2), Value = x, Period = dataResolution, Volume = 100 }).ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); var isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromMinutes(1)), isExtendedMarketHours, reference.AddMinutes(3), dataResolution, exchange.TimeZone); // 3:58 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume); // 3:59 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); // 4:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
/// <summary> /// Creates a subscription to process the history request /// </summary> private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history) { // data reader expects these values in local times var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency)); var reader = history.GetEnumerator(); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone); } var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end); var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader); return(new Subscription(null, security, config, subscriptionDataEnumerator, timeZoneOffsetProvider, start, end, false)); }
public void FillsForwardMidDay() { var dataResolution = Time.OneMinute; var reference = new DateTime(2015, 6, 25, 9, 30, 0); var data = Enumerable.Range(0, 2).Select(x => new TradeBar { Time = reference.AddMinutes(x*2), Value = x, Period = dataResolution }).ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); var isExtendedMarketHours = false; var fillForwardResolution = TimeSpan.FromMinutes(1); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(fillForwardResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution); // 9:31 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 9:32 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 9:33 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
/// <summary> /// Creates a subscription to process the history request /// </summary> protected Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history) { var config = request.ToSubscriptionDataConfig(); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var reader = history.GetEnumerator(); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // FillForwardEnumerator expects these values in local times var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, end, config.Increment, config.DataTimeZone); } var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); return(SubscriptionUtils.Create(subscriptionRequest, reader)); }
public void HandlesDaylightSavingTimeChange_InifinteLoop() { var dailyBarsEmitted = 0; var fillForwardBars = new List <BaseData>(); var data = new BaseData[] { new QuoteBar { Value = 0, Time = new DateTime(2019, 10, 4, 10, 0, 0), Period = Time.OneDay }, new QuoteBar { Value = 1, Time = new DateTime(2019, 10, 8, 11, 0, 0), Period = Time.OneDay } }.ToList(); var enumerator = data.GetEnumerator(); var algo = new AlgorithmStub(); var market = Market.Oanda; var security = algo.AddCfd("AU200AUD", Resolution.Daily, market); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, security.Exchange, Ref.Create(TimeSpan.FromDays(1)), false, data.Last().EndTime, Time.OneDay, TimeZones.Utc, data.First().EndTime); while (fillForwardEnumerator.MoveNext()) { fillForwardBars.Add(fillForwardEnumerator.Current); Console.WriteLine(fillForwardEnumerator.Current.Time.DayOfWeek + " " + fillForwardEnumerator.Current.Time + " - " + fillForwardEnumerator.Current.EndTime.DayOfWeek + " " + fillForwardEnumerator.Current.EndTime + " " + fillForwardEnumerator.Current.IsFillForward); dailyBarsEmitted++; } Assert.AreEqual(4, dailyBarsEmitted); Assert.AreEqual(new DateTime(2019, 10, 4, 10, 0, 0), fillForwardBars[0].Time); Assert.AreEqual(new DateTime(2019, 10, 6, 11, 0, 0), fillForwardBars[1].Time); Assert.AreEqual(new DateTime(2019, 10, 7, 11, 0, 0), fillForwardBars[2].Time); Assert.AreEqual(new DateTime(2019, 10, 8, 11, 0, 0), fillForwardBars[3].Time); fillForwardEnumerator.Dispose(); }
/// <summary> /// Creates an enumerator for the specified security/configuration /// </summary> private IEnumerator<BaseData> CreateSubscriptionEnumerator(Security security, SubscriptionDataConfig config, DateTime localStartTime, DateTime localEndTime, MapFileResolver mapFileResolver, IEnumerable<DateTime> tradeableDates, bool applySubscripterFilterEnumerator = true) { IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (applySubscripterFilterEnumerator) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime); } return enumerator; }
public void FillsForwardGapBeforeEndOfSubscription() { var dataResolution = Time.OneMinute; var reference = new DateTime(2015, 6, 25, 15, 57, 0); var data = new[] { new TradeBar { Time = reference, Value = 0, Period = dataResolution } }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); var isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromMinutes(1), isExtendedMarketHours, reference.Date.AddHours(16), dataResolution); // 3:58 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 3:39 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 4:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
public void FillsForwardHourlyOnMinutesBeginningOfDay() { var dataResolution = Time.OneHour; var reference = new DateTime(2015, 6, 25); var data = new BaseData[] { // thurs 6/25 new TradeBar{Value = 0, Time = reference, Period = dataResolution}, // fri 6/26 new TradeBar{Value = 1, Time = reference.Date.AddHours(9), Period = dataResolution}, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var ffResolution = TimeSpan.FromMinutes(15); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, ffResolution, isExtendedMarketHours, data.Last().EndTime, dataResolution); // 12:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); // 9:45 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(9.75), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); // 10:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(10), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
public void FillsForwardDailyOnHoursInMarketHours() { var dataResolution = Time.OneDay; var reference = new DateTime(2015, 6, 25); var data = new BaseData[] { // thurs 6/25 new TradeBar{Value = 0, Time = reference, Period = Time.OneDay}, // fri 6/26 new TradeBar{Value = 1, Time = reference.AddDays(1), Period = Time.OneDay}, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromHours(1), isExtendedMarketHours, data.Last().EndTime, dataResolution); // 12:00am Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 10:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(10), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 11:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(11), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 12:00pm (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(12), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 1:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(13), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 2:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(14), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 3:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(15), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 4:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(16), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 12:00am Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
public void FillsForwardFromPreMarket() { var dataResolution = Time.OneMinute; var reference = new DateTime(2015, 6, 25, 9, 28, 0); var data = new [] { new TradeBar { Time = reference, Value = 0, Period = dataResolution }, new TradeBar { Time = reference.AddMinutes(4), Value = 1, Period = dataResolution } }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); var isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromMinutes(1), isExtendedMarketHours, data.Last().EndTime, dataResolution); // 9:29 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 9:31 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 9:32 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(4), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 9:33 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(5), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
private Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined) { var config = security.SubscriptionDataConfig; var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { if (userDefined) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); } return null; } // ReSharper disable once PossibleMultipleEnumeration IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(), security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end); var subscription = new Subscription(security, enumerator, start, end, userDefined, false); return subscription; }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end) { // data reader expects these values in local times start = start.ConvertFromUtc(request.ExchangeHours.TimeZone); end = end.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.TimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData ); var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency)); IEnumerator <BaseData> reader = new SubscriptionDataReader(config, start, end, ResultHandlerStub.Instance, config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty, _factorFileProvider, _dataFileProvider, Time.EachTradeableDay(request.ExchangeHours, start, end), false, includeAuxilliaryData: false ); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, end); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > end) { return(false); } // filter out data before the start return(data.EndTime > start); }); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end); return(new Subscription(null, security, config, reader, timeZoneOffsetProvider, start, end, false)); }
/// <summary> /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured /// </summary> private IEnumerator<BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator<BaseData> enumerator) { if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol); } // optionally apply fill forward logic, but never for tick data if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution, request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (request.Configuration.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal); } return enumerator; }
public void SkipsAfterMarketData() { var dataResolution = Time.OneHour; var reference = new DateTime(2015, 6, 25, 14, 0, 0); var end = reference.Date.AddDays(1).AddHours(10); var data = new BaseData[] { new TradeBar { Time = reference, Value = 0, Period = dataResolution }, new TradeBar { Time = reference.AddHours(3), Value = 1, Period = dataResolution }, new TradeBar { Time = reference.Date.AddDays(1).AddHours(10) - dataResolution, Value = 2, Period = dataResolution } }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromHours(1), isExtendedMarketHours, end, dataResolution); // 3:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 4:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 6:00 - this is raw data, the FF enumerator doesn't try to perform filtering per se, just filtering on when to FF Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(4), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 10:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(end, fillForwardEnumerator.Current.EndTime); Assert.AreEqual(2, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end) { // data reader expects these values in local times start = start.ConvertFromUtc(request.ExchangeHours.TimeZone); end = end.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); var security = new Security( request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency), ErrorCurrencyConverter.Instance ); var dataReader = new SubscriptionDataReader(config, start, end, config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty, _factorFileProvider, Time.EachTradeableDay(request.ExchangeHours, start, end), false, _dataCacheProvider, false ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); }; dataReader.Initialize(); IEnumerator <BaseData> reader = dataReader; // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, end); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > end) { return(false); } // filter out data before the start return(data.EndTime > start); }); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end); var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end); return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider)); }
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { _cancellationTokenSource = new CancellationTokenSource(); Subscriptions = algorithm.SubscriptionManager.Subscriptions; _subscriptions = Subscriptions.Count; //Public Properties: IsActive = true; _endOfBridge = new bool[_subscriptions]; SubscriptionReaders = new IEnumerator<BaseData>[_subscriptions]; FillForwardFrontiers = new DateTime[_subscriptions]; RealtimePrices = new List<decimal>(_subscriptions); //Class Privates: _algorithm = algorithm; // find the minimum resolution, ignoring ticks var fillForwardResolution = Subscriptions .Where(x => x.Resolution != Resolution.Tick) .Select(x => x.Resolution.ToTimeSpan()) .DefaultIfEmpty(TimeSpan.FromSeconds(1)) .Min(); // figure out how many subscriptions are at the minimum resolution var subscriptionsAtMinimumResolution = (from sub in Subscriptions where sub.Resolution == Subscriptions.Min(x => x.Resolution) select sub).Count(); Bridge = new BlockingCollection<TimeSlice>(Math.Min(1000, 50000/subscriptionsAtMinimumResolution)); for (var i = 0; i < _subscriptions; i++) { _endOfBridge[i] = false; var config = Subscriptions[i]; var start = algorithm.StartDate; var end = algorithm.EndDate; var security = _algorithm.Securities[Subscriptions[i].Symbol]; var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date); IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false, // we want a backtest to use the same symbols regardless of start/end dates, so we set the resolution // to null which means give me the current/most recent symbol mapping for the requested symbol symbolResolutionDate: null ); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters SubscriptionReaders[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end); FillForwardFrontiers[i] = new DateTime(); // prime the pump for iteration in Run _endOfBridge[i] = !SubscriptionReaders[i].MoveNext(); if (_endOfBridge[i]) { Log.Trace("FileSystemDataFeed.Run(): Failed to load subscription: " + Subscriptions[i].Symbol); } } }
public void FillsForwardFromPreMarket() { var dataResolution = Time.OneMinute; var reference = new DateTime(2015, 6, 25, 9, 28, 0); var data = new [] { new TradeBar { Time = reference, Value = 0, Period = dataResolution, Volume = 100 }, new TradeBar { Time = reference.AddMinutes(4), Value = 1, Period = dataResolution, Volume = 200 } }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); var isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromMinutes(1)), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime); // 9:29 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume); // 9:31 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); // 9:32 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(4), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); // 9:33 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddMinutes(5), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume); Assert.IsFalse(fillForwardEnumerator.MoveNext()); fillForwardEnumerator.Dispose(); }
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return null; } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market); // ReSharper disable once PossibleMultipleEnumeration IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return subscription; }
public void FillsForwardMissingDaysOnFillForwardResolutionOfAnHour() { var dataResolution = Time.OneDay; var reference = new DateTime(2015, 6, 23); var data = new BaseData[] { // tues 6/23 new TradeBar{Value = 0, Time = reference, Period = dataResolution}, // wed 7/1 new TradeBar{Value = 1, Time = reference.AddDays(8), Period = dataResolution}, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var ffResolution = TimeSpan.FromHours(1); var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, ffResolution, isExtendedMarketHours, data.Last().EndTime, dataResolution); int dailyBars = 0; int hourlyBars = 0; while (fillForwardEnumerator.MoveNext()) { Console.WriteLine(fillForwardEnumerator.Current.EndTime); if (fillForwardEnumerator.Current.Time.TimeOfDay == TimeSpan.Zero) { dailyBars++; } else { hourlyBars++; } } // we expect 7 daily bars here, beginning tues, wed, thurs, fri, mon, tues, wed Assert.AreEqual(7, dailyBars); // we expect 6 days worth of ff hourly bars at 7 bars a day Assert.AreEqual(42, hourlyBars); }
public void FillsForwardDailyOnHoursInMarketHours() { var dataResolution = Time.OneDay; var reference = new DateTime(2015, 6, 25); var data = new BaseData[] { // thurs 6/25 new TradeBar { Value = 0, Time = reference, Period = Time.OneDay }, // fri 6/26 new TradeBar { Value = 1, Time = reference.AddDays(1), Period = Time.OneDay }, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromHours(1), isExtendedMarketHours, data.Last().EndTime, dataResolution); // 12:00am Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 10:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(10), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 11:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(11), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 12:00pm (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(12), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 1:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(13), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 2:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(14), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 3:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(15), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 4:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1).AddHours(16), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 12:00am Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
public void FillsForwardToNextDay() { var dataResolution = Time.OneHour; var reference = new DateTime(2015, 6, 25, 14, 0, 0); var end = reference.Date.AddDays(1).AddHours(10); var data = new [] { new TradeBar { Time = reference, Value = 0, Period = dataResolution }, new TradeBar { Time = end - dataResolution, Value = 1, Period = dataResolution } }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromHours(1), isExtendedMarketHours, end, dataResolution); // 3:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 4:00 (ff) Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddHours(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); // 10:00 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(end, fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
public void FillsForwardDailyMissingDays() { var dataResolution = Time.OneDay; var reference = new DateTime(2015, 6, 25); var data = new BaseData[] { // thurs 6/25 new TradeBar { Value = 0, Time = reference, Period = Time.OneDay, Volume = 100 }, // fri 6/26 new TradeBar { Value = 1, Time = reference.AddDays(5), Period = Time.OneDay, Volume = 200 }, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromDays(1)), isExtendedMarketHours, data.Last().EndTime.AddDays(1), dataResolution, exchange.TimeZone, data.First().EndTime); // 6/25 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume); // 6/26 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); // 6/29 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(5), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); // 6/30 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(6), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume); // 7/1 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(7), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume); Assert.IsFalse(fillForwardEnumerator.MoveNext()); fillForwardEnumerator.Dispose(); }
public void FillsForwardDailyMissingDays() { var dataResolution = Time.OneDay; var reference = new DateTime(2015, 6, 25); var data = new BaseData[] { // thurs 6/25 new TradeBar{Value = 0, Time = reference, Period = Time.OneDay}, // fri 6/26 new TradeBar{Value = 1, Time = reference.AddDays(5), Period = Time.OneDay}, }.ToList(); var enumerator = data.GetEnumerator(); var exchange = new EquityExchange(); bool isExtendedMarketHours = false; var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromDays(1), isExtendedMarketHours, data.Last().EndTime.AddDays(1), dataResolution); // 6/25 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(1), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); // 6/26 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(2), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); // 6/29 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(5), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(0, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); // 6/30 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(6), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); // 7/1 Assert.IsTrue(fillForwardEnumerator.MoveNext()); Assert.AreEqual(reference.AddDays(7), fillForwardEnumerator.Current.EndTime); Assert.AreEqual(1, fillForwardEnumerator.Current.Value); Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward); Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution); Assert.IsFalse(fillForwardEnumerator.MoveNext()); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request) { var config = request.ToSubscriptionDataConfig(); DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var dataReader = new SubscriptionDataReader(config, request, _mapFileProvider, _factorFileProvider, _dataCacheProvider, _dataProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, _mapFileProvider, request.StartTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out all aux data if (data.DataType == MarketDataType.Auxiliary) { return(false); } // filter out future data if (data.EndTime > request.EndTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > request.StartTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc) { // data reader expects these values in local times var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); _dataPermissionManager.AssertConfiguration(config); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var mapFileResolver = MapFileResolver.Empty; if (config.TickerShouldBeMapped()) { mapFileResolver = _mapFileProvider.Get(config.Market); var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol); } // Tradable dates are defined with the data time zone to access the right source var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours); var dataReader = new SubscriptionDataReader(config, startTimeLocal, endTimeLocal, mapFileResolver, _factorFileProvider, tradableDates, false, _dataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, mapFileResolver, false, startTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > endTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > startTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }
private static Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined) { var config = security.SubscriptionDataConfig; var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date); var symbolResolutionDate = userDefined ? (DateTime?)null : start; IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false, symbolResolutionDate); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(), security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end); var subscription = new Subscription(security, enumerator, start, end, userDefined, false); return subscription; }