public void FillForwardHoursAtEndOfDayByHalfHour()
{
var dataResolution = Time.OneHour;
var reference = new DateTime(2015, 6, 25, 14, 0, 0);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar {
Value = 0, Time = reference, Period = dataResolution
},
// fri 6/26
new TradeBar {
Value = 1, Time = reference.Date.AddDays(1), Period = dataResolution
},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var ffResolution = TimeSpan.FromMinutes(30);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(ffResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution);
// 3:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
// 3:30
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1.5), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
// 4:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
// 12:00am
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(data.Last().EndTime, fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
public void FillsForwardGapBeforeEndOfSubscription()
{
var dataResolution = Time.OneMinute;
var reference = new DateTime(2015, 6, 25, 15, 57, 0);
var data = new[]
{
new TradeBar
{
Time = reference,
Value = 0,
Period = dataResolution,
Volume = 100
}
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
var isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromMinutes(1)), isExtendedMarketHours, reference.Date.AddHours(16), dataResolution, exchange.TimeZone, data.First().EndTime);
// 3:58
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 3:39 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 4:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
fillForwardEnumerator.Dispose();
}
public void HandlesDaylightSavingTimeChange()
{
var dailyBarsEmitted = 0;
var fillForwardBars = new List <BaseData>();
// 3 QuoteBars as they would be read from the EURUSD oanda daily file by QuoteBar.Reader()
// The conversion from dataTimeZone to exchangeTimeZone has been done by hand
// dataTimeZone == UTC
/*
* 20180311 00:00,1.2308,1.2308,1.2308,1.2308,0,1.23096,1.23096,1.23096,1.23096,0
* 20180312 00:00,1.23082,1.23449,1.22898,1.23382,0,1.23097,1.23463,1.22911,1.23396,0
*/
var data = new BaseData[]
{
// Sunday 3/11
new QuoteBar {
Value = 0, Time = new DateTime(2018, 3, 10, 19, 0, 0), Period = Time.OneDay
},
// Monday 3/12
new QuoteBar {
Value = 1, Time = new DateTime(2018, 3, 11, 20, 0, 0), Period = Time.OneDay
},
}.ToList();
var enumerator = data.GetEnumerator();
var market = Market.Oanda;
var symbol = Symbol.Create("EURUSD", SecurityType.Forex, market);
var marketHours = MarketHoursDatabase.FromDataFolder();
var exchange = new ForexExchange(marketHours.GetExchangeHours(market, symbol, SecurityType.Forex));
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromDays(1)), false, data.Last().EndTime, Time.OneDay, TimeZones.Utc, data.First().EndTime);
while (fillForwardEnumerator.MoveNext())
{
fillForwardBars.Add(fillForwardEnumerator.Current);
Console.WriteLine(fillForwardEnumerator.Current.Time.DayOfWeek + " " + fillForwardEnumerator.Current.Time + " - " + fillForwardEnumerator.Current.EndTime.DayOfWeek + " " + fillForwardEnumerator.Current.EndTime);
dailyBarsEmitted++;
}
Assert.AreEqual(2, dailyBarsEmitted);
Assert.AreEqual(new DateTime(2018, 3, 10, 19, 0, 0), fillForwardBars[0].Time);
Assert.AreEqual(new DateTime(2018, 3, 11, 20, 0, 0), fillForwardBars[1].Time);
fillForwardEnumerator.Dispose();
}
public void FillsForwardHourlyOnMinutesBeginningOfDay()
{
var dataResolution = Time.OneHour;
var reference = new DateTime(2015, 6, 25);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar {
Value = 0, Time = reference, Period = dataResolution, Volume = 100
},
// fri 6/26
new TradeBar {
Value = 1, Time = reference.Date.AddHours(9), Period = dataResolution, Volume = 200
},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var ffResolution = TimeSpan.FromMinutes(15);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(ffResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime);
// 12:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:45 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(9.75), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 10:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(10), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
fillForwardEnumerator.Dispose();
}
/// <summary>
/// Creates an enumerator for the specified security/configuration
/// </summary>
private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security,
SubscriptionDataConfig config,
DateTime localStartTime,
DateTime localEndTime,
MapFileResolver mapFileResolver,
IEnumerable <DateTime> tradeableDates,
bool useSubscriptionDataReader,
bool aggregate)
{
IEnumerator <BaseData> enumerator;
if (useSubscriptionDataReader)
{
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver,
_factorFileProvider, tradeableDates, false);
}
else
{
var sourceFactory = (BaseData)Activator.CreateInstance(config.Type);
enumerator = (from date in tradeableDates
let source = sourceFactory.GetSource(config, date, false)
let factory = SubscriptionFactory.ForSource(source, config, date, false)
let entriesForDate = factory.Read(source)
from entry in entriesForDate
select entry).GetEnumerator();
}
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, config.Symbol);
}
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (config.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime);
}
return(enumerator);
}
public void FillsForwardMissingDaysOnFillForwardResolutionOfAnHour()
{
var dataResolution = Time.OneDay;
var reference = new DateTime(2015, 6, 23);
var data = new BaseData[]
{
// tues 6/23
new TradeBar {
Value = 0, Time = reference, Period = dataResolution, Volume = 100
},
// wed 7/1
new TradeBar {
Value = 1, Time = reference.AddDays(8), Period = dataResolution, Volume = 200
},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var ffResolution = TimeSpan.FromHours(1);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(ffResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime);
int dailyBars = 0;
int hourlyBars = 0;
while (fillForwardEnumerator.MoveNext())
{
Console.WriteLine(fillForwardEnumerator.Current.EndTime);
if (fillForwardEnumerator.Current.Time.TimeOfDay == TimeSpan.Zero)
{
dailyBars++;
}
else
{
hourlyBars++;
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
}
}
// we expect 7 daily bars here, beginning tues, wed, thurs, fri, mon, tues, wed
Assert.AreEqual(7, dailyBars);
// we expect 6 days worth of ff hourly bars at 7 bars a day
Assert.AreEqual(42, hourlyBars);
fillForwardEnumerator.Dispose();
}
public void FillForwardHoursAtEndOfDayByHalfHour()
{
var dataResolution = Time.OneHour;
var reference = new DateTime(2015, 6, 25, 14, 0, 0);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar{Value = 0, Time = reference, Period = dataResolution},
// fri 6/26
new TradeBar{Value = 1, Time = reference.Date.AddDays(1), Period = dataResolution},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var ffResolution = TimeSpan.FromMinutes(30);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, ffResolution, isExtendedMarketHours, data.Last().EndTime, dataResolution);
// 3:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
// 3:30
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1.5), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
// 4:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
// 12:00am
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(data.Last().EndTime, fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
public void FillsForwardMidDay()
{
var dataResolution = Time.OneMinute;
var reference = new DateTime(2015, 6, 25, 9, 30, 0);
var data = Enumerable.Range(0, 2).Select(x => new TradeBar
{
Time = reference.AddMinutes(x * 2),
Value = x,
Period = dataResolution,
Volume = (x + 1) * 100
}).ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
var isExtendedMarketHours = false;
var fillForwardResolution = TimeSpan.FromMinutes(1);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(fillForwardResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime);
// 9:31
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:32 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:33
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
fillForwardEnumerator.Dispose();
}
public void FillsForwardRestOfDay()
{
var dataResolution = Time.OneMinute;
var reference = new DateTime(2015, 6, 25, 15, 57, 0);
var data = Enumerable.Range(0, 1).Select(x => new TradeBar
{
Time = reference.AddMinutes(x * 2),
Value = x,
Period = dataResolution,
Volume = 100
}).ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
var isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromMinutes(1)), isExtendedMarketHours, reference.AddMinutes(3), dataResolution, exchange.TimeZone);
// 3:58
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 3:59 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 4:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
/// <summary>
/// Creates a subscription to process the history request
/// </summary>
private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history)
{
// data reader expects these values in local times
var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency));
var reader = history.GetEnumerator();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone);
}
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader);
return(new Subscription(null, security, config, subscriptionDataEnumerator, timeZoneOffsetProvider, start, end, false));
}
public void FillsForwardMidDay()
{
var dataResolution = Time.OneMinute;
var reference = new DateTime(2015, 6, 25, 9, 30, 0);
var data = Enumerable.Range(0, 2).Select(x => new TradeBar
{
Time = reference.AddMinutes(x*2),
Value = x,
Period = dataResolution
}).ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
var isExtendedMarketHours = false;
var fillForwardResolution = TimeSpan.FromMinutes(1);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(fillForwardResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution);
// 9:31
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 9:32 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 9:33
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
/// <summary>
/// Creates a subscription to process the history request
/// </summary>
protected Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history)
{
var config = request.ToSubscriptionDataConfig();
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var reader = history.GetEnumerator();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// FillForwardEnumerator expects these values in local times
var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, end, config.Increment, config.DataTimeZone);
}
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
public void HandlesDaylightSavingTimeChange_InifinteLoop()
{
var dailyBarsEmitted = 0;
var fillForwardBars = new List <BaseData>();
var data = new BaseData[]
{
new QuoteBar {
Value = 0, Time = new DateTime(2019, 10, 4, 10, 0, 0), Period = Time.OneDay
},
new QuoteBar {
Value = 1, Time = new DateTime(2019, 10, 8, 11, 0, 0), Period = Time.OneDay
}
}.ToList();
var enumerator = data.GetEnumerator();
var algo = new AlgorithmStub();
var market = Market.Oanda;
var security = algo.AddCfd("AU200AUD", Resolution.Daily, market);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, security.Exchange, Ref.Create(TimeSpan.FromDays(1)), false, data.Last().EndTime, Time.OneDay, TimeZones.Utc, data.First().EndTime);
while (fillForwardEnumerator.MoveNext())
{
fillForwardBars.Add(fillForwardEnumerator.Current);
Console.WriteLine(fillForwardEnumerator.Current.Time.DayOfWeek + " " + fillForwardEnumerator.Current.Time + " - " + fillForwardEnumerator.Current.EndTime.DayOfWeek + " " + fillForwardEnumerator.Current.EndTime + " " + fillForwardEnumerator.Current.IsFillForward);
dailyBarsEmitted++;
}
Assert.AreEqual(4, dailyBarsEmitted);
Assert.AreEqual(new DateTime(2019, 10, 4, 10, 0, 0), fillForwardBars[0].Time);
Assert.AreEqual(new DateTime(2019, 10, 6, 11, 0, 0), fillForwardBars[1].Time);
Assert.AreEqual(new DateTime(2019, 10, 7, 11, 0, 0), fillForwardBars[2].Time);
Assert.AreEqual(new DateTime(2019, 10, 8, 11, 0, 0), fillForwardBars[3].Time);
fillForwardEnumerator.Dispose();
}
/// <summary>
/// Creates an enumerator for the specified security/configuration
/// </summary>
private IEnumerator<BaseData> CreateSubscriptionEnumerator(Security security,
SubscriptionDataConfig config,
DateTime localStartTime,
DateTime localEndTime,
MapFileResolver mapFileResolver,
IEnumerable<DateTime> tradeableDates,
bool applySubscripterFilterEnumerator = true)
{
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver,
_factorFileProvider, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (applySubscripterFilterEnumerator)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime);
}
return enumerator;
}
public void FillsForwardGapBeforeEndOfSubscription()
{
var dataResolution = Time.OneMinute;
var reference = new DateTime(2015, 6, 25, 15, 57, 0);
var data = new[]
{
new TradeBar
{
Time = reference,
Value = 0,
Period = dataResolution
}
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
var isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromMinutes(1), isExtendedMarketHours, reference.Date.AddHours(16), dataResolution);
// 3:58
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 3:39 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 4:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
public void FillsForwardHourlyOnMinutesBeginningOfDay()
{
var dataResolution = Time.OneHour;
var reference = new DateTime(2015, 6, 25);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar{Value = 0, Time = reference, Period = dataResolution},
// fri 6/26
new TradeBar{Value = 1, Time = reference.Date.AddHours(9), Period = dataResolution},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var ffResolution = TimeSpan.FromMinutes(15);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, ffResolution, isExtendedMarketHours, data.Last().EndTime, dataResolution);
// 12:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
// 9:45 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(9.75), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
// 10:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(10), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
public void FillsForwardDailyOnHoursInMarketHours()
{
var dataResolution = Time.OneDay;
var reference = new DateTime(2015, 6, 25);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar{Value = 0, Time = reference, Period = Time.OneDay},
// fri 6/26
new TradeBar{Value = 1, Time = reference.AddDays(1), Period = Time.OneDay},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromHours(1), isExtendedMarketHours, data.Last().EndTime, dataResolution);
// 12:00am
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 10:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(10), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 11:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(11), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 12:00pm (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(12), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 1:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(13), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 2:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(14), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 3:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(15), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 4:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(16), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 12:00am
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
public void FillsForwardFromPreMarket()
{
var dataResolution = Time.OneMinute;
var reference = new DateTime(2015, 6, 25, 9, 28, 0);
var data = new []
{
new TradeBar
{
Time = reference,
Value = 0,
Period = dataResolution
},
new TradeBar
{
Time = reference.AddMinutes(4),
Value = 1,
Period = dataResolution
}
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
var isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromMinutes(1), isExtendedMarketHours, data.Last().EndTime, dataResolution);
// 9:29
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 9:31 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 9:32 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(4), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 9:33
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(5), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
private Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined)
{
var config = security.SubscriptionDataConfig;
var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
if (userDefined)
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
}
return null;
}
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(),
security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
var subscription = new Subscription(security, enumerator, start, end, userDefined, false);
return subscription;
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.TimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData
);
var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency));
IEnumerator <BaseData> reader = new SubscriptionDataReader(config,
start,
end,
ResultHandlerStub.Instance,
config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty,
_factorFileProvider,
_dataFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
includeAuxilliaryData: false
);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
return(new Subscription(null, security, config, reader, timeZoneOffsetProvider, start, end, false));
}
/// <summary>
/// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
/// </summary>
private IEnumerator<BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator<BaseData> enumerator)
{
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
}
// optionally apply fill forward logic, but never for tick data
if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution,
request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (request.Configuration.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal);
}
return enumerator;
}
public void SkipsAfterMarketData()
{
var dataResolution = Time.OneHour;
var reference = new DateTime(2015, 6, 25, 14, 0, 0);
var end = reference.Date.AddDays(1).AddHours(10);
var data = new BaseData[]
{
new TradeBar
{
Time = reference,
Value = 0,
Period = dataResolution
},
new TradeBar
{
Time = reference.AddHours(3),
Value = 1,
Period = dataResolution
},
new TradeBar
{
Time = reference.Date.AddDays(1).AddHours(10) - dataResolution,
Value = 2,
Period = dataResolution
}
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromHours(1), isExtendedMarketHours, end, dataResolution);
// 3:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 4:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 6:00 - this is raw data, the FF enumerator doesn't try to perform filtering per se, just filtering on when to FF
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(4), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 10:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(end, fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(2, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(CashBook.AccountCurrency, 0, 1m),
SymbolProperties.GetDefault(CashBook.AccountCurrency),
ErrorCurrencyConverter.Instance
);
var dataReader = new SubscriptionDataReader(config,
start,
end,
config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty,
_factorFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
_dataCacheProvider,
false
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); };
dataReader.Initialize();
IEnumerator <BaseData> reader = dataReader;
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
_cancellationTokenSource = new CancellationTokenSource();
Subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptions = Subscriptions.Count;
//Public Properties:
IsActive = true;
_endOfBridge = new bool[_subscriptions];
SubscriptionReaders = new IEnumerator<BaseData>[_subscriptions];
FillForwardFrontiers = new DateTime[_subscriptions];
RealtimePrices = new List<decimal>(_subscriptions);
//Class Privates:
_algorithm = algorithm;
// find the minimum resolution, ignoring ticks
var fillForwardResolution = Subscriptions
.Where(x => x.Resolution != Resolution.Tick)
.Select(x => x.Resolution.ToTimeSpan())
.DefaultIfEmpty(TimeSpan.FromSeconds(1))
.Min();
// figure out how many subscriptions are at the minimum resolution
var subscriptionsAtMinimumResolution =
(from sub in Subscriptions
where sub.Resolution == Subscriptions.Min(x => x.Resolution)
select sub).Count();
Bridge = new BlockingCollection<TimeSlice>(Math.Min(1000, 50000/subscriptionsAtMinimumResolution));
for (var i = 0; i < _subscriptions; i++)
{
_endOfBridge[i] = false;
var config = Subscriptions[i];
var start = algorithm.StartDate;
var end = algorithm.EndDate;
var security = _algorithm.Securities[Subscriptions[i].Symbol];
var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security,
start, end,
resultHandler,
tradeableDates,
false,
// we want a backtest to use the same symbols regardless of start/end dates, so we set the resolution
// to null which means give me the current/most recent symbol mapping for the requested symbol
symbolResolutionDate: null
);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
SubscriptionReaders[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
FillForwardFrontiers[i] = new DateTime();
// prime the pump for iteration in Run
_endOfBridge[i] = !SubscriptionReaders[i].MoveNext();
if (_endOfBridge[i])
{
Log.Trace("FileSystemDataFeed.Run(): Failed to load subscription: " + Subscriptions[i].Symbol);
}
}
}
public void SkipsAfterMarketData()
{
var dataResolution = Time.OneHour;
var reference = new DateTime(2015, 6, 25, 14, 0, 0);
var end = reference.Date.AddDays(1).AddHours(10);
var data = new BaseData[]
{
new TradeBar
{
Time = reference,
Value = 0,
Period = dataResolution
},
new TradeBar
{
Time = reference.AddHours(3),
Value = 1,
Period = dataResolution
},
new TradeBar
{
Time = reference.Date.AddDays(1).AddHours(10) - dataResolution,
Value = 2,
Period = dataResolution
}
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromHours(1), isExtendedMarketHours, end, dataResolution);
// 3:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 4:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 6:00 - this is raw data, the FF enumerator doesn't try to perform filtering per se, just filtering on when to FF
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(4), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 10:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(end, fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(2, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
public void FillsForwardFromPreMarket()
{
var dataResolution = Time.OneMinute;
var reference = new DateTime(2015, 6, 25, 9, 28, 0);
var data = new []
{
new TradeBar
{
Time = reference,
Value = 0,
Period = dataResolution,
Volume = 100
},
new TradeBar
{
Time = reference.AddMinutes(4),
Value = 1,
Period = dataResolution,
Volume = 200
}
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
var isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromMinutes(1)), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime);
// 9:29
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:31 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:32 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(4), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:33
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(5), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
fillForwardEnumerator.Dispose();
}
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution)
{
var config = security.SubscriptionDataConfig;
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return null;
}
// get the map file resolver for this market
var mapFileResolver = MapFileResolver.Empty;
if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return subscription;
}
public void FillsForwardMissingDaysOnFillForwardResolutionOfAnHour()
{
var dataResolution = Time.OneDay;
var reference = new DateTime(2015, 6, 23);
var data = new BaseData[]
{
// tues 6/23
new TradeBar{Value = 0, Time = reference, Period = dataResolution},
// wed 7/1
new TradeBar{Value = 1, Time = reference.AddDays(8), Period = dataResolution},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var ffResolution = TimeSpan.FromHours(1);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, ffResolution, isExtendedMarketHours, data.Last().EndTime, dataResolution);
int dailyBars = 0;
int hourlyBars = 0;
while (fillForwardEnumerator.MoveNext())
{
Console.WriteLine(fillForwardEnumerator.Current.EndTime);
if (fillForwardEnumerator.Current.Time.TimeOfDay == TimeSpan.Zero)
{
dailyBars++;
}
else
{
hourlyBars++;
}
}
// we expect 7 daily bars here, beginning tues, wed, thurs, fri, mon, tues, wed
Assert.AreEqual(7, dailyBars);
// we expect 6 days worth of ff hourly bars at 7 bars a day
Assert.AreEqual(42, hourlyBars);
}
public void FillsForwardDailyOnHoursInMarketHours()
{
var dataResolution = Time.OneDay;
var reference = new DateTime(2015, 6, 25);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar {
Value = 0, Time = reference, Period = Time.OneDay
},
// fri 6/26
new TradeBar {
Value = 1, Time = reference.AddDays(1), Period = Time.OneDay
},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromHours(1), isExtendedMarketHours, data.Last().EndTime, dataResolution);
// 12:00am
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 10:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(10), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 11:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(11), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 12:00pm (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(12), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 1:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(13), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 2:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(14), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 3:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(15), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 4:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1).AddHours(16), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 12:00am
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
public void FillsForwardToNextDay()
{
var dataResolution = Time.OneHour;
var reference = new DateTime(2015, 6, 25, 14, 0, 0);
var end = reference.Date.AddDays(1).AddHours(10);
var data = new []
{
new TradeBar
{
Time = reference,
Value = 0,
Period = dataResolution
},
new TradeBar
{
Time = end - dataResolution,
Value = 1,
Period = dataResolution
}
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromHours(1), isExtendedMarketHours, end, dataResolution);
// 3:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 4:00 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
// 10:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(end, fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
public void FillsForwardDailyMissingDays()
{
var dataResolution = Time.OneDay;
var reference = new DateTime(2015, 6, 25);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar {
Value = 0, Time = reference, Period = Time.OneDay, Volume = 100
},
// fri 6/26
new TradeBar {
Value = 1, Time = reference.AddDays(5), Period = Time.OneDay, Volume = 200
},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromDays(1)), isExtendedMarketHours, data.Last().EndTime.AddDays(1), dataResolution, exchange.TimeZone, data.First().EndTime);
// 6/25
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 6/26
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 6/29
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(5), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 6/30
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(6), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 7/1
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(7), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
fillForwardEnumerator.Dispose();
}
public void FillsForwardDailyMissingDays()
{
var dataResolution = Time.OneDay;
var reference = new DateTime(2015, 6, 25);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar{Value = 0, Time = reference, Period = Time.OneDay},
// fri 6/26
new TradeBar{Value = 1, Time = reference.AddDays(5), Period = Time.OneDay},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, TimeSpan.FromDays(1), isExtendedMarketHours, data.Last().EndTime.AddDays(1), dataResolution);
// 6/25
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
// 6/26
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
// 6/29
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(5), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
// 6/30
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(6), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
// 7/1
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(7), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request)
{
var config = request.ToSubscriptionDataConfig();
DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var dataReader = new SubscriptionDataReader(config,
request,
_mapFileProvider,
_factorFileProvider,
_dataCacheProvider,
_dataProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
_mapFileProvider,
request.StartTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out all aux data
if (data.DataType == MarketDataType.Auxiliary)
{
return(false);
}
// filter out future data
if (data.EndTime > request.EndTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > request.StartTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc)
{
// data reader expects these values in local times
var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
_dataPermissionManager.AssertConfiguration(config);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var mapFileResolver = MapFileResolver.Empty;
if (config.TickerShouldBeMapped())
{
mapFileResolver = _mapFileProvider.Get(config.Market);
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol);
}
// Tradable dates are defined with the data time zone to access the right source
var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours);
var dataReader = new SubscriptionDataReader(config,
startTimeLocal,
endTimeLocal,
mapFileResolver,
_factorFileProvider,
tradableDates,
false,
_dataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
mapFileResolver,
false,
startTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > endTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > startTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
private static Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined)
{
var config = security.SubscriptionDataConfig;
var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);
var symbolResolutionDate = userDefined ? (DateTime?)null : start;
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false, symbolResolutionDate);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(),
security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
var subscription = new Subscription(security, enumerator, start, end, userDefined, false);
return subscription;
}