public void EmptyNullEnumerators(bool skipsBasedOnEndTime)
{
var time = new DateTime(2020, 1, 1);
// empty enumerators
var enumerator1 = new List <BaseData>().GetEnumerator();
var enumerator2 = new List <BaseData>().GetEnumerator();
var enumerator3 = new List <BaseData>
{
new Tick(time, Symbols.SPY, 10, 10),
new Tick(time.AddSeconds(-1), Symbols.SPY, 20, 20),
new Tick(time.AddSeconds(1), Symbols.SPY, 30, 30)
}.GetEnumerator();
var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, null, enumerator2, enumerator3);
Assert.IsTrue(concat.MoveNext());
Assert.AreEqual(10, (concat.Current as Tick).AskPrice);
Assert.IsTrue(concat.MoveNext());
Assert.AreEqual(20, (concat.Current as Tick).AskPrice);
Assert.IsTrue(concat.MoveNext());
Assert.AreEqual(30, (concat.Current as Tick).AskPrice);
Assert.IsFalse(concat.MoveNext());
Assert.IsNull(concat.Current);
concat.Dispose();
}
public void SkipsBasedOnEndTime(bool skipsBasedOnEndTime)
{
var time = new DateTime(2020, 1, 1);
var enumerator1 = new List <BaseData> {
new Tick(time, Symbols.SPY, 10, 10)
}.GetEnumerator();
var enumerator2 = new List <BaseData>
{
new Tick(time.AddSeconds(-1), Symbols.SPY, 20, 20), //should be skipped because end time is before previous tick
new Tick(time.AddSeconds(1), Symbols.SPY, 30, 30)
}.GetEnumerator();
var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, enumerator2);
Assert.IsTrue(concat.MoveNext());
Assert.AreEqual(10, (concat.Current as Tick).AskPrice);
if (!skipsBasedOnEndTime)
{
Assert.IsTrue(concat.MoveNext());
Assert.AreEqual(20, (concat.Current as Tick).AskPrice);
}
Assert.IsTrue(concat.MoveNext());
Assert.AreEqual(30, (concat.Current as Tick).AskPrice);
Assert.IsFalse(concat.MoveNext());
Assert.IsNull(concat.Current);
concat.Dispose();
}
/// <summary>
/// Build and apply the warmup enumerators when required
/// </summary>
private IEnumerator <BaseData> GetWarmupEnumerator(SubscriptionRequest request, IEnumerator <BaseData> liveEnumerator)
{
if (_algorithm.IsWarmingUp)
{
var warmupRequest = new SubscriptionRequest(request, endTimeUtc: _timeProvider.GetUtcNow(),
// we will not fill forward each warmup enumerators separately but concatenated bellow
configuration: new SubscriptionDataConfig(request.Configuration, fillForward: false));
if (warmupRequest.TradableDays.Any())
{
// since we will source data locally and from the history provider, let's limit the history request size
// by setting a start date respecting the 'MaximumWarmupHistoryDaysLookBack'
var historyWarmup = warmupRequest;
var warmupHistoryStartDate = warmupRequest.EndTimeUtc.AddDays(-MaximumWarmupHistoryDaysLookBack);
if (warmupHistoryStartDate > warmupRequest.StartTimeUtc)
{
historyWarmup = new SubscriptionRequest(warmupRequest, startTimeUtc: warmupHistoryStartDate);
}
var synchronizedWarmupEnumerator = TryAddFillForwardEnumerator(warmupRequest,
// we concatenate the file based and history based warmup enumerators, dropping duplicate time stamps
new ConcatEnumerator(true, GetFileBasedWarmupEnumerator(warmupRequest), GetHistoryWarmupEnumerator(historyWarmup))
{
CanEmitNull = false
},
// if required by the original request, we will fill forward the Synced warmup data
request.Configuration.FillDataForward);
// the order here is important, concat enumerator will keep the last enumerator given and dispose of the rest
liveEnumerator = new ConcatEnumerator(true, synchronizedWarmupEnumerator, liveEnumerator);
}
}
return(liveEnumerator);
}
protected void FixKeyMap()
{
_keyToEntry.Clear();
IEnumerator ie = new ConcatEnumerator(_entries.GetEnumerator(), new ConcatEnumerator(_fixedEntries.GetEnumerator(), _macroEntries.GetEnumerator()));
while (ie.MoveNext())
{
Entry e = (Entry)ie.Current;
if (e.Key != Keys.None)
{
Keys k = e.Key | e.Modifiers;
if (!_keyToEntry.ContainsKey(k))
{
_keyToEntry.Add(k, e);
}
}
}
_keyMapIsDirty = false;
}
/// <summary>
/// Returns the content of the collection as <see cref="ConcatEnumerator"/>.
/// </summary>
public IEnumerator <T> Get()
{
if (this.getters == null)
{
Debug.LogWarning("Tried getting value event with no getter!");
return(default(IEnumerator <T>));
}
// Setup enumerator
ConcatEnumerator enumerator = new ConcatEnumerator();
enumerator.Init();
for (int i = 0; i < this.getters.Count; i++)
{
enumerator.AddCollection(this.getters[i]());
}
return(enumerator);
}
public void DropsEnumeratorsReturningNullAndTrue(bool skipsBasedOnEndTime)
{
var enumerator1 = new TestEnumerator();
var enumerator2 = new TestEnumerator();
var concat = new ConcatEnumerator(skipsBasedOnEndTime, enumerator1, null, enumerator2);
Assert.IsTrue(concat.MoveNext());
Assert.IsNull(concat.Current);
Assert.IsTrue(enumerator1.Disposed);
Assert.IsFalse(enumerator2.Disposed);
Assert.AreEqual(1, enumerator2.MoveNextCount);
Assert.IsTrue(concat.MoveNext());
// we assert it just keeps the last enumerator and drops the rest
Assert.IsTrue(enumerator1.Disposed);
Assert.IsFalse(enumerator2.Disposed);
Assert.IsNull(concat.Current);
Assert.AreEqual(2, enumerator2.MoveNextCount);
concat.Dispose();
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc)
{
// data reader expects these values in local times
var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
_dataPermissionManager.AssertConfiguration(config);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var mapFileResolver = MapFileResolver.Empty;
if (config.TickerShouldBeMapped())
{
mapFileResolver = _mapFileProvider.Get(config.Market);
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol);
}
// Tradable dates are defined with the data time zone to access the right source
var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours);
var dataReader = new SubscriptionDataReader(config,
startTimeLocal,
endTimeLocal,
mapFileResolver,
_factorFileProvider,
tradableDates,
false,
_dataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
mapFileResolver,
false,
startTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > endTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > startTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request)
{
var config = request.ToSubscriptionDataConfig();
DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var dataReader = new SubscriptionDataReader(config,
request,
_mapFileProvider,
_factorFileProvider,
_dataCacheProvider,
_dataProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
_mapFileProvider,
request.StartTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out all aux data
if (data.DataType == MarketDataType.Auxiliary)
{
return(false);
}
// filter out future data
if (data.EndTime > request.EndTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > request.StartTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
