/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
if (_isLiveMode)
{
// creating trade bar builder enumerator to model underlying price change
var underlyingEnumerator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
// configuring the enumerator
var subscriptionConfiguration = GetSubscriptionConfigurations(request).First();
var subscriptionRequest = new SubscriptionRequest(request, configuration: subscriptionConfiguration);
var configuredEnumerator = _enumeratorConfigurator(subscriptionRequest, underlyingEnumerator);
return(new DataQueueOptionChainUniverseDataCollectionEnumerator(subscriptionRequest, configuredEnumerator, _symbolUniverse, _timeProvider));
}
else
{
var factory = new BaseDataSubscriptionEnumeratorFactory(_isLiveMode, _mapFileResolver, _factorFileProvider);
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(request, factory.CreateEnumerator(sr, dataProvider)));
var sync = new SynchronizingEnumerator(enumerators);
return(new OptionChainUniverseDataCollectionEnumerator(sync, request.Security.Symbol));
}
}
public void WontRemoveEnumeratorsReturningTrueWithCurrentNull()
{
var time = new DateTime(2016, 03, 03, 12, 05, 00);
var stream1 = Enumerable.Range(0, 20)
// return null except the last value and check if its emitted
.Select(x => x == 19 ? new Tick {
Time = time.AddSeconds(x * 100), Quantity = 998877
} : null
).GetEnumerator();
var stream2 = Enumerable.Range(0, 5).Select(x => new Tick {
Time = time.AddSeconds(x * 2)
}).GetEnumerator();
var stream3 = Enumerable.Range(0, 20).Select(x => new Tick {
Time = time.AddSeconds(x * 0.5)
}).GetEnumerator();
var previous = new Tick {
Time = DateTime.MinValue
};
var synchronizer = new SynchronizingEnumerator(stream1, stream2, stream3);
while (synchronizer.MoveNext())
{
Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous.EndTime));
previous = synchronizer.Current as Tick;
}
Assert.AreEqual(998877, previous.Quantity);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
if (_isLiveMode)
{
var localTime = request.StartTimeUtc.ConvertFromUtc(request.Configuration.ExchangeTimeZone);
// loading the list of option contract and converting them into zip entries
var symbols = _symbolUniverse.LookupSymbols(request.Security.Symbol.Underlying.ToString(), request.Security.Type);
var zipEntries = symbols.Select(x => new ZipEntryName {
Time = localTime, Symbol = x
} as BaseData).ToList();
// creating trade bar builder enumerator to model underlying price change
var underlyingEnumerator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
// configuring the enumerator
var subscriptionConfiguration = GetSubscriptionConfigurations(request).First();
var subscriptionRequest = new SubscriptionRequest(request, configuration: subscriptionConfiguration);
var configuredEnumerator = _enumeratorConfigurator(subscriptionRequest, underlyingEnumerator);
return(new DataQueueOptionChainUniverseDataCollectionEnumerator(request.Security.Symbol, configuredEnumerator, zipEntries));
}
else
{
var factory = new BaseDataSubscriptionEnumeratorFactory(_mapFileResolver, _factorFileProvider);
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(request, factory.CreateEnumerator(sr, dataFileProvider)));
var sync = new SynchronizingEnumerator(enumerators);
return(new OptionChainUniverseDataCollectionEnumerator(sync, request.Security.Symbol));
}
}
/// <summary>
/// Step through time and order events and calculate capacity based on the volumes in the minutes surrounding the order.
/// </summary>
/// <returns>Capacity in USD</returns>
private decimal AlgorithmCapacity(
IEnumerable <Order> orders,
IEnumerable <SubscriptionDataConfig> configs,
DateTime start,
DateTime end)
{
var orderSymbols = new HashSet <Symbol>(orders.Select(x => x.Symbol));
foreach (var date in Time.EachDay(start, end))
{
var readers = ReadData(orders, orderSymbols, configs, date);
var dataEnumerators = readers.ToArray();
var feed = new SynchronizingEnumerator(dataEnumerators);
while (feed.MoveNext() && feed.Current != null)
{
var data = feed.Current;
UpdateCurrencyConversionData(data);
DataToAccountCurrency(data);
OnData(data);
}
}
return(Capacity.LastOrDefault()?.y ?? -1);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(request, _factory.CreateEnumerator(sr, dataFileProvider))
);
var sync = new SynchronizingEnumerator(enumerators);
return new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, request.Security.Symbol);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(request, _factory.CreateEnumerator(sr, dataFileProvider))
);
var sync = new SynchronizingEnumerator(enumerators);
return(new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, request.Security.Symbol));
}
public void WillStopIfAllEnumeratorsCurrentIsNullAndReturningFalse()
{
var stream1 = new TestEnumerator {
MoveNextReturnValue = false
};
var synchronizer = new SynchronizingEnumerator(stream1);
while (synchronizer.MoveNext())
{
Assert.Fail();
}
Assert.IsTrue(stream1.MoveNextWasCalled);
Assert.Pass();
}
public void SynchronizesData()
{
var time = new DateTime(2016, 03, 03, 12, 05, 00);
var stream1 = Enumerable.Range(0, 10).Select(x => new Tick {Time = time.AddSeconds(1)}).GetEnumerator();
var stream2 = Enumerable.Range(0, 5).Select(x => new Tick {Time = time.AddSeconds(2)}).GetEnumerator();
var stream3 = Enumerable.Range(0, 20).Select(x => new Tick {Time = time.AddSeconds(0.5)}).GetEnumerator();
var previous = DateTime.MinValue;
var synchronizer = new SynchronizingEnumerator(stream1, stream2, stream3);
while (synchronizer.MoveNext())
{
Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous));
previous = synchronizer.Current.EndTime;
}
}
public static async Task ForEach <TElement>(IEnumerable <TElement> elements, Action <TElement> perform, int mdop = -1)
{
if (mdop == -1)
{
mdop = Environment.ProcessorCount * 2;
}
if (mdop < 2)
{
mdop = 2;
}
SynchronizingEnumerator <TElement> enumerator = new SynchronizingEnumerator <TElement>(elements.GetEnumerator());
Task[] waiters = new Task[mdop - 1];
HashSet <Exception> exceptions = new HashSet <Exception>();
TElement current;
while (enumerator.TryTakeNext(out current))
{
TElement localCurrent = current;
int slot = await FindFreeSlotAsync(waiters, exceptions);
waiters[slot] = Task.Run(() => perform(localCurrent));
}
try
{
await Task.WhenAll(waiters.Where(x => x != null));
}
catch (Exception ex)
{
exceptions.Add(ex);
}
if (exceptions.Count > 0)
{
throw new AggregateException(exceptions);
}
}
public void WillRemoveEnumeratorsReturningFalse()
{
var time = new DateTime(2016, 03, 03, 12, 05, 00);
var stream1 = new TestEnumerator {
MoveNextReturnValue = false
};
var stream2 = Enumerable.Range(0, 10).Select(x => new Tick {
Time = time.AddSeconds(x * 2)
}).GetEnumerator();
var synchronizer = new SynchronizingEnumerator(stream1, stream2);
var emitted = false;
while (synchronizer.MoveNext())
{
emitted = true;
}
Assert.IsTrue(emitted);
Assert.IsTrue(stream1.MoveNextWasCalled);
Assert.AreEqual(1, stream1.MoveNextCallCount);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
if (_isLiveMode)
{
var subscriptionConfiguration = GetSubscriptionConfigurations(request).First();
var subscriptionRequest = new SubscriptionRequest(request, configuration: subscriptionConfiguration);
return(new DataQueueFuturesChainUniverseDataCollectionEnumerator(subscriptionRequest, _symbolUniverse, _timeProvider));
}
else
{
var factory = new BaseDataSubscriptionEnumeratorFactory(_isLiveMode);
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(request, factory.CreateEnumerator(sr, dataProvider)));
var sync = new SynchronizingEnumerator(enumerators);
return(new FuturesChainUniverseDataCollectionAggregatorEnumerator(sync, request.Security.Symbol));
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
if (_isLiveMode)
{
// configuring the enumerator
var subscriptionConfiguration = GetSubscriptionConfigurations(request).First();
var subscriptionRequest = new SubscriptionRequest(request, configuration: subscriptionConfiguration);
var configuredEnumerator = _enumeratorConfigurator(subscriptionRequest);
return(new DataQueueOptionChainUniverseDataCollectionEnumerator(subscriptionRequest, configuredEnumerator, _symbolUniverse, _timeProvider));
}
else
{
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(sr));
var sync = new SynchronizingEnumerator(enumerators);
return(new OptionChainUniverseDataCollectionEnumerator(sync, request.Security.Symbol));
}
}
public void SynchronizesData()
{
var time = new DateTime(2016, 03, 03, 12, 05, 00);
var stream1 = Enumerable.Range(0, 10).Select(x => new Tick {
Time = time.AddSeconds(1)
}).GetEnumerator();
var stream2 = Enumerable.Range(0, 5).Select(x => new Tick {
Time = time.AddSeconds(2)
}).GetEnumerator();
var stream3 = Enumerable.Range(0, 20).Select(x => new Tick {
Time = time.AddSeconds(0.5)
}).GetEnumerator();
var previous = DateTime.MinValue;
var synchronizer = new SynchronizingEnumerator(stream1, stream2, stream3);
while (synchronizer.MoveNext())
{
Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous));
previous = synchronizer.Current.EndTime;
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
if (_isLiveMode)
{
var localTime = request.StartTimeUtc.ConvertFromUtc(request.Configuration.ExchangeTimeZone);
// loading the list of futures contracts and converting them into zip entries
var symbols = _symbolUniverse.LookupSymbols(request.Security.Symbol.ID.Symbol, request.Security.Type);
var zipEntries = symbols.Select(x => new ZipEntryName {
Time = localTime, Symbol = x
} as BaseData).ToList();
var underlyingEnumerator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
underlyingEnumerator.ProcessData(new Tick {
Value = 0
});
// configuring the enumerator
var subscriptionConfiguration = GetSubscriptionConfigurations(request).First();
var subscriptionRequest = new SubscriptionRequest(request, configuration: subscriptionConfiguration);
var configuredEnumerator = _enumeratorConfigurator(subscriptionRequest, underlyingEnumerator);
return(new DataQueueFuturesChainUniverseDataCollectionEnumerator(request.Security.Symbol, configuredEnumerator, zipEntries));
}
else
{
var factory = new BaseDataSubscriptionEnumeratorFactory(_isLiveMode);
var enumerators = GetSubscriptionConfigurations(request)
.Select(c => new SubscriptionRequest(request, configuration: c))
.Select(sr => _enumeratorConfigurator(request, factory.CreateEnumerator(sr, dataProvider)));
var sync = new SynchronizingEnumerator(enumerators);
return(new FuturesChainUniverseDataCollectionAggregatorEnumerator(sync, request.Security.Symbol));
}
}
/// <summary>
/// Starts data generation
/// </summary>
public void Run()
{
var tickTypesPerSecurityType = SubscriptionManager.DefaultDataTypes();
// can specify a seed value in this ctor if determinism is desired
var random = new Random();
var randomValueGenerator = new RandomValueGenerator();
if (_settings.RandomSeedSet)
{
random = new Random(_settings.RandomSeed);
randomValueGenerator = new RandomValueGenerator(_settings.RandomSeed);
}
var symbolGenerator = BaseSymbolGenerator.Create(_settings, randomValueGenerator);
var maxSymbolCount = symbolGenerator.GetAvailableSymbolCount();
if (_settings.SymbolCount > maxSymbolCount)
{
Log.Error($"RandomDataGenerator.Run(): Limiting Symbol count to {maxSymbolCount}, we don't have more {_settings.SecurityType} tickers for {_settings.Market}");
_settings.SymbolCount = maxSymbolCount;
}
Log.Trace($"RandomDataGenerator.Run(): Begin data generation of {_settings.SymbolCount} randomly generated {_settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
foreach (var(symbolRef, currentSymbolGroup) in symbolGenerator.GenerateRandomSymbols()
.GroupBy(s => s.HasUnderlying ? s.Underlying : s)
.Select(g => (g.Key, g.OrderBy(s => s.HasUnderlying).ToList())))
{
Log.Trace($"RandomDataGenerator.Run(): Symbol[{++count}]: {symbolRef} Progress: {progress:0.0}% - Generating data...");
var tickGenerators = new List <IEnumerator <Tick> >();
var tickHistories = new Dictionary <Symbol, List <Tick> >();
Security underlyingSecurity = null;
foreach (var currentSymbol in currentSymbolGroup)
{
if (!_securityManager.TryGetValue(currentSymbol, out var security))
{
security = _securityManager.CreateSecurity(
currentSymbol,
new List <SubscriptionDataConfig>(),
underlying: underlyingSecurity);
_securityManager.Add(security);
}
underlyingSecurity ??= security;
tickGenerators.Add(
new TickGenerator(_settings, tickTypesPerSecurityType[currentSymbol.SecurityType].ToArray(), security, randomValueGenerator)
.GenerateTicks()
.GetEnumerator());
tickHistories.Add(
currentSymbol,
new List <Tick>());
}
using var sync = new SynchronizingEnumerator(tickGenerators);
while (sync.MoveNext())
{
var dataPoint = sync.Current;
if (!_securityManager.TryGetValue(dataPoint.Symbol, out var security))
{
Log.Error($"RandomDataGenerator.Run(): Could not find security for symbol {sync.Current.Symbol}");
continue;
}
tickHistories[security.Symbol].Add(dataPoint as Tick);
security.Update(new List <BaseData> {
dataPoint
}, dataPoint.GetType(), false);
}
foreach (var(currentSymbol, tickHistory) in tickHistories)
{
var symbol = currentSymbol;
// This is done so that we can update the Symbol in the case of a rename event
var delistDate = GetDelistingDate(_settings.Start, _settings.End, randomValueGenerator);
var willBeDelisted = randomValueGenerator.NextBool(1.0);
// Companies rarely IPO then disappear within 6 months
if (willBeDelisted && tickHistory.Select(tick => tick.Time.Month).Distinct().Count() <= 6)
{
willBeDelisted = false;
}
var dividendsSplitsMaps = new DividendSplitMapGenerator(
symbol,
_settings,
randomValueGenerator,
symbolGenerator,
random,
delistDate,
willBeDelisted);
// Keep track of renamed symbols and the time they were renamed.
var renamedSymbols = new Dictionary <Symbol, DateTime>();
if (_settings.SecurityType == SecurityType.Equity)
{
dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
if (!willBeDelisted)
{
dividendsSplitsMaps.DividendsSplits.Add(new CorporateFactorRow(new DateTime(2050, 12, 31), 1m, 1m));
if (dividendsSplitsMaps.MapRows.Count > 1)
{
// Remove the last element if we're going to have a 20501231 entry
dividendsSplitsMaps.MapRows.RemoveAt(dividendsSplitsMaps.MapRows.Count - 1);
}
dividendsSplitsMaps.MapRows.Add(new MapFileRow(new DateTime(2050, 12, 31), dividendsSplitsMaps.CurrentSymbol.Value));
}
// If the Symbol value has changed, update the current Symbol
if (symbol != dividendsSplitsMaps.CurrentSymbol)
{
// Add all Symbol rename events to dictionary
// We skip the first row as it contains the listing event instead of a rename event
foreach (var renameEvent in dividendsSplitsMaps.MapRows.Skip(1))
{
// Symbol.UpdateMappedSymbol does not update the underlying security ID Symbol, which
// is used to create the hash code. Create a new equity Symbol from scratch instead.
symbol = Symbol.Create(renameEvent.MappedSymbol, SecurityType.Equity, _settings.Market);
renamedSymbols.Add(symbol, renameEvent.Date);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} will be renamed on {renameEvent.Date}");
}
}
else
{
// This ensures that ticks will be written for the current Symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
symbol = dividendsSplitsMaps.CurrentSymbol;
// Write Splits and Dividend events to directory factor_files
var factorFile = new CorporateFactorProvider(symbol.Value, dividendsSplitsMaps.DividendsSplits, _settings.Start);
var mapFile = new MapFile(symbol.Value, dividendsSplitsMaps.MapRows);
factorFile.WriteToFile(symbol);
mapFile.WriteToCsv(_settings.Market, symbol.SecurityType);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} Dividends, splits, and map files have been written to disk.");
}
else
{
// This ensures that ticks will be written for the current Symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
// define aggregators via settings
var aggregators = CreateAggregators(_settings, tickTypesPerSecurityType[currentSymbol.SecurityType].ToArray()).ToList();
Symbol previousSymbol = null;
var currentCount = 0;
var monthsTrading = 0;
foreach (var renamed in renamedSymbols)
{
var previousRenameDate = previousSymbol == null ? new DateTime(1, 1, 1) : renamedSymbols[previousSymbol];
var previousRenameDateDay = new DateTime(previousRenameDate.Year, previousRenameDate.Month, previousRenameDate.Day);
var renameDate = renamed.Value;
var renameDateDay = new DateTime(renameDate.Year, renameDate.Month, renameDate.Day);
foreach (var tick in tickHistory.Where(tick => tick.Time >= previousRenameDate && previousRenameDateDay != TickDay(tick)))
{
// Prevents the aggregator from being updated with ticks after the rename event
if (TickDay(tick) > renameDateDay)
{
break;
}
if (tick.Time.Month != previousMonth)
{
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: Month: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
monthsTrading++;
}
foreach (var item in aggregators)
{
tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
item.Consolidator.Update(tick);
}
if (monthsTrading >= 6 && willBeDelisted && tick.Time > delistDate)
{
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {renamed.Key} delisted at {tick.Time:MMMM yyyy}");
break;
}
}
// count each stage as a point, so total points is 2*Symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * _settings.SymbolCount);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {renamed.Key} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, renamed.Key, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
// Filter out the data we're going to write here because filtering them in the consolidator update phase
// makes it write all dates for some unknown reason
writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != DataDay(data)));
}
// update progress
progress = 100 * (2 * count) / (2.0 * _settings.SymbolCount);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
previousSymbol = renamed.Key;
currentCount++;
}
}
}
Log.Trace("RandomDataGenerator.Run(): Random data generation has completed.");
DateTime TickDay(Tick tick) => new(tick.Time.Year, tick.Time.Month, tick.Time.Day);
DateTime DataDay(BaseData data) => new(data.Time.Year, data.Time.Month, data.Time.Day);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator <BaseData> enumerator;
var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(x => new Tick {
Time = x, Symbol = config.Symbol
}).GetEnumerator();
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
var cf = new CoarseFundamental();
// load coarse data day by day
enumerator = (from date in Time.EachTradeableDayInTimeZone(security.Exchange.Hours, _algorithm.StartDate, _algorithm.EndDate, config.DataTimeZone, config.ExtendedMarketHours)
let source = cf.GetSource(config, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, config, date, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date.AddDays(1), config.Symbol, coarseFundamentalForDate)
).GetEnumerator();
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
ScheduleEnumerator(enumerator, enqueueable, 5, 100000, 2);
enumerator = enqueueable;
}
else if (config.SecurityType == SecurityType.Option && security is Option)
{
var configs = universe.GetSubscriptions(security);
var enumerators = configs.Select(c =>
CreateSubscriptionEnumerator(security, c, localStartTime, localEndTime, _mapFileProvider.Get(c.Market), tradeableDates, false, true)
).ToList();
var sync = new SynchronizingEnumerator(enumerators);
enumerator = new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, config.Symbol);
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else
{
// normal reader for all others
enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, MapFileResolver.Empty, tradeableDates, true, false);
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
_subscriptions.TryAdd(subscription);
UpdateFillForwardResolution();
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance
);
var mapFileResolver = config.SecurityType == SecurityType.Equity
? _mapFileProvider.Get(config.Market)
: MapFileResolver.Empty;
var dataReader = new SubscriptionDataReader(config,
start,
end,
mapFileResolver,
_factorFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
_dataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); };
var enumerator = CorporateEventEnumeratorFactory.CreateEnumerators(
config,
_factorFileProvider,
dataReader,
mapFileResolver,
false);
IEnumerator <BaseData> reader = new SynchronizingEnumerator(dataReader, enumerator);
// has to be initialized after adding all the enumerators since it will execute a MoveNext
dataReader.Initialize();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
