/// <summary>
/// Dequeues the current working bar
/// </summary>
/// <param name="utcTriggerTime">The current trigger time in UTC</param>
/// <returns>The base data instance, or null, if no data is to be emitted</returns>
public BaseData TriggerArchive(DateTime utcTriggerTime)
{
BaseData bar;
if (!_queue.TryDequeue(out bar))
{
// if a bar wasn't ready, check for fill forward
if (_previous != null && _config.FillDataForward)
{
// exchanges hours are in local time, so convert to local before checking if exchange is open
var localTriggerTime = utcTriggerTime.ConvertFromUtc(_config.TimeZone);
// only perform fill forward behavior if the exchange is considered open
var barStartTime = localTriggerTime - _increment;
if (_security.Exchange.IsOpenDuringBar(barStartTime, localTriggerTime, _config.ExtendedMarketHours))
{
bar = _previous.Clone(true);
bar.Time = barStartTime.ExchangeRoundDown(_increment, _security.Exchange.Hours, _security.IsExtendedMarketHours);
}
}
}
// we don't have data, so just return null
if (bar == null) return null;
// reset the previous bar for fill forward
_previous = bar.Clone();
return bar;
}
/// <summary>
/// Applies cash settlement rules
/// </summary>
/// <param name="portfolio">The algorithm's portfolio</param>
/// <param name="security">The fill's security</param>
/// <param name="applicationTimeUtc">The fill time (in UTC)</param>
/// <param name="currency">The currency symbol</param>
/// <param name="amount">The amount of cash to apply</param>
public void ApplyFunds(SecurityPortfolioManager portfolio, Security security, DateTime applicationTimeUtc, string currency, decimal amount)
{
if (amount > 0)
{
// positive amount: sell order filled
portfolio.UnsettledCashBook[currency].AddAmount(amount);
// find the correct settlement date (usually T+3 or T+1)
var settlementDate = applicationTimeUtc.ConvertFromUtc(security.Exchange.TimeZone).Date;
for (var i = 0; i < _numberOfDays; i++)
{
settlementDate = settlementDate.AddDays(1);
// only count days when market is open
if (!security.Exchange.Hours.IsDateOpen(settlementDate))
i--;
}
// use correct settlement time
var settlementTimeUtc = settlementDate.Add(_timeOfDay).ConvertToUtc(security.Exchange.Hours.TimeZone);
portfolio.AddUnsettledCashAmount(new UnsettledCashAmount(settlementTimeUtc, currency, amount));
}
else
{
// negative amount: buy order filled
portfolio.CashBook[currency].AddAmount(amount);
}
}
/// <summary>
/// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data
/// </summary>
/// <param name="utcDateTime">The UTC frontier date time</param>
/// <param name="algorithmTimeZone">The algorithm's time zone, required for computing algorithm and slice time</param>
/// <param name="cashBook">The algorithm's cash book, required for generating cash update pairs</param>
/// <param name="data">The data in this <see cref="TimeSlice"/></param>
/// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param>
/// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns>
public static TimeSlice Create(DateTime utcDateTime, DateTimeZone algorithmTimeZone, CashBook cashBook, List<KeyValuePair<Security, List<BaseData>>> data, SecurityChanges changes)
{
int count = 0;
var security = new List<KeyValuePair<Security, BaseData>>();
var custom = new List<KeyValuePair<Security, List<BaseData>>>();
var consolidator = new List<KeyValuePair<SubscriptionDataConfig, List<BaseData>>>();
var allDataForAlgorithm = new List<BaseData>(data.Count);
var cash = new List<KeyValuePair<Cash, BaseData>>(cashBook.Count);
var cashSecurities = new HashSet<Symbol>();
foreach (var cashItem in cashBook.Values)
{
cashSecurities.Add(cashItem.SecuritySymbol);
}
Split split;
Dividend dividend;
Delisting delisting;
SymbolChangedEvent symbolChange;
var algorithmTime = utcDateTime.ConvertFromUtc(algorithmTimeZone);
var tradeBars = new TradeBars(algorithmTime);
var ticks = new Ticks(algorithmTime);
var splits = new Splits(algorithmTime);
var dividends = new Dividends(algorithmTime);
var delistings = new Delistings(algorithmTime);
var symbolChanges = new SymbolChangedEvents(algorithmTime);
foreach (var kvp in data)
{
var list = kvp.Value;
var symbol = kvp.Key.Symbol;
// keep count of all data points
if (list.Count == 1 && list[0] is BaseDataCollection)
{
count += ((BaseDataCollection) list[0]).Data.Count;
}
else
{
count += list.Count;
}
BaseData update = null;
var consolidatorUpdate = new List<BaseData>(list.Count);
for (int i = 0; i < list.Count; i++)
{
var baseData = list[i];
if (!kvp.Key.SubscriptionDataConfig.IsInternalFeed)
{
// this is all the data that goes into the algorithm
allDataForAlgorithm.Add(baseData);
if (kvp.Key.SubscriptionDataConfig.IsCustomData)
{
// this is all the custom data
custom.Add(kvp);
}
}
// don't add internal feed data to ticks/bars objects
if (baseData.DataType != MarketDataType.Auxiliary)
{
if (!kvp.Key.SubscriptionDataConfig.IsInternalFeed)
{
// populate ticks and tradebars dictionaries with no aux data
if (baseData.DataType == MarketDataType.Tick)
{
List<Tick> ticksList;
if (!ticks.TryGetValue(symbol, out ticksList))
{
ticksList = new List<Tick> {(Tick) baseData};
ticks[symbol] = ticksList;
}
ticksList.Add((Tick) baseData);
}
else if (baseData.DataType == MarketDataType.TradeBar)
{
tradeBars[symbol] = (TradeBar) baseData;
}
// this is data used to update consolidators
consolidatorUpdate.Add(baseData);
}
// this is the data used set market prices
update = baseData;
}
// include checks for various aux types so we don't have to construct the dictionaries in Slice
else if ((delisting = baseData as Delisting) != null)
{
delistings[symbol] = delisting;
}
else if ((dividend = baseData as Dividend) != null)
{
dividends[symbol] = dividend;
}
else if ((split = baseData as Split) != null)
{
splits[symbol] = split;
}
else if ((symbolChange = baseData as SymbolChangedEvent) != null)
{
// symbol changes is keyed by the requested symbol
symbolChanges[kvp.Key.SubscriptionDataConfig.Symbol] = symbolChange;
}
}
// check for 'cash securities' if we found valid update data for this symbol
// and we need this data to update cash conversion rates, long term we should
// have Cash hold onto it's security, then he can update himself, or rather, just
// patch through calls to conversion rate to compue it on the fly using Security.Price
if (update != null && cashSecurities.Contains(kvp.Key.Symbol))
{
foreach (var cashKvp in cashBook)
{
if (cashKvp.Value.SecuritySymbol == kvp.Key.Symbol)
{
cash.Add(new KeyValuePair<Cash, BaseData>(cashKvp.Value, update));
}
}
}
security.Add(new KeyValuePair<Security, BaseData>(kvp.Key, update));
consolidator.Add(new KeyValuePair<SubscriptionDataConfig, List<BaseData>>(kvp.Key.SubscriptionDataConfig, consolidatorUpdate));
}
var slice = new Slice(utcDateTime.ConvertFromUtc(algorithmTimeZone), allDataForAlgorithm, tradeBars, ticks, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0);
return new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes);
}
/// <summary>
/// A time period has lapsed, trigger a save/queue of the current value of data.
/// </summary>
/// <param name="utcTriggerTime">The time we're triggering this archive for</param>
/// <param name="fillForward">Data stream is a fillforward type</param>
public void TriggerArchive(DateTime utcTriggerTime, bool fillForward)
{
var localTriggerTime = utcTriggerTime.ConvertFromUtc(_config.TimeZone);
lock (_lock)
{
try
{
//When there's nothing to do:
if (_data == null && !fillForward)
{
Log.Debug("StreamStore.TriggerArchive(): No data to store, and not fill forward: " + Symbol);
}
if (_data != null)
{
//Create clone and reset original
Log.Debug("StreamStore.TriggerArchive(): Enqueued new data: S:" + _data.Symbol + " V:" + _data.Value);
_previousData = _data.Clone();
_queue.Enqueue(_data.Clone());
_data = null;
}
else if (fillForward && _data == null && _previousData != null)
{
// the time is actually the end time of a bar, check to see if the start time
// is within market hours, which is really just checking the _previousData's EndTime
if (!_security.Exchange.IsOpenDuringBar(localTriggerTime - _increment, localTriggerTime, _config.ExtendedMarketHours))
{
Log.Debug("StreamStore.TriggerArchive(): Exchange is closed: " + Symbol);
return;
}
//There was no other data in this timer period, and this is a fillforward subscription:
Log.Debug("StreamStore.TriggerArchive(): Fillforward, Previous Enqueued: S:" + _previousData.Symbol + " V:" + _previousData.Value);
var cloneForward = _previousData.Clone(true);
cloneForward.Time = _previousData.Time.Add(_increment);
_queue.Enqueue(cloneForward);
_previousData = cloneForward.Clone();
}
}
catch (Exception err)
{
Log.Error("StreamStore.TriggerAchive(fillforward): Failed to archive: " + err.Message);
}
}
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(universe.Market, config.Symbol, universe.SecurityType);
var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
IEnumerator<BaseData> enumerator;
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
.Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
.Select(dt => new Tick { Time = dt }).GetEnumerator();
}
else if (config.Type == typeof (CoarseFundamental))
{
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>>
{
{config.SecurityType, new List<Symbol>{config.Symbol}}
});
var enqueable = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
});
enumerator = enqueable;
}
else
{
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
var source = sourceProvider.GetSource(config, currentLocalDate, true);
var factorEnumerator = factory.Read(source).GetEnumerator();
var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(rateLimit);
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
else
{
enqueable.Enqueue(data);
}
});
enumerator = enqueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
return subscription;
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="universe"></param>
/// <param name="security">The security to create a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
{
Subscription subscription = null;
try
{
var config = security.SubscriptionDataConfig;
var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);
IEnumerator<BaseData> enumerator;
if (config.IsCustomData)
{
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseData>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
var source = sourceProvider.GetSource(config, currentLocalDate, true);
var factoryReadEnumerator = factory.Read(source).GetEnumerator();
var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(rateLimit);
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
_exchange.SetHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
// TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
// finally, make our subscriptions aware of the frontier of the data feed, this will help
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionRequest"/> class
/// </summary>
public SubscriptionRequest(bool isUniverseSubscription,
Universe universe,
Security security,
SubscriptionDataConfig configuration,
DateTime startTimeUtc,
DateTime endTimeUtc)
{
IsUniverseSubscription = isUniverseSubscription;
Universe = universe;
Security = security;
Configuration = configuration;
StartTimeUtc = startTimeUtc;
EndTimeUtc = endTimeUtc;
_localStartTime = new Lazy<DateTime>(() => StartTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone));
_localEndTime = new Lazy<DateTime>(() => EndTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone));
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(
IUniverse universe,
DateTime startTimeUtc,
DateTime endTimeUtc
)
{
// grab the relevant exchange hours
SubscriptionDataConfig config = universe.Configuration;
var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder()
.GetExchangeHours(config.Market, null, config.SecurityType);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);
// define our data enumerator
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, false);
// create the subscription
var subscription = new LiveSubscription(universe, security, enumerator, startTimeUtc, endTimeUtc);
// only message the user if it's one of their universe types
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config);
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator<BaseData> enumerator;
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator();
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else if (config.Type == typeof (CoarseFundamental))
{
var cf = new CoarseFundamental();
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// load coarse data day by day
var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate)
let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date
let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false)
let source = cf.GetSource(config, dateInDataTimeZone, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate);
ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2);
enumerator = enqueueable;
}
else
{
// normal reader for all others
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false);
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
_subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
}
/// <summary>
/// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data
/// </summary>
/// <param name="utcDateTime">The UTC frontier date time</param>
/// <param name="algorithmTimeZone">The algorithm's time zone, required for computing algorithm and slice time</param>
/// <param name="cashBook">The algorithm's cash book, required for generating cash update pairs</param>
/// <param name="data">The data in this <see cref="TimeSlice"/></param>
/// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param>
/// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns>
public static TimeSlice Create(DateTime utcDateTime, DateTimeZone algorithmTimeZone, CashBook cashBook, List<DataFeedPacket> data, SecurityChanges changes)
{
int count = 0;
var security = new List<UpdateData<Security>>();
var custom = new List<UpdateData<Security>>();
var consolidator = new List<UpdateData<SubscriptionDataConfig>>();
var allDataForAlgorithm = new List<BaseData>(data.Count);
var cash = new List<UpdateData<Cash>>(cashBook.Count);
var cashSecurities = new HashSet<Symbol>();
foreach (var cashItem in cashBook.Values)
{
cashSecurities.Add(cashItem.SecuritySymbol);
}
Split split;
Dividend dividend;
Delisting delisting;
SymbolChangedEvent symbolChange;
// we need to be able to reference the slice being created in order to define the
// evaluation of option price models, so we define a 'future' that can be referenced
// in the option price model evaluation delegates for each contract
Slice slice = null;
var sliceFuture = new Lazy<Slice>(() => slice);
var algorithmTime = utcDateTime.ConvertFromUtc(algorithmTimeZone);
var tradeBars = new TradeBars(algorithmTime);
var quoteBars = new QuoteBars(algorithmTime);
var ticks = new Ticks(algorithmTime);
var splits = new Splits(algorithmTime);
var dividends = new Dividends(algorithmTime);
var delistings = new Delistings(algorithmTime);
var optionChains = new OptionChains(algorithmTime);
var symbolChanges = new SymbolChangedEvents(algorithmTime);
foreach (var packet in data)
{
var list = packet.Data;
var symbol = packet.Security.Symbol;
if (list.Count == 0) continue;
// keep count of all data points
if (list.Count == 1 && list[0] is BaseDataCollection)
{
var baseDataCollectionCount = ((BaseDataCollection)list[0]).Data.Count;
if (baseDataCollectionCount == 0)
{
continue;
}
count += baseDataCollectionCount;
}
else
{
count += list.Count;
}
if (!packet.Configuration.IsInternalFeed && packet.Configuration.IsCustomData)
{
// This is all the custom data
custom.Add(new UpdateData<Security>(packet.Security, packet.Configuration.Type, list));
}
var securityUpdate = new List<BaseData>(list.Count);
var consolidatorUpdate = new List<BaseData>(list.Count);
for (int i = 0; i < list.Count; i++)
{
var baseData = list[i];
if (!packet.Configuration.IsInternalFeed)
{
// this is all the data that goes into the algorithm
allDataForAlgorithm.Add(baseData);
}
// don't add internal feed data to ticks/bars objects
if (baseData.DataType != MarketDataType.Auxiliary)
{
if (!packet.Configuration.IsInternalFeed)
{
PopulateDataDictionaries(baseData, ticks, tradeBars, quoteBars, optionChains);
// special handling of options data to build the option chain
if (packet.Security.Type == SecurityType.Option)
{
if (baseData.DataType == MarketDataType.OptionChain)
{
optionChains[baseData.Symbol] = (OptionChain) baseData;
}
else if (!HandleOptionData(algorithmTime, baseData, optionChains, packet.Security, sliceFuture))
{
continue;
}
}
// this is data used to update consolidators
consolidatorUpdate.Add(baseData);
}
// this is the data used set market prices
securityUpdate.Add(baseData);
}
// include checks for various aux types so we don't have to construct the dictionaries in Slice
else if ((delisting = baseData as Delisting) != null)
{
delistings[symbol] = delisting;
}
else if ((dividend = baseData as Dividend) != null)
{
dividends[symbol] = dividend;
}
else if ((split = baseData as Split) != null)
{
splits[symbol] = split;
}
else if ((symbolChange = baseData as SymbolChangedEvent) != null)
{
// symbol changes is keyed by the requested symbol
symbolChanges[packet.Configuration.Symbol] = symbolChange;
}
}
if (securityUpdate.Count > 0)
{
// check for 'cash securities' if we found valid update data for this symbol
// and we need this data to update cash conversion rates, long term we should
// have Cash hold onto it's security, then he can update himself, or rather, just
// patch through calls to conversion rate to compue it on the fly using Security.Price
if (cashSecurities.Contains(packet.Security.Symbol))
{
foreach (var cashKvp in cashBook)
{
if (cashKvp.Value.SecuritySymbol == packet.Security.Symbol)
{
var cashUpdates = new List<BaseData> {securityUpdate[securityUpdate.Count - 1]};
cash.Add(new UpdateData<Cash>(cashKvp.Value, packet.Configuration.Type, cashUpdates));
}
}
}
security.Add(new UpdateData<Security>(packet.Security, packet.Configuration.Type, securityUpdate));
}
if (consolidatorUpdate.Count > 0)
{
consolidator.Add(new UpdateData<SubscriptionDataConfig>(packet.Configuration, packet.Configuration.Type, consolidatorUpdate));
}
}
slice = new Slice(algorithmTime, allDataForAlgorithm, tradeBars, quoteBars, ticks, optionChains, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0);
return new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes);
}
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution)
{
var config = security.SubscriptionDataConfig;
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return null;
}
// get the map file resolver for this market
var mapFileResolver = MapFileResolver.Empty;
if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return subscription;
}
/// <summary>
/// Determines whether or not the specified security can be removed from
/// this universe. This is useful to prevent securities from being taken
/// out of a universe before the algorithm has had enough time to make
/// decisions on the security
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="security">The security to check if its ok to remove</param>
/// <returns>True if we can remove the security, false otherwise</returns>
public override bool CanRemoveMember(DateTime utcTime, Security security)
{
// if we haven't begun receiving data for this security then it's safe to remove
var lastData = security.Cache.GetData();
if (lastData == null)
{
return true;
}
// only remove members on day changes, this prevents us from needing to
// fast forward contracts continuously as price moves and out filtered
// contracts change thoughout the day
var localTime = utcTime.ConvertFromUtc(security.Exchange.TimeZone);
if (localTime.Date != lastData.Time.Date)
{
return true;
}
return false;
}
private Subscription CreateSubscription(Universe universe, Security security, DateTime startTimeUtc, DateTime endTimeUtc)
{
var config = security.SubscriptionDataConfig;
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return null;
}
// get the map file resolver for this market
var mapFileResolver = MapFileResolver.Empty;
if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);
// ReSharper disable once PossibleMultipleEnumeration
var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates);
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return subscription;
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator<BaseData> enumerator;
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
.Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
.Select(dt => new Tick {Time = dt}).GetEnumerator();
}
else
{
// normal reader for all others
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, tradeableDates, false);
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
// only message the user if it's one of their universe types
var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental);
PrimeSubscriptionPump(subscription, messageUser);
_subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="security">The security to create a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription)
{
Subscription subscription = null;
try
{
var config = security.SubscriptionDataConfig;
var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone);
var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone);
IEnumerator<BaseData> enumerator;
if (config.IsCustomData)
{
// custom data uses backtest readers
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false);
// apply fast forwarding, this is especially important for RemoteFile types that
// can send in large chunks of old, irrelevant data
var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment);
// apply rate limits (1x per increment, max 30 minutes between calls)
// TODO : Pull limits from config file?
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
// add the enumerator to the exchange
_customExchange.AddEnumerator(rateLimit);
// this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider);
_exchange.SetHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
// TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
// finally, make our subscriptions aware of the frontier of the data feed, this will help
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
/// <summary>
/// Returns an enumerator that defines when this user defined universe will be invoked
/// </summary>
/// <returns>An enumerator of DateTime that defines when this universe will be invoked</returns>
public virtual IEnumerable<DateTime> GetTriggerTimes(DateTime startTimeUtc, DateTime endTimeUtc, MarketHoursDatabase marketHoursDatabase)
{
var exchangeHours = marketHoursDatabase.GetExchangeHours(Configuration);
var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var first = true;
foreach (var dateTime in LinqExtensions.Range(localStartTime, localEndTime, dt => dt + Interval))
{
if (first)
{
yield return dateTime;
first = false;
}
if (exchangeHours.IsOpen(dateTime, dateTime + Interval, Configuration.ExtendedMarketHours))
{
yield return dateTime;
}
}
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(
IUniverse universe,
DateTime startTimeUtc,
DateTime endTimeUtc
)
{
// grab the relevant exchange hours
var config = universe.Configuration;
var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder()
.GetExchangeHours(config.Market, null, config.SecurityType);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
IEnumerator<BaseData> enumerator;
if (config.Type == typeof (CoarseFundamental))
{
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<string>>
{
{config.SecurityType, new List<string>{config.Symbol}}
});
var enqueable = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
});
enumerator = enqueable;
}
else
{
var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);
// define our data enumerator
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true);
_customExchange.AddEnumerator(reader);
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
else
{
enqueable.Enqueue(data);
}
});
enumerator = enqueable;
}
// create the subscription
var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc);
return subscription;
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="universe"></param>
/// <param name="security">The security to create a subscription for</param>
/// <param name="config">The subscription config to be added</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime utcStartTime, DateTime utcEndTime)
{
Subscription subscription = null;
try
{
var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);
IEnumerator<BaseData> enumerator;
if (config.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseData>(() =>
{
var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
var factoryReadEnumerator = factory.Read(source).GetEnumerator();
var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider);
_customExchange.AddEnumerator(config.Symbol, frontierAware);
var enqueable = new EnqueueableEnumerator<BaseData>();
_customExchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
if (config.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}