/// <summary> /// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data /// </summary> /// <param name="utcDateTime">The UTC frontier date time</param> /// <param name="algorithmTimeZone">The algorithm's time zone, required for computing algorithm and slice time</param> /// <param name="cashBook">The algorithm's cash book, required for generating cash update pairs</param> /// <param name="data">The data in this <see cref="TimeSlice"/></param> /// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param> /// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns> public static TimeSlice Create(DateTime utcDateTime, DateTimeZone algorithmTimeZone, CashBook cashBook, List <DataFeedPacket> data, SecurityChanges changes) { int count = 0; var security = new List <UpdateData <Security> >(); var custom = new List <UpdateData <Security> >(); var consolidator = new List <UpdateData <SubscriptionDataConfig> >(); var allDataForAlgorithm = new List <BaseData>(data.Count); var cash = new List <UpdateData <Cash> >(cashBook.Count); var cashSecurities = new HashSet <Symbol>(); foreach (var cashItem in cashBook.Values) { cashSecurities.Add(cashItem.SecuritySymbol); } Split split; Dividend dividend; Delisting delisting; SymbolChangedEvent symbolChange; // we need to be able to reference the slice being created in order to define the // evaluation of option price models, so we define a 'future' that can be referenced // in the option price model evaluation delegates for each contract Slice slice = null; var sliceFuture = new Lazy <Slice>(() => slice); var algorithmTime = utcDateTime.ConvertFromUtc(algorithmTimeZone); var tradeBars = new TradeBars(algorithmTime); var quoteBars = new QuoteBars(algorithmTime); var ticks = new Ticks(algorithmTime); var splits = new Splits(algorithmTime); var dividends = new Dividends(algorithmTime); var delistings = new Delistings(algorithmTime); var optionChains = new OptionChains(algorithmTime); var futuresChains = new FuturesChains(algorithmTime); var symbolChanges = new SymbolChangedEvents(algorithmTime); foreach (var packet in data) { var list = packet.Data; var symbol = packet.Security.Symbol; if (list.Count == 0) { continue; } // keep count of all data points if (list.Count == 1 && list[0] is BaseDataCollection) { var baseDataCollectionCount = ((BaseDataCollection)list[0]).Data.Count; if (baseDataCollectionCount == 0) { continue; } count += baseDataCollectionCount; } else { count += list.Count; } if (!packet.Configuration.IsInternalFeed && packet.Configuration.IsCustomData) { // This is all the custom data custom.Add(new UpdateData <Security>(packet.Security, packet.Configuration.Type, list)); } var securityUpdate = new List <BaseData>(list.Count); var consolidatorUpdate = new List <BaseData>(list.Count); for (int i = 0; i < list.Count; i++) { var baseData = list[i]; if (!packet.Configuration.IsInternalFeed) { // this is all the data that goes into the algorithm allDataForAlgorithm.Add(baseData); } // don't add internal feed data to ticks/bars objects if (baseData.DataType != MarketDataType.Auxiliary) { if (!packet.Configuration.IsInternalFeed) { PopulateDataDictionaries(baseData, ticks, tradeBars, quoteBars, optionChains, futuresChains); // special handling of options data to build the option chain if (packet.Security.Type == SecurityType.Option) { if (baseData.DataType == MarketDataType.OptionChain) { optionChains[baseData.Symbol] = (OptionChain)baseData; } else if (!HandleOptionData(algorithmTime, baseData, optionChains, packet.Security, sliceFuture)) { continue; } } // special handling of futures data to build the futures chain if (packet.Security.Type == SecurityType.Future) { if (baseData.DataType == MarketDataType.FuturesChain) { futuresChains[baseData.Symbol] = (FuturesChain)baseData; } else if (!HandleFuturesData(algorithmTime, baseData, futuresChains, packet.Security)) { continue; } } // this is data used to update consolidators consolidatorUpdate.Add(baseData); } // this is the data used set market prices securityUpdate.Add(baseData); } // include checks for various aux types so we don't have to construct the dictionaries in Slice else if ((delisting = baseData as Delisting) != null) { delistings[symbol] = delisting; } else if ((dividend = baseData as Dividend) != null) { dividends[symbol] = dividend; } else if ((split = baseData as Split) != null) { splits[symbol] = split; } else if ((symbolChange = baseData as SymbolChangedEvent) != null) { // symbol changes is keyed by the requested symbol symbolChanges[packet.Configuration.Symbol] = symbolChange; } } if (securityUpdate.Count > 0) { // check for 'cash securities' if we found valid update data for this symbol // and we need this data to update cash conversion rates, long term we should // have Cash hold onto it's security, then he can update himself, or rather, just // patch through calls to conversion rate to compue it on the fly using Security.Price if (cashSecurities.Contains(packet.Security.Symbol)) { foreach (var cashKvp in cashBook) { if (cashKvp.Value.SecuritySymbol == packet.Security.Symbol) { var cashUpdates = new List <BaseData> { securityUpdate[securityUpdate.Count - 1] }; cash.Add(new UpdateData <Cash>(cashKvp.Value, packet.Configuration.Type, cashUpdates)); } } } security.Add(new UpdateData <Security>(packet.Security, packet.Configuration.Type, securityUpdate)); } if (consolidatorUpdate.Count > 0) { consolidator.Add(new UpdateData <SubscriptionDataConfig>(packet.Configuration, packet.Configuration.Type, consolidatorUpdate)); } } slice = new Slice(algorithmTime, allDataForAlgorithm, tradeBars, quoteBars, ticks, optionChains, futuresChains, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0); return(new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes)); }
/// <summary> /// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data /// </summary> /// <param name="utcDateTime">The UTC frontier date time</param> /// <param name="data">The data in this <see cref="TimeSlice"/></param> /// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param> /// <param name="universeData"></param> /// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns> public TimeSlice Create(DateTime utcDateTime, List <DataFeedPacket> data, SecurityChanges changes, Dictionary <Universe, BaseDataCollection> universeData) { int count = 0; var security = new List <UpdateData <ISecurityPrice> >(); var custom = new List <UpdateData <ISecurityPrice> >(); var consolidator = new List <UpdateData <SubscriptionDataConfig> >(); var allDataForAlgorithm = new List <BaseData>(data.Count); var optionUnderlyingUpdates = new Dictionary <Symbol, BaseData>(); Split split; Dividend dividend; Delisting delisting; SymbolChangedEvent symbolChange; // we need to be able to reference the slice being created in order to define the // evaluation of option price models, so we define a 'future' that can be referenced // in the option price model evaluation delegates for each contract Slice slice = null; var sliceFuture = new Lazy <Slice>(() => slice); var algorithmTime = utcDateTime.ConvertFromUtc(_timeZone); var tradeBars = new TradeBars(algorithmTime); var quoteBars = new QuoteBars(algorithmTime); var ticks = new Ticks(algorithmTime); var splits = new Splits(algorithmTime); var dividends = new Dividends(algorithmTime); var delistings = new Delistings(algorithmTime); var optionChains = new OptionChains(algorithmTime); var futuresChains = new FuturesChains(algorithmTime); var symbolChanges = new SymbolChangedEvents(algorithmTime); if (universeData.Count > 0) { // count universe data foreach (var kvp in universeData) { count += kvp.Value.Data.Count; } } // ensure we read equity data before option data, so we can set the current underlying price foreach (var packet in data) { // filter out packets for removed subscriptions if (packet.IsSubscriptionRemoved) { continue; } var list = packet.Data; var symbol = packet.Configuration.Symbol; if (list.Count == 0) { continue; } // keep count of all data points if (list.Count == 1 && list[0] is BaseDataCollection) { var baseDataCollectionCount = ((BaseDataCollection)list[0]).Data.Count; if (baseDataCollectionCount == 0) { continue; } count += baseDataCollectionCount; } else { count += list.Count; } if (!packet.Configuration.IsInternalFeed && packet.Configuration.IsCustomData) { // This is all the custom data custom.Add(new UpdateData <ISecurityPrice>(packet.Security, packet.Configuration.Type, list)); } var securityUpdate = new List <BaseData>(list.Count); var consolidatorUpdate = new List <BaseData>(list.Count); for (int i = 0; i < list.Count; i++) { var baseData = list[i]; if (!packet.Configuration.IsInternalFeed) { // this is all the data that goes into the algorithm allDataForAlgorithm.Add(baseData); } // don't add internal feed data to ticks/bars objects if (baseData.DataType != MarketDataType.Auxiliary) { if (!packet.Configuration.IsInternalFeed) { PopulateDataDictionaries(baseData, ticks, tradeBars, quoteBars, optionChains, futuresChains); // special handling of options data to build the option chain if (symbol.SecurityType == SecurityType.Option) { if (baseData.DataType == MarketDataType.OptionChain) { optionChains[baseData.Symbol] = (OptionChain)baseData; } else if (!HandleOptionData(algorithmTime, baseData, optionChains, packet.Security, sliceFuture, optionUnderlyingUpdates)) { continue; } } // special handling of futures data to build the futures chain if (symbol.SecurityType == SecurityType.Future) { if (baseData.DataType == MarketDataType.FuturesChain) { futuresChains[baseData.Symbol] = (FuturesChain)baseData; } else if (!HandleFuturesData(algorithmTime, baseData, futuresChains, packet.Security)) { continue; } } // this is data used to update consolidators consolidatorUpdate.Add(baseData); } // this is the data used set market prices // do not add it if it is a Suspicious tick var tick = baseData as Tick; if (tick != null && tick.Suspicious) { continue; } securityUpdate.Add(baseData); // option underlying security update if (symbol.SecurityType == SecurityType.Equity) { optionUnderlyingUpdates[symbol] = baseData; } } // include checks for various aux types so we don't have to construct the dictionaries in Slice else if ((delisting = baseData as Delisting) != null) { delistings[symbol] = delisting; } else if ((dividend = baseData as Dividend) != null) { dividends[symbol] = dividend; } else if ((split = baseData as Split) != null) { splits[symbol] = split; } else if ((symbolChange = baseData as SymbolChangedEvent) != null) { // symbol changes is keyed by the requested symbol symbolChanges[packet.Configuration.Symbol] = symbolChange; } } if (securityUpdate.Count > 0) { security.Add(new UpdateData <ISecurityPrice>(packet.Security, packet.Configuration.Type, securityUpdate)); } if (consolidatorUpdate.Count > 0) { consolidator.Add(new UpdateData <SubscriptionDataConfig>(packet.Configuration, packet.Configuration.Type, consolidatorUpdate)); } } slice = new Slice(algorithmTime, allDataForAlgorithm, tradeBars, quoteBars, ticks, optionChains, futuresChains, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0); return(new TimeSlice(utcDateTime, count, slice, data, security, consolidator, custom, changes, universeData)); }
/// <summary> /// Adds the specified <see cref="BaseData"/> instance to the appropriate <see cref="DataDictionary{T}"/> /// </summary> private void PopulateDataDictionaries(BaseData baseData, Ticks ticks, TradeBars tradeBars, QuoteBars quoteBars, OptionChains optionChains, FuturesChains futuresChains) { var symbol = baseData.Symbol; // populate data dictionaries switch (baseData.DataType) { case MarketDataType.Tick: ticks.Add(symbol, (Tick)baseData); break; case MarketDataType.TradeBar: tradeBars[symbol] = (TradeBar)baseData; break; case MarketDataType.QuoteBar: quoteBars[symbol] = (QuoteBar)baseData; break; case MarketDataType.OptionChain: optionChains[symbol] = (OptionChain)baseData; break; case MarketDataType.FuturesChain: futuresChains[symbol] = (FuturesChain)baseData; break; } }
private bool HandleFuturesData(DateTime algorithmTime, BaseData baseData, FuturesChains futuresChains, ISecurityPrice security) { var symbol = baseData.Symbol; FuturesChain chain; var canonical = Symbol.Create(symbol.ID.Symbol, SecurityType.Future, symbol.ID.Market); if (!futuresChains.TryGetValue(canonical, out chain)) { chain = new FuturesChain(canonical, algorithmTime); futuresChains[canonical] = chain; } var universeData = baseData as FuturesChainUniverseDataCollection; if (universeData != null) { foreach (var contractSymbol in universeData.FilteredContracts) { chain.FilteredContracts.Add(contractSymbol); } return(false); } FuturesContract contract; if (!chain.Contracts.TryGetValue(baseData.Symbol, out contract)) { var underlyingSymbol = baseData.Symbol.Underlying; contract = new FuturesContract(baseData.Symbol, underlyingSymbol) { Time = baseData.EndTime, LastPrice = security.Close, Volume = (long)security.Volume, BidPrice = security.BidPrice, BidSize = (long)security.BidSize, AskPrice = security.AskPrice, AskSize = (long)security.AskSize, OpenInterest = security.OpenInterest }; chain.Contracts[baseData.Symbol] = contract; } // populate ticks and tradebars dictionaries with no aux data switch (baseData.DataType) { case MarketDataType.Tick: var tick = (Tick)baseData; chain.Ticks.Add(tick.Symbol, tick); UpdateContract(contract, tick); break; case MarketDataType.TradeBar: var tradeBar = (TradeBar)baseData; chain.TradeBars[symbol] = tradeBar; UpdateContract(contract, tradeBar); break; case MarketDataType.QuoteBar: var quote = (QuoteBar)baseData; chain.QuoteBars[symbol] = quote; UpdateContract(contract, quote); break; case MarketDataType.Base: chain.AddAuxData(baseData); break; } return(true); }
/// <summary> /// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data /// </summary> /// <param name="utcDateTime">The UTC frontier date time</param> /// <param name="data">The data in this <see cref="TimeSlice"/></param> /// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param> /// <param name="universeData"></param> /// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns> public TimeSlice Create(DateTime utcDateTime, List <DataFeedPacket> data, SecurityChanges changes, Dictionary <Universe, BaseDataCollection> universeData) { int count = 0; var security = new List <UpdateData <ISecurityPrice> >(data.Count); List <UpdateData <ISecurityPrice> > custom = null; var consolidator = new List <UpdateData <SubscriptionDataConfig> >(data.Count); var allDataForAlgorithm = new List <BaseData>(data.Count); var optionUnderlyingUpdates = new Dictionary <Symbol, BaseData>(); Split split; Dividend dividend; Delisting delisting; SymbolChangedEvent symbolChange; // we need to be able to reference the slice being created in order to define the // evaluation of option price models, so we define a 'future' that can be referenced // in the option price model evaluation delegates for each contract Slice slice = null; var sliceFuture = new Lazy <Slice>(() => slice); var algorithmTime = utcDateTime.ConvertFromUtc(_timeZone); TradeBars tradeBars = null; QuoteBars quoteBars = null; Ticks ticks = null; Splits splits = null; Dividends dividends = null; Delistings delistings = null; OptionChains optionChains = null; FuturesChains futuresChains = null; SymbolChangedEvents symbolChanges = null; UpdateEmptyCollections(algorithmTime); if (universeData.Count > 0) { // count universe data foreach (var kvp in universeData) { count += kvp.Value.Data.Count; } } // ensure we read equity data before option data, so we can set the current underlying price foreach (var packet in data) { // filter out packets for removed subscriptions if (packet.IsSubscriptionRemoved) { continue; } var list = packet.Data; var symbol = packet.Configuration.Symbol; if (list.Count == 0) { continue; } // keep count of all data points if (list.Count == 1 && list[0] is BaseDataCollection) { var baseDataCollectionCount = ((BaseDataCollection)list[0]).Data.Count; if (baseDataCollectionCount == 0) { continue; } count += baseDataCollectionCount; } else { count += list.Count; } if (!packet.Configuration.IsInternalFeed && packet.Configuration.IsCustomData) { if (custom == null) { custom = new List <UpdateData <ISecurityPrice> >(1); } // This is all the custom data custom.Add(new UpdateData <ISecurityPrice>(packet.Security, packet.Configuration.Type, list, packet.Configuration.IsInternalFeed)); } var securityUpdate = new List <BaseData>(list.Count); var consolidatorUpdate = new List <BaseData>(list.Count); var containsFillForwardData = false; for (var i = 0; i < list.Count; i++) { var baseData = list[i]; if (!packet.Configuration.IsInternalFeed) { // this is all the data that goes into the algorithm allDataForAlgorithm.Add(baseData); } containsFillForwardData |= baseData.IsFillForward; // don't add internal feed data to ticks/bars objects if (baseData.DataType != MarketDataType.Auxiliary) { var tick = baseData as Tick; if (!packet.Configuration.IsInternalFeed) { // populate data dictionaries switch (baseData.DataType) { case MarketDataType.Tick: if (ticks == null) { ticks = new Ticks(algorithmTime); } ticks.Add(baseData.Symbol, (Tick)baseData); break; case MarketDataType.TradeBar: if (tradeBars == null) { tradeBars = new TradeBars(algorithmTime); } var newTradeBar = (TradeBar)baseData; TradeBar existingTradeBar; // if we have an existing bar keep the highest resolution one // e.g Hour and Minute resolution subscriptions for the same symbol // see CustomUniverseWithBenchmarkRegressionAlgorithm if (!tradeBars.TryGetValue(baseData.Symbol, out existingTradeBar) || existingTradeBar.Period > newTradeBar.Period) { tradeBars[baseData.Symbol] = newTradeBar; } break; case MarketDataType.QuoteBar: if (quoteBars == null) { quoteBars = new QuoteBars(algorithmTime); } var newQuoteBar = (QuoteBar)baseData; QuoteBar existingQuoteBar; // if we have an existing bar keep the highest resolution one // e.g Hour and Minute resolution subscriptions for the same symbol // see CustomUniverseWithBenchmarkRegressionAlgorithm if (!quoteBars.TryGetValue(baseData.Symbol, out existingQuoteBar) || existingQuoteBar.Period > newQuoteBar.Period) { quoteBars[baseData.Symbol] = newQuoteBar; } break; case MarketDataType.OptionChain: if (optionChains == null) { optionChains = new OptionChains(algorithmTime); } optionChains[baseData.Symbol] = (OptionChain)baseData; break; case MarketDataType.FuturesChain: if (futuresChains == null) { futuresChains = new FuturesChains(algorithmTime); } futuresChains[baseData.Symbol] = (FuturesChain)baseData; break; } // this is data used to update consolidators // do not add it if it is a Suspicious tick if (tick == null || !tick.Suspicious) { consolidatorUpdate.Add(baseData); } } // special handling of options data to build the option chain if (symbol.SecurityType == SecurityType.Option || symbol.SecurityType == SecurityType.FutureOption) { // internal feeds, like open interest, will not create the chain but will update it if it exists // this is because the open interest could arrive at some closed market hours in which there is no other data and we don't // want to generate a chain object in this case if (optionChains == null && !packet.Configuration.IsInternalFeed) { optionChains = new OptionChains(algorithmTime); } if (baseData.DataType == MarketDataType.OptionChain) { optionChains[baseData.Symbol] = (OptionChain)baseData; } else if (optionChains != null && !HandleOptionData(algorithmTime, baseData, optionChains, packet.Security, sliceFuture, optionUnderlyingUpdates)) { continue; } } // special handling of futures data to build the futures chain if (symbol.SecurityType == SecurityType.Future) { // internal feeds, like open interest, will not create the chain but will update it if it exists // this is because the open interest could arrive at some closed market hours in which there is no other data and we don't // want to generate a chain object in this case if (futuresChains == null && !packet.Configuration.IsInternalFeed) { futuresChains = new FuturesChains(algorithmTime); } if (baseData.DataType == MarketDataType.FuturesChain) { futuresChains[baseData.Symbol] = (FuturesChain)baseData; } else if (futuresChains != null && !HandleFuturesData(algorithmTime, baseData, futuresChains, packet.Security)) { continue; } } // this is the data used set market prices // do not add it if it is a Suspicious tick if (tick != null && tick.Suspicious) { continue; } securityUpdate.Add(baseData); // option underlying security update if (!packet.Configuration.IsInternalFeed) { optionUnderlyingUpdates[symbol] = baseData; } } else if (!packet.Configuration.IsInternalFeed) { // include checks for various aux types so we don't have to construct the dictionaries in Slice if ((delisting = baseData as Delisting) != null) { if (delistings == null) { delistings = new Delistings(algorithmTime); } delistings[symbol] = delisting; } else if ((dividend = baseData as Dividend) != null) { if (dividends == null) { dividends = new Dividends(algorithmTime); } dividends[symbol] = dividend; } else if ((split = baseData as Split) != null) { if (splits == null) { splits = new Splits(algorithmTime); } splits[symbol] = split; } else if ((symbolChange = baseData as SymbolChangedEvent) != null) { if (symbolChanges == null) { symbolChanges = new SymbolChangedEvents(algorithmTime); } // symbol changes is keyed by the requested symbol symbolChanges[packet.Configuration.Symbol] = symbolChange; } } } if (securityUpdate.Count > 0) { security.Add(new UpdateData <ISecurityPrice>(packet.Security, packet.Configuration.Type, securityUpdate, packet.Configuration.IsInternalFeed, containsFillForwardData)); } if (consolidatorUpdate.Count > 0) { consolidator.Add(new UpdateData <SubscriptionDataConfig>(packet.Configuration, packet.Configuration.Type, consolidatorUpdate, packet.Configuration.IsInternalFeed, containsFillForwardData)); } } slice = new Slice(algorithmTime, allDataForAlgorithm, tradeBars ?? _emptyTradeBars, quoteBars ?? _emptyQuoteBars, ticks ?? _emptyTicks, optionChains ?? _emptyOptionChains, futuresChains ?? _emptyFuturesChains, splits ?? _emptySplits, dividends ?? _emptyDividends, delistings ?? _emptyDelistings, symbolChanges ?? _emptySymbolChangedEvents, allDataForAlgorithm.Count > 0); return(new TimeSlice(utcDateTime, count, slice, data, security, consolidator, custom ?? _emptyCustom, changes, universeData)); }