/// <summary> /// Dequeues the current working bar /// </summary> /// <param name="utcTriggerTime">The current trigger time in UTC</param> /// <returns>The base data instance, or null, if no data is to be emitted</returns> public BaseData TriggerArchive(DateTime utcTriggerTime) { BaseData bar; if (!_queue.TryDequeue(out bar)) { // if a bar wasn't ready, check for fill forward if (_previous != null && _config.FillDataForward) { // exchanges hours are in local time, so convert to local before checking if exchange is open var localTriggerTime = utcTriggerTime.ConvertFromUtc(_config.TimeZone); // only perform fill forward behavior if the exchange is considered open var barStartTime = localTriggerTime - _increment; if (_security.Exchange.IsOpenDuringBar(barStartTime, localTriggerTime, _config.ExtendedMarketHours)) { bar = _previous.Clone(true); bar.Time = barStartTime.ExchangeRoundDown(_increment, _security.Exchange.Hours, _security.IsExtendedMarketHours); } } } // we don't have data, so just return null if (bar == null) return null; // reset the previous bar for fill forward _previous = bar.Clone(); return bar; }
/// <summary> /// Applies cash settlement rules /// </summary> /// <param name="portfolio">The algorithm's portfolio</param> /// <param name="security">The fill's security</param> /// <param name="applicationTimeUtc">The fill time (in UTC)</param> /// <param name="currency">The currency symbol</param> /// <param name="amount">The amount of cash to apply</param> public void ApplyFunds(SecurityPortfolioManager portfolio, Security security, DateTime applicationTimeUtc, string currency, decimal amount) { if (amount > 0) { // positive amount: sell order filled portfolio.UnsettledCashBook[currency].AddAmount(amount); // find the correct settlement date (usually T+3 or T+1) var settlementDate = applicationTimeUtc.ConvertFromUtc(security.Exchange.TimeZone).Date; for (var i = 0; i < _numberOfDays; i++) { settlementDate = settlementDate.AddDays(1); // only count days when market is open if (!security.Exchange.Hours.IsDateOpen(settlementDate)) i--; } // use correct settlement time var settlementTimeUtc = settlementDate.Add(_timeOfDay).ConvertToUtc(security.Exchange.Hours.TimeZone); portfolio.AddUnsettledCashAmount(new UnsettledCashAmount(settlementTimeUtc, currency, amount)); } else { // negative amount: buy order filled portfolio.CashBook[currency].AddAmount(amount); } }
/// <summary> /// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data /// </summary> /// <param name="utcDateTime">The UTC frontier date time</param> /// <param name="algorithmTimeZone">The algorithm's time zone, required for computing algorithm and slice time</param> /// <param name="cashBook">The algorithm's cash book, required for generating cash update pairs</param> /// <param name="data">The data in this <see cref="TimeSlice"/></param> /// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param> /// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns> public static TimeSlice Create(DateTime utcDateTime, DateTimeZone algorithmTimeZone, CashBook cashBook, List<KeyValuePair<Security, List<BaseData>>> data, SecurityChanges changes) { int count = 0; var security = new List<KeyValuePair<Security, BaseData>>(); var custom = new List<KeyValuePair<Security, List<BaseData>>>(); var consolidator = new List<KeyValuePair<SubscriptionDataConfig, List<BaseData>>>(); var allDataForAlgorithm = new List<BaseData>(data.Count); var cash = new List<KeyValuePair<Cash, BaseData>>(cashBook.Count); var cashSecurities = new HashSet<Symbol>(); foreach (var cashItem in cashBook.Values) { cashSecurities.Add(cashItem.SecuritySymbol); } Split split; Dividend dividend; Delisting delisting; SymbolChangedEvent symbolChange; var algorithmTime = utcDateTime.ConvertFromUtc(algorithmTimeZone); var tradeBars = new TradeBars(algorithmTime); var ticks = new Ticks(algorithmTime); var splits = new Splits(algorithmTime); var dividends = new Dividends(algorithmTime); var delistings = new Delistings(algorithmTime); var symbolChanges = new SymbolChangedEvents(algorithmTime); foreach (var kvp in data) { var list = kvp.Value; var symbol = kvp.Key.Symbol; // keep count of all data points if (list.Count == 1 && list[0] is BaseDataCollection) { count += ((BaseDataCollection) list[0]).Data.Count; } else { count += list.Count; } BaseData update = null; var consolidatorUpdate = new List<BaseData>(list.Count); for (int i = 0; i < list.Count; i++) { var baseData = list[i]; if (!kvp.Key.SubscriptionDataConfig.IsInternalFeed) { // this is all the data that goes into the algorithm allDataForAlgorithm.Add(baseData); if (kvp.Key.SubscriptionDataConfig.IsCustomData) { // this is all the custom data custom.Add(kvp); } } // don't add internal feed data to ticks/bars objects if (baseData.DataType != MarketDataType.Auxiliary) { if (!kvp.Key.SubscriptionDataConfig.IsInternalFeed) { // populate ticks and tradebars dictionaries with no aux data if (baseData.DataType == MarketDataType.Tick) { List<Tick> ticksList; if (!ticks.TryGetValue(symbol, out ticksList)) { ticksList = new List<Tick> {(Tick) baseData}; ticks[symbol] = ticksList; } ticksList.Add((Tick) baseData); } else if (baseData.DataType == MarketDataType.TradeBar) { tradeBars[symbol] = (TradeBar) baseData; } // this is data used to update consolidators consolidatorUpdate.Add(baseData); } // this is the data used set market prices update = baseData; } // include checks for various aux types so we don't have to construct the dictionaries in Slice else if ((delisting = baseData as Delisting) != null) { delistings[symbol] = delisting; } else if ((dividend = baseData as Dividend) != null) { dividends[symbol] = dividend; } else if ((split = baseData as Split) != null) { splits[symbol] = split; } else if ((symbolChange = baseData as SymbolChangedEvent) != null) { // symbol changes is keyed by the requested symbol symbolChanges[kvp.Key.SubscriptionDataConfig.Symbol] = symbolChange; } } // check for 'cash securities' if we found valid update data for this symbol // and we need this data to update cash conversion rates, long term we should // have Cash hold onto it's security, then he can update himself, or rather, just // patch through calls to conversion rate to compue it on the fly using Security.Price if (update != null && cashSecurities.Contains(kvp.Key.Symbol)) { foreach (var cashKvp in cashBook) { if (cashKvp.Value.SecuritySymbol == kvp.Key.Symbol) { cash.Add(new KeyValuePair<Cash, BaseData>(cashKvp.Value, update)); } } } security.Add(new KeyValuePair<Security, BaseData>(kvp.Key, update)); consolidator.Add(new KeyValuePair<SubscriptionDataConfig, List<BaseData>>(kvp.Key.SubscriptionDataConfig, consolidatorUpdate)); } var slice = new Slice(utcDateTime.ConvertFromUtc(algorithmTimeZone), allDataForAlgorithm, tradeBars, ticks, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0); return new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes); }
/// <summary> /// A time period has lapsed, trigger a save/queue of the current value of data. /// </summary> /// <param name="utcTriggerTime">The time we're triggering this archive for</param> /// <param name="fillForward">Data stream is a fillforward type</param> public void TriggerArchive(DateTime utcTriggerTime, bool fillForward) { var localTriggerTime = utcTriggerTime.ConvertFromUtc(_config.TimeZone); lock (_lock) { try { //When there's nothing to do: if (_data == null && !fillForward) { Log.Debug("StreamStore.TriggerArchive(): No data to store, and not fill forward: " + Symbol); } if (_data != null) { //Create clone and reset original Log.Debug("StreamStore.TriggerArchive(): Enqueued new data: S:" + _data.Symbol + " V:" + _data.Value); _previousData = _data.Clone(); _queue.Enqueue(_data.Clone()); _data = null; } else if (fillForward && _data == null && _previousData != null) { // the time is actually the end time of a bar, check to see if the start time // is within market hours, which is really just checking the _previousData's EndTime if (!_security.Exchange.IsOpenDuringBar(localTriggerTime - _increment, localTriggerTime, _config.ExtendedMarketHours)) { Log.Debug("StreamStore.TriggerArchive(): Exchange is closed: " + Symbol); return; } //There was no other data in this timer period, and this is a fillforward subscription: Log.Debug("StreamStore.TriggerArchive(): Fillforward, Previous Enqueued: S:" + _previousData.Symbol + " V:" + _previousData.Value); var cloneForward = _previousData.Clone(true); cloneForward.Time = _previousData.Time.Add(_increment); _queue.Enqueue(cloneForward); _previousData = cloneForward.Clone(); } } catch (Exception err) { Log.Error("StreamStore.TriggerAchive(fillforward): Failed to archive: " + err.Message); } } }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(universe.Market, config.Symbol, universe.SecurityType); var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator<BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick { Time = dt }).GetEnumerator(); } else if (config.Type == typeof (CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>> { {config.SecurityType, new List<Symbol>{config.Symbol}} }); var enqueable = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); return subscription; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge); return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionRequest"/> class /// </summary> public SubscriptionRequest(bool isUniverseSubscription, Universe universe, Security security, SubscriptionDataConfig configuration, DateTime startTimeUtc, DateTime endTimeUtc) { IsUniverseSubscription = isUniverseSubscription; Universe = universe; Security = security; Configuration = configuration; StartTimeUtc = startTimeUtc; EndTimeUtc = endTimeUtc; _localStartTime = new Lazy<DateTime>(() => StartTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone)); _localEndTime = new Lazy<DateTime>(() => EndTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone)); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription( IUniverse universe, DateTime startTimeUtc, DateTime endTimeUtc ) { // grab the relevant exchange hours SubscriptionDataConfig config = universe.Configuration; var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder() .GetExchangeHours(config.Market, null, config.SecurityType); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); // define our data enumerator var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, false); // create the subscription var subscription = new LiveSubscription(universe, security, enumerator, startTimeUtc, endTimeUtc); // only message the user if it's one of their universe types _subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config); var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator<BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator(); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } else if (config.Type == typeof (CoarseFundamental)) { var cf = new CoarseFundamental(); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // load coarse data day by day var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate) let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false) let source = cf.GetSource(config, dateInDataTimeZone, false) let coarseFundamentalForDate = factory.Read(source) select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate); ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2); enumerator = enqueueable; } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); }
/// <summary> /// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data /// </summary> /// <param name="utcDateTime">The UTC frontier date time</param> /// <param name="algorithmTimeZone">The algorithm's time zone, required for computing algorithm and slice time</param> /// <param name="cashBook">The algorithm's cash book, required for generating cash update pairs</param> /// <param name="data">The data in this <see cref="TimeSlice"/></param> /// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param> /// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns> public static TimeSlice Create(DateTime utcDateTime, DateTimeZone algorithmTimeZone, CashBook cashBook, List<DataFeedPacket> data, SecurityChanges changes) { int count = 0; var security = new List<UpdateData<Security>>(); var custom = new List<UpdateData<Security>>(); var consolidator = new List<UpdateData<SubscriptionDataConfig>>(); var allDataForAlgorithm = new List<BaseData>(data.Count); var cash = new List<UpdateData<Cash>>(cashBook.Count); var cashSecurities = new HashSet<Symbol>(); foreach (var cashItem in cashBook.Values) { cashSecurities.Add(cashItem.SecuritySymbol); } Split split; Dividend dividend; Delisting delisting; SymbolChangedEvent symbolChange; // we need to be able to reference the slice being created in order to define the // evaluation of option price models, so we define a 'future' that can be referenced // in the option price model evaluation delegates for each contract Slice slice = null; var sliceFuture = new Lazy<Slice>(() => slice); var algorithmTime = utcDateTime.ConvertFromUtc(algorithmTimeZone); var tradeBars = new TradeBars(algorithmTime); var quoteBars = new QuoteBars(algorithmTime); var ticks = new Ticks(algorithmTime); var splits = new Splits(algorithmTime); var dividends = new Dividends(algorithmTime); var delistings = new Delistings(algorithmTime); var optionChains = new OptionChains(algorithmTime); var symbolChanges = new SymbolChangedEvents(algorithmTime); foreach (var packet in data) { var list = packet.Data; var symbol = packet.Security.Symbol; if (list.Count == 0) continue; // keep count of all data points if (list.Count == 1 && list[0] is BaseDataCollection) { var baseDataCollectionCount = ((BaseDataCollection)list[0]).Data.Count; if (baseDataCollectionCount == 0) { continue; } count += baseDataCollectionCount; } else { count += list.Count; } if (!packet.Configuration.IsInternalFeed && packet.Configuration.IsCustomData) { // This is all the custom data custom.Add(new UpdateData<Security>(packet.Security, packet.Configuration.Type, list)); } var securityUpdate = new List<BaseData>(list.Count); var consolidatorUpdate = new List<BaseData>(list.Count); for (int i = 0; i < list.Count; i++) { var baseData = list[i]; if (!packet.Configuration.IsInternalFeed) { // this is all the data that goes into the algorithm allDataForAlgorithm.Add(baseData); } // don't add internal feed data to ticks/bars objects if (baseData.DataType != MarketDataType.Auxiliary) { if (!packet.Configuration.IsInternalFeed) { PopulateDataDictionaries(baseData, ticks, tradeBars, quoteBars, optionChains); // special handling of options data to build the option chain if (packet.Security.Type == SecurityType.Option) { if (baseData.DataType == MarketDataType.OptionChain) { optionChains[baseData.Symbol] = (OptionChain) baseData; } else if (!HandleOptionData(algorithmTime, baseData, optionChains, packet.Security, sliceFuture)) { continue; } } // this is data used to update consolidators consolidatorUpdate.Add(baseData); } // this is the data used set market prices securityUpdate.Add(baseData); } // include checks for various aux types so we don't have to construct the dictionaries in Slice else if ((delisting = baseData as Delisting) != null) { delistings[symbol] = delisting; } else if ((dividend = baseData as Dividend) != null) { dividends[symbol] = dividend; } else if ((split = baseData as Split) != null) { splits[symbol] = split; } else if ((symbolChange = baseData as SymbolChangedEvent) != null) { // symbol changes is keyed by the requested symbol symbolChanges[packet.Configuration.Symbol] = symbolChange; } } if (securityUpdate.Count > 0) { // check for 'cash securities' if we found valid update data for this symbol // and we need this data to update cash conversion rates, long term we should // have Cash hold onto it's security, then he can update himself, or rather, just // patch through calls to conversion rate to compue it on the fly using Security.Price if (cashSecurities.Contains(packet.Security.Symbol)) { foreach (var cashKvp in cashBook) { if (cashKvp.Value.SecuritySymbol == packet.Security.Symbol) { var cashUpdates = new List<BaseData> {securityUpdate[securityUpdate.Count - 1]}; cash.Add(new UpdateData<Cash>(cashKvp.Value, packet.Configuration.Type, cashUpdates)); } } } security.Add(new UpdateData<Security>(packet.Security, packet.Configuration.Type, securityUpdate)); } if (consolidatorUpdate.Count > 0) { consolidator.Add(new UpdateData<SubscriptionDataConfig>(packet.Configuration, packet.Configuration.Type, consolidatorUpdate)); } } slice = new Slice(algorithmTime, allDataForAlgorithm, tradeBars, quoteBars, ticks, optionChains, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0); return new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes); }
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return null; } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market); // ReSharper disable once PossibleMultipleEnumeration IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return subscription; }
/// <summary> /// Determines whether or not the specified security can be removed from /// this universe. This is useful to prevent securities from being taken /// out of a universe before the algorithm has had enough time to make /// decisions on the security /// </summary> /// <param name="utcTime">The current utc time</param> /// <param name="security">The security to check if its ok to remove</param> /// <returns>True if we can remove the security, false otherwise</returns> public override bool CanRemoveMember(DateTime utcTime, Security security) { // if we haven't begun receiving data for this security then it's safe to remove var lastData = security.Cache.GetData(); if (lastData == null) { return true; } // only remove members on day changes, this prevents us from needing to // fast forward contracts continuously as price moves and out filtered // contracts change thoughout the day var localTime = utcTime.ConvertFromUtc(security.Exchange.TimeZone); if (localTime.Date != lastData.Time.Date) { return true; } return false; }
private Subscription CreateSubscription(Universe universe, Security security, DateTime startTimeUtc, DateTime endTimeUtc) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return null; } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market); // ReSharper disable once PossibleMultipleEnumeration var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return subscription; }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator<BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick {Time = dt}).GetEnumerator(); } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, tradeableDates, false); } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); // only message the user if it's one of their universe types var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental); PrimeSubscriptionPump(subscription, messageUser); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone); var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // custom data uses backtest readers var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false); // apply fast forwarding, this is especially important for RemoteFile types that // can send in large chunks of old, irrelevant data var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment); // apply rate limits (1x per increment, max 30 minutes between calls) // TODO : Pull limits from config file? var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); // add the enumerator to the exchange _customExchange.AddEnumerator(rateLimit); // this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Returns an enumerator that defines when this user defined universe will be invoked /// </summary> /// <returns>An enumerator of DateTime that defines when this universe will be invoked</returns> public virtual IEnumerable<DateTime> GetTriggerTimes(DateTime startTimeUtc, DateTime endTimeUtc, MarketHoursDatabase marketHoursDatabase) { var exchangeHours = marketHoursDatabase.GetExchangeHours(Configuration); var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); var first = true; foreach (var dateTime in LinqExtensions.Range(localStartTime, localEndTime, dt => dt + Interval)) { if (first) { yield return dateTime; first = false; } if (exchangeHours.IsOpen(dateTime, dateTime + Interval, Configuration.ExtendedMarketHours)) { yield return dateTime; } } }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription( IUniverse universe, DateTime startTimeUtc, DateTime endTimeUtc ) { // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder() .GetExchangeHours(config.Market, null, config.SecurityType); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator<BaseData> enumerator; if (config.Type == typeof (CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<string>> { {config.SecurityType, new List<string>{config.Symbol}} }); var enqueable = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); // define our data enumerator var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true); _customExchange.AddEnumerator(reader); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc); return subscription; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="config">The subscription config to be added</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider); _customExchange.AddEnumerator(config.Symbol, frontierAware); var enqueable = new EnqueueableEnumerator<BaseData>(); _customExchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider); _exchange.SetDataHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data if (config.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return subscription; }