/// <summary> /// Initializes a new instance of the <see cref="UniverseSelectionEventArgs"/> class /// </summary> /// <param name="universe">The universe that raised this event</param> /// <param name="configuration">Theconfiguration for the data</param> /// <param name="dateTimeUtc">The date time this event was fired in UTC</param> /// <param name="data">The data contained within this event</param> public UniverseSelectionEventArgs(Universe universe, SubscriptionDataConfig configuration, DateTime dateTimeUtc, IReadOnlyList<BaseData> data) { Universe = universe; Configuration = configuration; DateTimeUtc = dateTimeUtc; Data = data; }
/// <summary> /// Creates a new universe that restricts the universe selection data to symbols that passed the /// first universe's selection critera /// /// NOTE: The <paramref name="second"/> universe instance provided to this method should not be manually /// added to the algorithm. The <paramref name="first"/> should still be manually (assuming no other changes). /// </summary> /// <param name="second">The universe to be filtere</param> /// <param name="first">The universe providing the set of symbols used for filtered</param> /// <returns>A new universe that can be added to the algorithm that represents invoking the second /// using the selections from the first as a filter.</returns> public static Universe PrefilterUsing(this Universe second, Universe first) { return new SelectSymbolsUniverseDecorator(second, (utcTime, data) => { var clone = (BaseDataCollection)data.Clone(); clone.Data = clone.Data.Where(d => first.ContainsMember(d.Symbol)).ToList(); return second.SelectSymbols(utcTime, clone); }); }
/// <summary> /// Creates a new universe that logically is the result of wiring the two universes together such that /// the first will produce subscriptions for the second and the second will only select on data that has /// passed the first. /// /// NOTE: The <paramref name="first"/> and <paramref name="second"/> universe instances provided /// to this method should not be manually added to the algorithm. /// </summary> /// <param name="first">The first universe in this 'chain'</param> /// <param name="second">The second universe in this 'chain'</param> /// <param name="configurationPerSymbol">True if each symbol as its own configuration, false otherwise</param> /// <returns>A new universe that can be added to the algorithm that represents invoking the first universe /// and then the second universe using the outputs of the first. </returns> public static Universe ChainedTo(this Universe first, Universe second, bool configurationPerSymbol) { var prefilteredSecond = second.PrefilterUsing(first); return new GetSubscriptionRequestsUniverseDecorator(first, (security, currentTimeUtc, maximumEndTimeUtc) => { return first.GetSubscriptionRequests(security, currentTimeUtc, maximumEndTimeUtc).Select(request => new SubscriptionRequest( template: request, isUniverseSubscription: true, universe: prefilteredSecond, security: security, configuration: configurationPerSymbol ? new SubscriptionDataConfig(prefilteredSecond.Configuration, symbol: security.Symbol) : prefilteredSecond.Configuration, startTimeUtc: currentTimeUtc - prefilteredSecond.Configuration.Resolution.ToTimeSpan(), endTimeUtc: currentTimeUtc.AddSeconds(-1) )); }); }
/// <summary> /// Adds a new subscription to provide data for the specified security. /// </summary> /// <param name="universe">The universe the subscription is to be added to</param> /// <param name="security">The security to add a subscription for</param> /// <param name="utcStartTime">The start time of the subscription</param> /// <param name="utcEndTime">The end time of the subscription</param> public bool AddSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { _fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm); var subscription = CreateSubscription(universe, _resultHandler, security, utcStartTime, utcEndTime, _fillForwardResolution); if (subscription == null) { // subscription will be null when there's no tradeable dates for the security between the requested times, so // don't even try to load the data return false; } Log.Debug("FileSystemDataFeed.AddSubscription(): Added " + security.Symbol.ID + " Start: " + utcStartTime + " End: " + utcEndTime); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); // prime the pump, run method checks current before move next calls //PrimeSubscriptionPump(subscription, true); _changes += SecurityChanges.Added(security); _fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm); return true; }
/// <summary> /// Initializes a new instance of the <see cref="SelectSymbolsUniverseDecorator"/> class /// </summary> /// <param name="universe">The universe to be decorated</param> /// <param name="selectSymbols">The new implementation of <see cref="SelectSymbols"/></param> public SelectSymbolsUniverseDecorator(Universe universe, SelectSymbolsDelegate selectSymbols) : base(universe) { _selectSymbols = selectSymbols; }
/// <summary> /// Event invocator for the <see cref="UniverseSelection"/> event /// </summary> /// <param name="universe">The universe to perform selection on the data</param> /// <param name="config">The configuration of the universe</param> /// <param name="dateTimeUtc">The current date time in UTC</param> /// <param name="data">The universe selection data to be operated on</param> protected virtual void OnUniverseSelection(Universe universe, SubscriptionDataConfig config, DateTime dateTimeUtc, IReadOnlyList<BaseData> data) { if (UniverseSelection != null) { var eventArgs = new UniverseSelectionEventArgs(universe, config, dateTimeUtc, data); var multicast = (MulticastDelegate)UniverseSelection; foreach (UniverseSelectionHandler handler in multicast.GetInvocationList()) { _changes += handler(this, eventArgs); } } }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionRequest"/> class /// </summary> public SubscriptionRequest(bool isUniverseSubscription, Universe universe, Security security, SubscriptionDataConfig configuration, DateTime startTimeUtc, DateTime endTimeUtc) { IsUniverseSubscription = isUniverseSubscription; Universe = universe; Security = security; Configuration = configuration; StartTimeUtc = startTimeUtc; EndTimeUtc = endTimeUtc; _localStartTime = new Lazy<DateTime>(() => StartTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone)); _localEndTime = new Lazy<DateTime>(() => EndTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone)); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge); return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); IEnumerator<BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(dt => new Tick { Time = dt }).GetEnumerator(); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } else if (config.Type == typeof (CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] {security.Symbol}); var enqueable = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true); return subscription; }
/// <summary> /// Initializes a new instance of the <see cref="UniverseDecorator"/> class /// </summary> /// <param name="universe">The decorated universe. All overridable methods delegate to this instance.</param> protected UniverseDecorator(Universe universe) : base(universe.Configuration, universe.SecurityInitializer) { Universe = universe; }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator<BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick {Time = dt}).GetEnumerator(); } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, tradeableDates, false); } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); // only message the user if it's one of their universe types var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental); PrimeSubscriptionPump(subscription, messageUser); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); }
/// <summary> /// Sets the security initializer, used to initialize/configure securities after creation /// </summary> /// <param name="securityInitializer">The security initializer</param> public override void SetSecurityInitializer(ISecurityInitializer securityInitializer) { base.SetSecurityInitializer(securityInitializer); Universe.SetSecurityInitializer(securityInitializer); FineFundamentalUniverse.SetSecurityInitializer(securityInitializer); }
/// <summary> /// Initializes a new instance of the <see cref="FineFundamentalFilteredUniverse"/> class /// </summary> /// <param name="universe">The universe to be filtered</param> /// <param name="fineSelector">The fine selection function</param> public FineFundamentalFilteredUniverse(Universe universe, Func <IEnumerable <FineFundamental>, IEnumerable <Symbol> > fineSelector) : base(universe, universe.SelectSymbols) { FineFundamentalUniverse = new FineFundamentalUniverse(universe.UniverseSettings, universe.SecurityInitializer, fineSelector); FineFundamentalUniverse.SelectionChanged += (sender, args) => OnSelectionChanged(((SelectionEventArgs)args).CurrentSelection); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="config">The subscription config to be added</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider); _customExchange.AddEnumerator(config.Symbol, frontierAware); var enqueable = new EnqueueableEnumerator<BaseData>(); _customExchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider); _exchange.SetDataHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data if (config.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config); var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator<BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator(); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } else if (config.Type == typeof (CoarseFundamental)) { var cf = new CoarseFundamental(); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // load coarse data day by day var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate) let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false) let source = cf.GetSource(config, dateInDataTimeZone, false) let coarseFundamentalForDate = factory.Read(source) select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate); ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2); enumerator = enqueueable; } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); }
/// <summary> /// Applies universe selection the the data feed and algorithm /// </summary> /// <param name="universe">The universe to perform selection on</param> /// <param name="dateTimeUtc">The current date time in utc</param> /// <param name="universeData">The data provided to perform selection with</param> public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData) { var settings = universe.SubscriptionSettings; var limit = 1000; //daily/hourly limit var resolution = settings.Resolution; switch (resolution) { case Resolution.Tick: limit = _algorithm.Securities.TickLimit; break; case Resolution.Second: limit = _algorithm.Securities.SecondLimit; break; case Resolution.Minute: limit = _algorithm.Securities.MinuteLimit; break; } // subtract current subscriptions that can't be removed limit -= _algorithm.Securities.Count(x => x.Value.Resolution == resolution && x.Value.HoldStock); if (limit < 1) { // if we don't have room for more securities then we can't really do anything here. _algorithm.Error("Unable to add more securities from universe selection due to holding stock."); return SecurityChanges.None; } // perform initial filtering and limit the result var selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData.Data); // check for no changes first if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { return SecurityChanges.None; } var selections = selectSymbolsResult.Take(limit).ToHashSet(); // create a hash set of our existing subscriptions by sid var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol); var additions = new List<Security>(); var removals = new List<Security>(); // determine which data subscriptions need to be removed for this market foreach (var subscription in _dataFeed.Subscriptions) { // universes can only remove members of their own if (!universe.ContainsMember(subscription.Security)) continue; // never remove universe selection subscriptions if (subscription.IsUniverseSelectionSubscription) continue; var config = subscription.Configuration; // never remove internal feeds if (config.IsInternalFeed) continue; // if we've selected this subscription again, keep it if (selections.Contains(config.Symbol)) continue; // let the algorithm know this security has been removed from the universe removals.Add(subscription.Security); // but don't physically remove it from the algorithm if we hold stock or have open orders against it var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol); if (!subscription.Security.HoldStock && !openOrders.Any()) { // we need to mark this security as untradeable while it has no data subscription // it is expected that this function is called while in sync with the algo thread, // so we can make direct edits to the security here subscription.Security.Cache.Reset(); if (_dataFeed.RemoveSubscription(subscription)) { universe.RemoveMember(subscription.Security); } } } // find new selections and add them to the algorithm foreach (var symbol in selections) { // we already have a subscription for this symbol so don't re-add it if (existingSubscriptions.Contains(symbol)) continue; // create the new security, the algorithm thread will add this at the appropriate time Security security; if (!_algorithm.Securities.TryGetValue(symbol, out security)) { security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase, symbol, settings.Resolution, settings.FillForward, settings.Leverage, settings.ExtendedMarketHours, false, false, false); } additions.Add(security); // add the new subscriptions to the data feed if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone))) { universe.AddMember(security); } } // return None if there's no changes, otherwise return what we've modified return additions.Count + removals.Count != 0 ? new SecurityChanges(additions, removals) : SecurityChanges.None; }
/// <summary> /// Applies universe selection the the data feed and algorithm /// </summary> /// <param name="universe">The universe to perform selection on</param> /// <param name="dateTimeUtc">The current date time in utc</param> /// <param name="universeData">The data provided to perform selection with</param> public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData) { // perform initial filtering and limit the result var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData); // check for no changes first if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { return SecurityChanges.None; } // materialize the enumerable into a set for processing var selections = selectSymbolsResult.ToHashSet(); var additions = new List<Security>(); var removals = new List<Security>(); var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); // determine which data subscriptions need to be removed from this universe foreach (var member in universe.Members.Values) { // if we've selected this subscription again, keep it if (selections.Contains(member.Symbol)) continue; // don't remove if the universe wants to keep him in if (!universe.CanRemoveMember(dateTimeUtc, member)) continue; // remove the member - this marks this member as not being // selected by the universe, but it may remain in the universe // until open orders are closed and the security is liquidated removals.Add(member); // but don't physically remove it from the algorithm if we hold stock or have open orders against it var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol); if (!member.HoldStock && !openOrders.Any()) { // safe to remove the member from the universe universe.RemoveMember(dateTimeUtc, member); // we need to mark this security as untradeable while it has no data subscription // it is expected that this function is called while in sync with the algo thread, // so we can make direct edits to the security here member.Cache.Reset(); foreach (var subscription in universe.GetSubscriptionRequests(member, dateTimeUtc, algorithmEndDateUtc)) { if (subscription.IsUniverseSubscription) { removals.Remove(member); } else { _dataFeed.RemoveSubscription(subscription.Configuration); } } // remove symbol mappings for symbols removed from universes // TODO : THIS IS BAD! SymbolCache.TryRemove(member.Symbol); } } // find new selections and add them to the algorithm foreach (var symbol in selections) { // create the new security, the algorithm thread will add this at the appropriate time Security security; if (!_algorithm.Securities.TryGetValue(symbol, out security)) { security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase); } var addedMember = universe.AddMember(dateTimeUtc, security); var addedSubscription = false; foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc)) { // ask the limiter if we can add another subscription at that resolution string reason; if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason)) { // should we be counting universe subscriptions against user subscriptions limits? _algorithm.Error(reason); Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason); continue; } // add the new subscriptions to the data feed _dataFeed.AddSubscription(request); // only update our security changes if we actually added data if (!request.IsUniverseSubscription) { addedSubscription = addedMember; } } if (addedSubscription) { additions.Add(security); } } // Add currency data feeds that weren't explicitly added in Initialize if (additions.Count > 0) { var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets); foreach (var security in addedSecurities) { // assume currency feeds are always one subscription per, these are typically quote subscriptions _dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc)); } } // return None if there's no changes, otherwise return what we've modified var securityChanges = additions.Count + removals.Count != 0 ? new SecurityChanges(additions, removals) : SecurityChanges.None; if (securityChanges != SecurityChanges.None) { Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges); } return securityChanges; }
/// <summary> /// Initializes a new instance of the <see cref="FineFundamentalFilteredUniverse"/> class /// </summary> /// <param name="universe">The universe to be filtered</param> /// <param name="fineSelector">The fine selection function</param> public FineFundamentalFilteredUniverse(Universe universe, Func <IEnumerable <FineFundamental>, IEnumerable <Symbol> > fineSelector) : base(universe, universe.SelectSymbols) { FineFundamentalUniverse = new FineFundamentalUniverse(universe.UniverseSettings, universe.SecurityInitializer, fineSelector); }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionRequest"/> class /// </summary> public SubscriptionRequest(SubscriptionRequest template, bool? isUniverseSubscription = null, Universe universe = null, Security security = null, SubscriptionDataConfig configuration = null, DateTime? startTimeUtc = null, DateTime? endTimeUtc = null ) : this(isUniverseSubscription ?? template.IsUniverseSubscription, universe ?? template.Universe, security ?? template.Security, configuration ?? template.Configuration, startTimeUtc ?? template.StartTimeUtc, endTimeUtc ?? template.EndTimeUtc ) { }
/// <summary> /// Adds a new subscription to provide data for the specified security. /// </summary> /// <param name="universe">The universe the subscription is to be added to</param> /// <param name="security">The security to add a subscription for</param> /// <param name="utcStartTime">The start time of the subscription</param> /// <param name="utcEndTime">The end time of the subscription</param> /// <returns>True if the subscription was created and added successfully, false otherwise</returns> public bool AddSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { // create and add the subscription to our collection var subscription = CreateSubscription(universe, security, utcStartTime, utcEndTime); // for some reason we couldn't create the subscription if (subscription == null) { Log.Trace("Unable to add subscription for: " + security.Symbol.ToString()); return false; } Log.Trace("LiveTradingDataFeed.AddSubscription(): Added " + security.Symbol.ToString()); _subscriptions[subscription.Security.Symbol] = subscription; // send the subscription for the new symbol through to the data queuehandler // unless it is custom data, custom data is retrieved using the same as backtest if (!subscription.Configuration.IsCustomData) { _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>> { {security.Type, new List<Symbol> {security.Symbol}} }); } // keep track of security changes, we emit these to the algorithm // as notifications, used in universe selection _changes += SecurityChanges.Added(security); // update our fill forward resolution setting _fillForwardResolution.Value = ResolveFillForwardResolution(_algorithm); return true; }
private Subscription CreateSubscription(Universe universe, Security security, DateTime startTimeUtc, DateTime endTimeUtc) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return null; } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market); // ReSharper disable once PossibleMultipleEnumeration var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return subscription; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(universe.Market, config.Symbol, universe.SecurityType); var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator<BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick { Time = dt }).GetEnumerator(); } else if (config.Type == typeof (CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>> { {config.SecurityType, new List<Symbol>{config.Symbol}} }); var enqueable = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); return subscription; }
/// <summary> /// Initializes a new instance of the <see cref="GetSubscriptionRequestsUniverseDecorator"/> class /// </summary> /// <param name="universe">The universe to be decorated</param> /// <param name="getRequests"></param> public GetSubscriptionRequestsUniverseDecorator(Universe universe, GetSubscriptionRequestsDelegate getRequests) : base(universe) { _getRequests = getRequests; }
/// <summary> /// Applies universe selection the the data feed and algorithm /// </summary> /// <param name="universe">The universe to perform selection on</param> /// <param name="dateTimeUtc">The current date time in utc</param> /// <param name="universeData">The data provided to perform selection with</param> public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData) { var settings = universe.UniverseSettings; // perform initial filtering and limit the result var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData); // check for no changes first if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { return SecurityChanges.None; } // materialize the enumerable into a set for processing var selections = selectSymbolsResult.ToHashSet(); // create a hash set of our existing subscriptions by sid var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol); var additions = new List<Security>(); var removals = new List<Security>(); // determine which data subscriptions need to be removed from this universe foreach (var member in universe.Members.Values) { var config = member.SubscriptionDataConfig; // if we've selected this subscription again, keep it if (selections.Contains(config.Symbol)) continue; // don't remove if the universe wants to keep him in if (!universe.CanRemoveMember(dateTimeUtc, member)) continue; // remove the member - this marks this member as not being // selected by the universe, but it may remain in the universe // until open orders are closed and the security is liquidated removals.Add(member); // but don't physically remove it from the algorithm if we hold stock or have open orders against it var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol); if (!member.HoldStock && !openOrders.Any()) { // safe to remove the member from the universe universe.RemoveMember(dateTimeUtc, member); // we need to mark this security as untradeable while it has no data subscription // it is expected that this function is called while in sync with the algo thread, // so we can make direct edits to the security here member.Cache.Reset(); _dataFeed.RemoveSubscription(member.Symbol); } } // find new selections and add them to the algorithm foreach (var symbol in selections) { // we already have a subscription for this symbol so don't re-add it if (existingSubscriptions.Contains(symbol)) continue; // ask the limiter if we can add another subscription at that resolution string reason; if (!_limiter.CanAddSubscription(settings.Resolution, out reason)) { _algorithm.Error(reason); Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscriptions: " + reason); break; } // create the new security, the algorithm thread will add this at the appropriate time Security security; if (!_algorithm.Securities.TryGetValue(symbol, out security)) { security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, universe.SecurityInitializer, symbol, settings.Resolution, settings.FillForward, settings.Leverage, settings.ExtendedMarketHours, false, false, false); } additions.Add(security); // add the new subscriptions to the data feed if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone))) { universe.AddMember(dateTimeUtc, security); } } // Add currency data feeds that weren't explicitly added in Initialize if (additions.Count > 0) { var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets); foreach (var security in addedSecurities) { _dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)); } } // return None if there's no changes, otherwise return what we've modified return additions.Count + removals.Count != 0 ? new SecurityChanges(additions, removals) : SecurityChanges.None; }
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return null; } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market); // ReSharper disable once PossibleMultipleEnumeration IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return subscription; }
/// <summary> /// Initializes a new instance of the <see cref="UniverseDecorator"/> class /// </summary> /// <param name="universe">The decorated universe. All overridable methods delegate to this instance.</param> protected UniverseDecorator(Universe universe) : base(universe.Configuration) { Universe = universe; }