/// <summary> /// Creates an enumerator to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var enumerator = (IEnumerator <BaseData>)_universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase) .Select(x => new Tick { Time = x, Symbol = request.Configuration.Symbol }) .GetEnumerator(); var universe = request.Universe as UserDefinedUniverse; if (universe != null) { enumerator = new InjectionEnumerator(enumerator); // Trigger universe selection when security added/removed after Initialize universe.CollectionChanged += (sender, args) => { // If it is an add we will set time 1 tick ahead to properly sync data // with next timeslice, if it is a remove then we will set time to now IList items; DateTime time; if (args.Action == NotifyCollectionChangedAction.Add) { items = args.NewItems; time = _timeProvider.GetUtcNow().AddTicks(1); } else if (args.Action == NotifyCollectionChangedAction.Remove) { items = args.OldItems; time = _timeProvider.GetUtcNow(); } else { items = null; time = DateTime.MinValue; } // Check that we have our items and time if (items == null || time == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } // the data point time should always be in exchange timezone time = time.ConvertFromUtc(request.Configuration.ExchangeTimeZone); var collection = new BaseDataCollection(time, symbol); ((InjectionEnumerator)enumerator).InjectDataPoint(collection); }; } return(enumerator); }
public void FundamentalUniverseSelectionModelCanBeInherited() { var code = @" from AlgorithmImports import * from Selection.FundamentalUniverseSelectionModel import FundamentalUniverseSelectionModel class MockUniverseSelectionModel(FundamentalUniverseSelectionModel): def __init__(self): super().__init__(False) def SelectCoarse(self, algorithm, coarse): return [Symbol.Create('SPY', SecurityType.Equity, Market.USA)]"; using (Py.GIL()) { dynamic pyModel = PythonEngine .ModuleFromString(Guid.NewGuid().ToString(), code) .GetAttr("MockUniverseSelectionModel"); var model = new UniverseSelectionModelPythonWrapper(pyModel()); var universes = model.CreateUniverses(new QCAlgorithm()).ToList(); Assert.AreEqual(1, universes.Count); var data = new BaseDataCollection(); data.Data.Add(new CoarseFundamental()); var universe = universes.First(); var symbols = universe.SelectSymbols(DateTime.Now, data).ToList(); Assert.AreEqual(1, symbols.Count); var expected = Symbol.Create("SPY", SecurityType.Equity, Market.USA); var symbol = symbols.First(); Assert.AreEqual(expected, symbol); } }
/// <summary> /// Creates the correct enumerator factory for the given request /// </summary> private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request) { if (request.IsUniverseSubscription) { if (request.Universe is UserDefinedUniverse) { // Trigger universe selection when security added/removed after Initialize var universe = (UserDefinedUniverse)request.Universe; universe.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; return(new UserDefinedUniverseSubscriptionEnumeratorFactory(request.Universe as UserDefinedUniverse, MarketHoursDatabase.FromDataFolder())); } if (request.Configuration.Type == typeof(CoarseFundamental)) { return(new BaseDataCollectionSubscriptionEnumeratorFactory()); } if (request.Configuration.Type == typeof(FineFundamental)) { return(new FineFundamentalSubscriptionEnumeratorFactory()); } if (request.Universe is OptionChainUniverse) { return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e))); } } var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; return(new PostCreateConfigureSubscriptionEnumeratorFactory( new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, mapFileResolver, _factorFileProvider, false, true), enumerator => ConfigureEnumerator(request, false, enumerator) )); }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public IEnumerable <BaseData> Read(SubscriptionDataSource source) { SubscriptionDataSourceReader.CheckRemoteFileCache(); IStreamReader reader = null; var instances = new BaseDataCollection(); try { switch (source.TransportMedium) { default: case SubscriptionTransportMedium.Rest: reader = new RestSubscriptionStreamReader(source.Source); break; case SubscriptionTransportMedium.LocalFile: reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source); break; case SubscriptionTransportMedium.RemoteFile: reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache); break; } var raw = ""; try { raw = reader.ReadLine(); var result = _factory.Reader(_config, raw, _date, _isLiveMode); instances = result as BaseDataCollection; if (instances == null) { OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection")); } } catch (Exception err) { OnReaderError(raw, err); } foreach (var instance in instances.Data) { yield return(instance); } } finally { if (reader != null) { reader.Dispose(); } } }
/// <summary> /// Creates the correct enumerator factory for the given request /// </summary> private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request) { if (request.IsUniverseSubscription) { if (request.Universe is ITimeTriggeredUniverse) { var universe = request.Universe as UserDefinedUniverse; if (universe != null) { // Trigger universe selection when security added/removed after Initialize universe.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_algorithm.UtcTime, symbol); var changes = _universeSelection.ApplyUniverseSelection(universe, _algorithm.UtcTime, collection); _algorithm.OnSecuritiesChanged(changes); }; } return(new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder())); } if (request.Configuration.Type == typeof(CoarseFundamental)) { return(new BaseDataCollectionSubscriptionEnumeratorFactory()); } if (request.Universe is OptionChainUniverse) { return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider)); } if (request.Universe is FuturesChainUniverse) { return(new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e))); } } return(_subscriptionfactory); }
/// <summary> /// Creates an enumerator to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var enumerator = (IEnumerator <BaseData>)_universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase) .Select(x => new Tick { Time = x, Symbol = request.Configuration.Symbol }) .GetEnumerator(); var universe = request.Universe as UserDefinedUniverse; if (universe != null) { enumerator = new InjectionEnumerator(enumerator); // Trigger universe selection when security added/removed after Initialize universe.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; var time = _timeProvider.GetUtcNow(); if (items == null || time == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } // the data point time should always be in exchange timezone time = time.ConvertFromUtc(request.Configuration.ExchangeTimeZone); var collection = new BaseDataCollection(time, symbol); ((InjectionEnumerator)enumerator).InjectDataPoint(collection); }; } return(enumerator); }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public override IEnumerable <BaseData> Read(SubscriptionDataSource source) { foreach (var point in base.Read(source)) { if (point is BaseDataCollection) { // if underlying already is returning a collection let it through as is yield return(point); } else { if (_collection != null && _collection.EndTime != point.EndTime) { // when we get a new time we flush current collection instance, if any yield return(_collection); _collection = null; } if (_collection == null) { _collection = (BaseDataCollection)Activator.CreateInstance(_collectionType); _collection.Time = point.Time; _collection.Symbol = point.Symbol; _collection.EndTime = point.EndTime; } // aggregate the data points _collection.Data.Add(point); } } // underlying reader ended, flush current collection instance if any if (_collection != null) { yield return(_collection); _collection = null; } }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public override IEnumerable <BaseData> Read(SubscriptionDataSource source) { SubscriptionDataSourceReader.CheckRemoteFileCache(); IStreamReader reader = null; try { reader = CreateStreamReader(source); if (reader == null) { yield break; } var raw = ""; while (!reader.EndOfStream) { BaseDataCollection instances = null; try { raw = reader.ReadLine(); var result = _factory.Reader(_config, raw, _date, IsLiveMode); instances = result as BaseDataCollection; if (instances == null && !reader.ShouldBeRateLimited) { OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection")); continue; } } catch (Exception err) { OnReaderError(raw, err); if (!reader.ShouldBeRateLimited) { continue; } } if (IsLiveMode // this shouldn't happen, rest reader is the only one to be rate limited // and in live mode, but just in case... || instances == null && reader.ShouldBeRateLimited) { // in live trading these data points will be unrolled at the // 'LiveCustomDataSubscriptionEnumeratorFactory' level yield return(instances); } else { foreach (var instance in instances.Data) { if (instance != null && instance.EndTime != default(DateTime)) { yield return(instance); } } } } } finally { reader.DisposeSafely(); } }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; var timeTriggered = request.Universe as ITimeTriggeredUniverse; if (timeTriggered != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize if (timeTriggered is UserDefinedUniverse) { var userDefined = (UserDefinedUniverse)timeTriggered; userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; } } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else if (request.Universe is OptionChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString()); Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) => { // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data. var aggregator = input as TradeBarBuilderEnumerator; if (aggregator != null) { _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick)data); }); } var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration }); UpdateFillForwardResolution(subscriptionConfigs); return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone)); }; var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Options."); } var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else if (request.Universe is FuturesChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString()); var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Futures."); } var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } // create the subscription var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, tzOffsetProvider, enumerator); var subscription = new Subscription(request.Universe, request.Security, config, subscriptionDataEnumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); return(subscription); }
/// <summary> /// Applies universe selection the the data feed and algorithm /// </summary> /// <param name="universe">The universe to perform selection on</param> /// <param name="dateTimeUtc">The current date time in utc</param> /// <param name="universeData">The data provided to perform selection with</param> public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData) { var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); if (dateTimeUtc > algorithmEndDateUtc) { return(SecurityChanges.None); } IEnumerable <Symbol> selectSymbolsResult; // check if this universe must be filtered with fine fundamental data var fineFiltered = universe as FineFundamentalFilteredUniverse; if (fineFiltered != null) { // perform initial filtering and limit the result selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData); if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { // prepare a BaseDataCollection of FineFundamental instances var fineCollection = new BaseDataCollection(); var dataProvider = new DefaultDataProvider(); // use all available threads, the entire system is waiting for this to complete var options = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; Parallel.ForEach(selectSymbolsResult, options, symbol => { var config = FineFundamentalUniverse.CreateConfiguration(symbol); var security = _securityService.CreateSecurity(symbol, config); var localStartTime = dateTimeUtc.ConvertFromUtc(config.ExchangeTimeZone).AddDays(-1); var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { localStartTime }); var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), localStartTime, localStartTime); using (var enumerator = factory.CreateEnumerator(request, dataProvider)) { if (enumerator.MoveNext()) { lock (fineCollection.Data) { fineCollection.Data.Add(enumerator.Current); } } } }); // WARNING -- HACK ATTACK -- WARNING // Fine universes are considered special due to their chaining behavior. // As such, we need a means of piping the fine data read in here back to the data feed // so that it can be properly emitted via a TimeSlice.Create call. There isn't a mechanism // in place for this function to return such data. The following lines are tightly coupled // to the universeData dictionaries in SubscriptionSynchronizer and LiveTradingDataFeed and // rely on reference semantics to work. // Coarse raw data has SID collision on: CRHCY R735QTJ8XC9X var coarseData = universeData.Data.OfType <CoarseFundamental>() .DistinctBy(c => c.Symbol) .ToDictionary(c => c.Symbol); universeData.Data = new List <BaseData>(); foreach (var fine in fineCollection.Data.OfType <FineFundamental>()) { var fundamentals = new Fundamentals { Symbol = fine.Symbol, Time = fine.Time, EndTime = fine.EndTime, DataType = fine.DataType, CompanyReference = fine.CompanyReference, EarningReports = fine.EarningReports, EarningRatios = fine.EarningRatios, FinancialStatements = fine.FinancialStatements, OperationRatios = fine.OperationRatios, SecurityReference = fine.SecurityReference, ValuationRatios = fine.ValuationRatios }; CoarseFundamental coarse; if (coarseData.TryGetValue(fine.Symbol, out coarse)) { // the only time the coarse data won't exist is if the selection function // doesn't use the data provided, and instead returns a constant list of // symbols -- coupled with a potential hole in the data fundamentals.Value = coarse.Value; fundamentals.Market = coarse.Market; fundamentals.Volume = coarse.Volume; fundamentals.DollarVolume = coarse.DollarVolume; fundamentals.HasFundamentalData = coarse.HasFundamentalData; // set the fine fundamental price property to yesterday's closing price fine.Value = coarse.Value; } universeData.Data.Add(fundamentals); } // END -- HACK ATTACK -- END // perform the fine fundamental universe selection selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection); } } else { // perform initial filtering and limit the result selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData); } // check for no changes first if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { return(SecurityChanges.None); } // materialize the enumerable into a set for processing var selections = selectSymbolsResult.ToHashSet(); var additions = new List <Security>(); var removals = new List <Security>(); // remove previously deselected members which were kept in the universe because of holdings or open orders foreach (var member in _pendingRemovals.ToList()) { if (IsSafeToRemove(member)) { RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc); _pendingRemovals.Remove(member); } } // determine which data subscriptions need to be removed from this universe foreach (var member in universe.Members.Values) { // if we've selected this subscription again, keep it if (selections.Contains(member.Symbol)) { continue; } // don't remove if the universe wants to keep him in if (!universe.CanRemoveMember(dateTimeUtc, member)) { continue; } // remove the member - this marks this member as not being // selected by the universe, but it may remain in the universe // until open orders are closed and the security is liquidated removals.Add(member); if (IsSafeToRemove(member)) { RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc); } else { _pendingRemovals.Add(member); } } var keys = _pendingSecurityAdditions.Keys; if (keys.Any() && keys.Single() != dateTimeUtc) { // if the frontier moved forward then we've added these securities to the algorithm _pendingSecurityAdditions.Clear(); } Dictionary <Symbol, Security> pendingAdditions; if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions)) { // keep track of created securities so we don't create the same security twice, leads to bad things :) pendingAdditions = new Dictionary <Symbol, Security>(); _pendingSecurityAdditions[dateTimeUtc] = pendingAdditions; } // find new selections and add them to the algorithm foreach (var symbol in selections) { // create the new security, the algorithm thread will add this at the appropriate time Security security; if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security)) { // For now this is required for retro compatibility with usages of security.Subscriptions var configs = _algorithm.SubscriptionManager.SubscriptionDataConfigService.Add(symbol, universe.UniverseSettings.Resolution, universe.UniverseSettings.FillForward, universe.UniverseSettings.ExtendedMarketHours); security = _securityService.CreateSecurity(symbol, configs, universe.UniverseSettings.Leverage, symbol.ID.SecurityType == SecurityType.Option); pendingAdditions.Add(symbol, security); SetUnderlyingSecurity(universe, security); } var addedSubscription = false; foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc, _algorithm.SubscriptionManager.SubscriptionDataConfigService)) { if (security.Symbol == request.Configuration.Symbol && // Just in case check its the same symbol, else AddData will throw. !security.Subscriptions.Contains(request.Configuration)) { // For now this is required for retro compatibility with usages of security.Subscriptions security.AddData(request.Configuration); } _dataFeed.AddSubscription(request); // only update our security changes if we actually added data if (!request.IsUniverseSubscription) { addedSubscription = true; } } if (addedSubscription) { var addedMember = universe.AddMember(dateTimeUtc, security); if (addedMember) { additions.Add(security); } } } // return None if there's no changes, otherwise return what we've modified var securityChanges = additions.Count + removals.Count != 0 ? new SecurityChanges(additions, removals) : SecurityChanges.None; // Add currency data feeds that weren't explicitly added in Initialize if (additions.Count > 0) { var addedSubscriptionDataConfigs = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds( _algorithm.Securities, _algorithm.SubscriptionManager, _algorithm.BrokerageModel.DefaultMarkets, securityChanges, _securityService); foreach (var subscriptionDataConfig in addedSubscriptionDataConfigs) { var security = _algorithm.Securities[subscriptionDataConfig.Symbol]; _dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, subscriptionDataConfig, dateTimeUtc, algorithmEndDateUtc)); } } if (securityChanges != SecurityChanges.None) { Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges); } return(securityChanges); }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public IEnumerable <BaseData> Read(SubscriptionDataSource source) { SubscriptionDataSourceReader.CheckRemoteFileCache(); IStreamReader reader = null; try { try { switch (source.TransportMedium) { default: case SubscriptionTransportMedium.Rest: reader = new RestSubscriptionStreamReader(source.Source, source.Headers, _isLiveMode); break; case SubscriptionTransportMedium.LocalFile: reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source); break; case SubscriptionTransportMedium.RemoteFile: reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache, source.Headers); break; } } catch (Exception e) { OnInvalidSource(source, e); yield break; } var raw = ""; while (!reader.EndOfStream) { BaseDataCollection instances = null; try { raw = reader.ReadLine(); var result = _factory.Reader(_config, raw, _date, _isLiveMode); instances = result as BaseDataCollection; if (instances == null && !reader.ShouldBeRateLimited) { OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection")); continue; } } catch (Exception err) { OnReaderError(raw, err); if (!reader.ShouldBeRateLimited) { continue; } } if (_isLiveMode // this shouldn't happen, rest reader is the only one to be rate limited // and in live mode, but just in case... || instances == null && reader.ShouldBeRateLimited) { yield return(instances); } else { foreach (var instance in instances.Data) { if (instance != null && instance.EndTime != default(DateTime)) { yield return(instance); } } } } } finally { if (reader != null) { reader.Dispose(); } } }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> public TimeSlice Sync(IEnumerable <Subscription> subscriptions) { long earlyBirdTicks; var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection var universeData = new Dictionary <Universe, BaseDataCollection>(); var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); SecurityChanges newChanges; do { earlyBirdTicks = MaxDateTimeTicks; newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var packet = new DataFeedPacket(subscription.Security, subscription.Configuration); while (subscription.Current.EmitTimeUtc <= _frontier) { packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); break; } } if (packet.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universe, out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(_frontier, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universe] = collection; } } } if (subscription.Current != null) { if (earlyBirdTicks == MaxDateTimeTicks) { earlyBirdTicks = subscription.Current.EmitTimeUtc.Ticks; } else { // take the earliest between the next piece of data or the current earliest bird earlyBirdTicks = Math.Min(earlyBirdTicks, subscription.Current.EmitTimeUtc.Ticks); } } } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, _frontier, baseDataCollection); } changes += newChanges; }while (newChanges != SecurityChanges.None); var timeSlice = TimeSlice.Create(_frontier, _sliceTimeZone, _cashBook, data, changes, universeDataForTimeSliceCreate); // next frontier time _frontier = new DateTime(Math.Max(earlyBirdTicks, _frontier.Ticks), DateTimeKind.Utc); return(timeSlice); }
public void CoarseUniverseRotatesActiveSecurity() { var startDate = new DateTime(2014, 3, 24); var endDate = new DateTime(2014, 3, 28); var timeProvider = new ManualTimeProvider(TimeZones.NewYork); timeProvider.SetCurrentTime(startDate); var coarseTimes = new List <DateTime> { new DateTime(2014, 3, 25), new DateTime(2014, 3, 25, 23, 0, 0), new DateTime(2014, 3, 27, 1, 0, 0) }.ToHashSet(); var coarseSymbols = new List <Symbol> { Symbols.SPY, Symbols.AAPL, Symbols.MSFT }; var coarseUsaSymbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false); var coarseDataEmittedCount = 0; var lastTime = DateTime.MinValue; var dataQueueHandler = new FuncDataQueueHandler(fdqh => { var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); if (time != lastTime) { lastTime = time; if (coarseTimes.Contains(time)) { // emit coarse data at selected times var coarseData = new BaseDataCollection { Symbol = coarseUsaSymbol }; foreach (var symbol in coarseSymbols) { coarseData.Data.Add( new CoarseFundamental { Symbol = symbol, Time = time, Market = Market.USA, Value = 100 }); } coarseDataEmittedCount++; return(new List <BaseData> { coarseData }); } } return(Enumerable.Empty <BaseData>()); }); var feed = new TestableLiveTradingDataFeed(dataQueueHandler); var algorithm = new AlgorithmStub(feed); algorithm.SetLiveMode(true); var mock = new Mock <ITransactionHandler>(); mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>()); algorithm.Transactions.SetOrderProcessor(mock.Object); var synchronizer = new TestableLiveSynchronizer(timeProvider); synchronizer.Initialize(algorithm, algorithm.DataManager); var mapFileProvider = new LocalDiskMapFileProvider(); feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(), mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer); var symbolIndex = 0; var coarseUniverseSelectionCount = 0; algorithm.AddUniverse( coarse => { coarseUniverseSelectionCount++; // rotate single symbol in universe if (symbolIndex == coarseSymbols.Count) { symbolIndex = 0; } return(new[] { coarseSymbols[symbolIndex++] }); }); algorithm.PostInitialize(); var cancellationTokenSource = new CancellationTokenSource(); Exception exceptionThrown = null; // create a timer to advance time much faster than realtime var timerInterval = TimeSpan.FromMilliseconds(50); var timer = Ref.Create <Timer>(null); timer.Value = new Timer(state => { try { // stop the timer to prevent reentrancy timer.Value.Change(Timeout.Infinite, Timeout.Infinite); var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); if (currentTime.Date > endDate.Date) { feed.Exit(); cancellationTokenSource.Cancel(); return; } timeProvider.Advance(TimeSpan.FromHours(1)); var activeSecuritiesCount = algorithm.ActiveSecurities.Count; Assert.That(activeSecuritiesCount <= 1); // restart the timer timer.Value.Change(timerInterval, timerInterval); } catch (Exception exception) { Log.Error(exception); exceptionThrown = exception; feed.Exit(); cancellationTokenSource.Cancel(); } }, null, TimeSpan.FromSeconds(1), timerInterval); foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token)) { } timer.Value.Dispose(); if (exceptionThrown != null) { throw new Exception("Exception in timer: ", exceptionThrown); } Assert.AreEqual(coarseTimes.Count, coarseDataEmittedCount); Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount); }
/// <summary> /// Applies universe selection the the data feed and algorithm /// </summary> /// <param name="universe">The universe to perform selection on</param> /// <param name="dateTimeUtc">The current date time in utc</param> /// <param name="universeData">The data provided to perform selection with</param> public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData) { IEnumerable <Symbol> selectSymbolsResult; // check if this universe must be filtered with fine fundamental data var fineFiltered = universe as FineFundamentalFilteredUniverse; if (fineFiltered != null) { // perform initial filtering and limit the result selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData); // prepare a BaseDataCollection of FineFundamental instances var fineCollection = new BaseDataCollection(); var dataProvider = new DefaultDataProvider(); foreach (var symbol in selectSymbolsResult) { var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { dateTimeUtc }); var config = FineFundamentalUniverse.CreateConfiguration(symbol); Security security; if (!_algorithm.Securities.TryGetValue(symbol, out security)) { security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase); _algorithm.Securities.Add(security); } var request = new SubscriptionRequest(true, universe, security, config, dateTimeUtc, dateTimeUtc); var enumerator = factory.CreateEnumerator(request, dataProvider); if (enumerator.MoveNext()) { fineCollection.Data.Add(enumerator.Current); } } // perform the fine fundamental universe selection selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection); } else { // perform initial filtering and limit the result selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData); } // check for no changes first if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { return(SecurityChanges.None); } // materialize the enumerable into a set for processing var selections = selectSymbolsResult.ToHashSet(); var additions = new List <Security>(); var removals = new List <Security>(); var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); // remove previously deselected members which were kept in the universe because of holdings or open orders foreach (var member in _pendingRemovals.ToList()) { var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol); if (!member.HoldStock && !openOrders.Any()) { RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc); _pendingRemovals.Remove(member); } } // determine which data subscriptions need to be removed from this universe foreach (var member in universe.Members.Values) { // if we've selected this subscription again, keep it if (selections.Contains(member.Symbol)) { continue; } // don't remove if the universe wants to keep him in if (!universe.CanRemoveMember(dateTimeUtc, member)) { continue; } // remove the member - this marks this member as not being // selected by the universe, but it may remain in the universe // until open orders are closed and the security is liquidated removals.Add(member); // but don't physically remove it from the algorithm if we hold stock or have open orders against it var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol); if (!member.HoldStock && !openOrders.Any()) { RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc); } else { _pendingRemovals.Add(member); } } // find new selections and add them to the algorithm foreach (var symbol in selections) { // create the new security, the algorithm thread will add this at the appropriate time Security security; if (!_algorithm.Securities.TryGetValue(symbol, out security)) { security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase); } var addedSubscription = false; foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc)) { // ask the limiter if we can add another subscription at that resolution string reason; if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason)) { // should we be counting universe subscriptions against user subscriptions limits? _algorithm.Error(reason); Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason); continue; } // add the new subscriptions to the data feed _dataFeed.AddSubscription(request); // only update our security changes if we actually added data if (!request.IsUniverseSubscription) { addedSubscription = true; } } if (addedSubscription) { var addedMember = universe.AddMember(dateTimeUtc, security); if (addedMember) { additions.Add(security); } } } // Add currency data feeds that weren't explicitly added in Initialize if (additions.Count > 0) { var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets); foreach (var security in addedSecurities) { // assume currency feeds are always one subscription per, these are typically quote subscriptions _dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc)); } } // return None if there's no changes, otherwise return what we've modified var securityChanges = additions.Count + removals.Count != 0 ? new SecurityChanges(additions, removals) : SecurityChanges.None; if (securityChanges != SecurityChanges.None) { Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges); } return(securityChanges); }
public void HandlesCoarseFundamentalData() { var algorithm = new AlgorithmStub(); Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA); algorithm.SetUniverse(new FuncUniverse( new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false), new SubscriptionSettings(Resolution.Second, 1, true, false), coarse => coarse.Take(10).Select(x => x.Symbol) )); var lck = new object(); BaseDataCollection list = null; const int coarseDataPointCount = 100000; var timer = new Timer(state => { var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork); Console.WriteLine(currentTime + ": timer.Elapsed"); lock (state) { list = new BaseDataCollection { Symbol = symbol }; list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental { Symbol = SymbolCache.GetSymbol(x.ToString()), Time = currentTime - Time.OneDay, // hard-coded coarse period of one day })); } }, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500)); bool yieldedUniverseData = false; var feed = RunDataFeed(algorithm, fdqh => { lock (lck) { if (list != null) { try { var tmp = list; return(new List <BaseData> { tmp }); } finally { list = null; yieldedUniverseData = true; } } } return(Enumerable.Empty <BaseData>()); }); Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription)); var universeSelectionHadAllData = false; feed.UniverseSelection += (sender, args) => { universeSelectionHadAllData = args.Data.Count == coarseDataPointCount; Console.WriteLine(DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).ToString("o") + ": Fired universe selection. Count: " + args.Data.Count); return(SecurityChanges.None); }; ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts => { }); Assert.IsTrue(yieldedUniverseData); Assert.IsTrue(universeSelectionHadAllData); }
public void FineCoarseFundamentalDataGetsPipedCorrectly() { var lck = new object(); BaseDataCollection list = null; var timer = new Timer(state => { lock (state) { list = new BaseDataCollection { Symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false) }; list.Data.Add(new CoarseFundamental { Symbol = Symbols.AAPL, Time = new DateTime(2014, 04, 24) }); } }, lck, TimeSpan.FromSeconds(0.5), TimeSpan.FromMilliseconds(-1)); var yieldedUniverseData = false; var feed = RunDataFeed(getNextTicksFunction: fdqh => { lock (lck) { if (list != null) { try { var tmp = list; return(new List <BaseData> { tmp }); } finally { list = null; yieldedUniverseData = true; } } } return(Enumerable.Empty <BaseData>()); }); var fineWasCalled = false; _algorithm.AddUniverse(coarse => coarse.Take(1).Select(x => x.Symbol), fine => { var symbol = fine.First().Symbol; if (symbol == Symbols.AAPL) { fineWasCalled = true; } return(new[] { symbol }); }); var receivedFundamentalsData = false; ConsumeBridge(feed, TimeSpan.FromSeconds(2), ts => { if (ts.UniverseData.Count > 0 && ts.UniverseData.First().Value.Data.First() is Fundamentals) { receivedFundamentalsData = true; } }, sendUniverseData: true); timer.Dispose(); Assert.IsTrue(yieldedUniverseData); Assert.IsTrue(receivedFundamentalsData); Assert.IsTrue(fineWasCalled); }
/// <summary> /// Applies universe selection the the data feed and algorithm /// </summary> /// <param name="universe">The universe to perform selection on</param> /// <param name="dateTimeUtc">The current date time in utc</param> /// <param name="universeData">The data provided to perform selection with</param> public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData) { var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); if (dateTimeUtc > algorithmEndDateUtc) { return(SecurityChanges.None); } IEnumerable <Symbol> selectSymbolsResult; // check if this universe must be filtered with fine fundamental data var fineFiltered = universe as FineFundamentalFilteredUniverse; if (fineFiltered != null) { // perform initial filtering and limit the result selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData); if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { // prepare a BaseDataCollection of FineFundamental instances var fineCollection = new BaseDataCollection(); var dataProvider = new DefaultDataProvider(); foreach (var symbol in selectSymbolsResult) { var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { dateTimeUtc }); var config = FineFundamentalUniverse.CreateConfiguration(symbol); var exchangeHours = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType).ExchangeHours; var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(symbol.ID.Market, symbol, symbol.ID.SecurityType, CashBook.AccountCurrency); var quoteCash = _algorithm.Portfolio.CashBook[symbolProperties.QuoteCurrency]; var security = new Equity(symbol, exchangeHours, quoteCash, symbolProperties); var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), dateTimeUtc, dateTimeUtc); using (var enumerator = factory.CreateEnumerator(request, dataProvider)) { if (enumerator.MoveNext()) { fineCollection.Data.Add(enumerator.Current); } } } // perform the fine fundamental universe selection selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection); } } else { // perform initial filtering and limit the result selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData); } // check for no changes first if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { return(SecurityChanges.None); } // materialize the enumerable into a set for processing var selections = selectSymbolsResult.ToHashSet(); var additions = new List <Security>(); var removals = new List <Security>(); // remove previously deselected members which were kept in the universe because of holdings or open orders foreach (var member in _pendingRemovals.ToList()) { if (IsSafeToRemove(member)) { RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc); _pendingRemovals.Remove(member); } } // determine which data subscriptions need to be removed from this universe foreach (var member in universe.Members.Values) { // if we've selected this subscription again, keep it if (selections.Contains(member.Symbol)) { continue; } // don't remove if the universe wants to keep him in if (!universe.CanRemoveMember(dateTimeUtc, member)) { continue; } // remove the member - this marks this member as not being // selected by the universe, but it may remain in the universe // until open orders are closed and the security is liquidated removals.Add(member); if (IsSafeToRemove(member)) { RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc); } else { _pendingRemovals.Add(member); } } var keys = _pendingSecurityAdditions.Keys; if (keys.Any() && keys.Single() != dateTimeUtc) { // if the frontier moved forward then we've added these securities to the algorithm _pendingSecurityAdditions.Clear(); } Dictionary <Symbol, Security> pendingAdditions; if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions)) { // keep track of created securities so we don't create the same security twice, leads to bad things :) pendingAdditions = new Dictionary <Symbol, Security>(); _pendingSecurityAdditions[dateTimeUtc] = pendingAdditions; } // find new selections and add them to the algorithm foreach (var symbol in selections) { // create the new security, the algorithm thread will add this at the appropriate time Security security; if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security)) { security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase); pendingAdditions.Add(symbol, security); } var addedSubscription = false; foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc)) { // add the new subscriptions to the data feed _dataFeed.AddSubscription(request); // only update our security changes if we actually added data if (!request.IsUniverseSubscription) { addedSubscription = true; } } if (addedSubscription) { var addedMember = universe.AddMember(dateTimeUtc, security); if (addedMember) { additions.Add(security); } } } // return None if there's no changes, otherwise return what we've modified var securityChanges = additions.Count + removals.Count != 0 ? new SecurityChanges(additions, removals) : SecurityChanges.None; // Add currency data feeds that weren't explicitly added in Initialize if (additions.Count > 0) { var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets, securityChanges); foreach (var security in addedSecurities) { // assume currency feeds are always one subscription per, these are typically quote subscriptions _dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, new SubscriptionDataConfig(security.Subscriptions.First()), dateTimeUtc, algorithmEndDateUtc)); } } if (securityChanges != SecurityChanges.None) { Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges); } return(securityChanges); }
/// <summary> /// Applies universe selection the the data feed and algorithm /// </summary> /// <param name="universe">The universe to perform selection on</param> /// <param name="dateTimeUtc">The current date time in utc</param> /// <param name="universeData">The data provided to perform selection with</param> public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData) { var settings = universe.UniverseSettings; // perform initial filtering and limit the result var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData); // check for no changes first if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { return(SecurityChanges.None); } // materialize the enumerable into a set for processing var selections = selectSymbolsResult.ToHashSet(); // create a hash set of our existing subscriptions by sid var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol); var additions = new List <Security>(); var removals = new List <Security>(); // determine which data subscriptions need to be removed from this universe foreach (var member in universe.Members.Values) { var config = member.SubscriptionDataConfig; // if we've selected this subscription again, keep it if (selections.Contains(config.Symbol)) { continue; } // don't remove if the universe wants to keep him in if (!universe.CanRemoveMember(dateTimeUtc, member)) { continue; } // remove the member - this marks this member as not being // selected by the universe, but it may remain in the universe // until open orders are closed and the security is liquidated removals.Add(member); // but don't physically remove it from the algorithm if we hold stock or have open orders against it var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol); if (!member.HoldStock && !openOrders.Any()) { // safe to remove the member from the universe universe.RemoveMember(dateTimeUtc, member); // we need to mark this security as untradeable while it has no data subscription // it is expected that this function is called while in sync with the algo thread, // so we can make direct edits to the security here member.Cache.Reset(); _dataFeed.RemoveSubscription(member.Symbol); } } // find new selections and add them to the algorithm foreach (var symbol in selections) { // we already have a subscription for this symbol so don't re-add it if (existingSubscriptions.Contains(symbol)) { continue; } // ask the limiter if we can add another subscription at that resolution string reason; if (!_limiter.CanAddSubscription(settings.Resolution, out reason)) { _algorithm.Error(reason); Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscriptions: " + reason); break; } // create the new security, the algorithm thread will add this at the appropriate time Security security; if (!_algorithm.Securities.TryGetValue(symbol, out security)) { security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, universe.SecurityInitializer, symbol, settings.Resolution, settings.FillForward, settings.Leverage, settings.ExtendedMarketHours, false, false, false); } additions.Add(security); // add the new subscriptions to the data feed if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone))) { universe.AddMember(dateTimeUtc, security); } } // Add currency data feeds that weren't explicitly added in Initialize if (additions.Count > 0) { var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets); foreach (var security in addedSecurities) { _dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)); } } // return None if there's no changes, otherwise return what we've modified return(additions.Count + removals.Count != 0 ? new SecurityChanges(additions, removals) : SecurityChanges.None); }
/// <summary> /// Returns the symbols defined by the user for this universe /// </summary> /// <param name="utcTime">The current utc time</param> /// <param name="data">The symbols to remain in the universe</param> /// <returns>The data that passes the filter</returns> public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data) { return(_symbols); }
/// <summary> /// Applies universe selection the the data feed and algorithm /// </summary> /// <param name="universe">The universe to perform selection on</param> /// <param name="dateTimeUtc">The current date time in utc</param> /// <param name="universeData">The data provided to perform selection with</param> public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData) { // perform initial filtering and limit the result var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData); // check for no changes first if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { return(SecurityChanges.None); } // materialize the enumerable into a set for processing var selections = selectSymbolsResult.ToHashSet(); var additions = new List <Security>(); var removals = new List <Security>(); var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); // determine which data subscriptions need to be removed from this universe foreach (var member in universe.Members.Values) { // if we've selected this subscription again, keep it if (selections.Contains(member.Symbol)) { continue; } // don't remove if the universe wants to keep him in if (!universe.CanRemoveMember(dateTimeUtc, member)) { continue; } // remove the member - this marks this member as not being // selected by the universe, but it may remain in the universe // until open orders are closed and the security is liquidated removals.Add(member); // but don't physically remove it from the algorithm if we hold stock or have open orders against it var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol); if (!member.HoldStock && !openOrders.Any()) { // safe to remove the member from the universe universe.RemoveMember(dateTimeUtc, member); // we need to mark this security as untradeable while it has no data subscription // it is expected that this function is called while in sync with the algo thread, // so we can make direct edits to the security here member.Cache.Reset(); foreach (var subscription in universe.GetSubscriptionRequests(member, dateTimeUtc, algorithmEndDateUtc)) { if (subscription.IsUniverseSubscription) { removals.Remove(member); } else { _dataFeed.RemoveSubscription(subscription.Configuration); } } // remove symbol mappings for symbols removed from universes // TODO : THIS IS BAD! SymbolCache.TryRemove(member.Symbol); } } // find new selections and add them to the algorithm foreach (var symbol in selections) { // create the new security, the algorithm thread will add this at the appropriate time Security security; if (!_algorithm.Securities.TryGetValue(symbol, out security)) { security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase); } var addedMember = universe.AddMember(dateTimeUtc, security); var addedSubscription = false; foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc)) { // ask the limiter if we can add another subscription at that resolution string reason; if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason)) { // should we be counting universe subscriptions against user subscriptions limits? _algorithm.Error(reason); Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason); continue; } // add the new subscriptions to the data feed _dataFeed.AddSubscription(request); // only update our security changes if we actually added data if (!request.IsUniverseSubscription) { addedSubscription = addedMember; } } if (addedSubscription) { additions.Add(security); } } // Add currency data feeds that weren't explicitly added in Initialize if (additions.Count > 0) { var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets); foreach (var security in addedSecurities) { // assume currency feeds are always one subscription per, these are typically quote subscriptions _dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc)); } } // return None if there's no changes, otherwise return what we've modified var securityChanges = additions.Count + removals.Count != 0 ? new SecurityChanges(additions, removals) : SecurityChanges.None; if (securityChanges != SecurityChanges.None) { Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges); } return(securityChanges); }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="cancellationToken">The cancellation token to stop enumeration</param> public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions, CancellationToken cancellationToken) { var delayedSubscriptionFinished = new Queue <Subscription>(); while (!cancellationToken.IsCancellationRequested) { var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(1); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection Dictionary <Universe, BaseDataCollection> universeData = null; // lazy construction for performance var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); var frontierUtc = _timeProvider.GetUtcNow(); _frontierTimeProvider.SetCurrentTimeUtc(frontierUtc); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } DataFeedPacket packet = null; while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { if (packet == null) { // for performance, lets be selfish about creating a new instance packet = new DataFeedPacket( subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse ); } // If our subscription is a universe, and we get a delisting event emitted for it, then // the universe itself should be unselected and removed, because the Symbol that the // universe is based on has been delisted. Doing the disposal here allows us to // process the delisting at this point in time before emitting out to the algorithm. // This is very useful for universes that can be delisted, such as ETF constituent // universes (e.g. for ETF constituent universes, since the ETF itself is used to create // the universe Symbol (and set as its underlying), once the ETF is delisted, the // universe should cease to exist, since there are no more constituents of that ETF). if (subscription.IsUniverseSelectionSubscription && subscription.Current.Data is Delisting) { subscription.Universes.Single().Dispose(); } packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished.Enqueue(subscription); break; } } if (packet?.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData != null && universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, frontierUtc, subscription.Configuration.Symbol, packetData); } if (universeData == null) { universeData = new Dictionary <Universe, BaseDataCollection>(); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested) { var universe = subscription.Universes.Single(); // check if a universe selection isn't already scheduled for this disposed universe if (universeData == null || !universeData.ContainsKey(universe)) { if (universeData == null) { universeData = new Dictionary <Universe, BaseDataCollection>(); } // we force trigger one last universe selection for this disposed universe, so it deselects all subscriptions it added universeData[universe] = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol); } // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } if (universeData != null && universeData.Count > 0) { // if we are going to perform universe selection we emit an empty // time pulse to align algorithm time with current frontier yield return(_timeSliceFactory.CreateTimePulse(frontierUtc)); foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); } changes += newChanges; }while (newChanges != SecurityChanges.None || _universeSelection.AddPendingInternalDataFeeds(frontierUtc)); var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate); while (delayedSubscriptionFinished.Count > 0) { // these subscriptions added valid data to the packet // we need to trigger OnSubscriptionFinished after we create the TimeSlice // else it will drop the data var subscription = delayedSubscriptionFinished.Dequeue(); OnSubscriptionFinished(subscription); } yield return(timeSlice); } }
/// <summary> /// Applies universe selection the the data feed and algorithm /// </summary> /// <param name="universe">The universe to perform selection on</param> /// <param name="dateTimeUtc">The current date time in utc</param> /// <param name="universeData">The data provided to perform selection with</param> public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData) { var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); if (dateTimeUtc > algorithmEndDateUtc) { return(SecurityChanges.None); } IEnumerable <Symbol> selectSymbolsResult; // check if this universe must be filtered with fine fundamental data var fineFiltered = universe as FineFundamentalFilteredUniverse; if (fineFiltered != null // if the universe has been disposed we don't perform selection. This us handled bellow by 'Universe.PerformSelection' // but in this case we directly call 'SelectSymbols' because we want to perform fine selection even if coarse returns the same // symbols, see 'Universe.PerformSelection', which detects this and returns 'Universe.Unchanged' && !universe.DisposeRequested) { // perform initial filtering and limit the result selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData); if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { // prepare a BaseDataCollection of FineFundamental instances var fineCollection = new BaseDataCollection(); // Create a dictionary of CoarseFundamental keyed by Symbol that also has FineFundamental // Coarse raw data has SID collision on: CRHCY R735QTJ8XC9X var allCoarse = universeData.Data.OfType <CoarseFundamental>(); var coarseData = allCoarse.Where(c => c.HasFundamentalData) .DistinctBy(c => c.Symbol) .ToDictionary(c => c.Symbol); // Remove selected symbols that does not have fine fundamental data var anyDoesNotHaveFundamentalData = false; // only pre filter selected symbols if there actually is any coarse data. This way we can support custom universe filtered by fine fundamental data // which do not use coarse data as underlying, in which case it could happen that we try to load fine fundamental data that is missing, but no problem, // 'FineFundamentalSubscriptionEnumeratorFactory' won't emit it if (allCoarse.Any()) { selectSymbolsResult = selectSymbolsResult .Where( symbol => { var result = coarseData.ContainsKey(symbol); anyDoesNotHaveFundamentalData |= !result; return(result); } ); } if (!_anyDoesNotHaveFundamentalDataWarningLogged && anyDoesNotHaveFundamentalData) { _algorithm.Debug("Note: Your coarse selection filter was updated to exclude symbols without fine fundamental data. Make sure your coarse filter excludes symbols where HasFundamental is false."); _anyDoesNotHaveFundamentalDataWarningLogged = true; } // use all available threads, the entire system is waiting for this to complete var options = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; Parallel.ForEach(selectSymbolsResult, options, symbol => { var config = FineFundamentalUniverse.CreateConfiguration(symbol); var security = _securityService.CreateSecurity(symbol, config, addToSymbolCache: false); var localStartTime = dateTimeUtc.ConvertFromUtc(config.ExchangeTimeZone).AddDays(-1); var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { localStartTime }); var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), localStartTime, localStartTime); using (var enumerator = factory.CreateEnumerator(request, _dataProvider)) { if (enumerator.MoveNext()) { lock (fineCollection.Data) { fineCollection.Data.Add(enumerator.Current); } } } }); // WARNING -- HACK ATTACK -- WARNING // Fine universes are considered special due to their chaining behavior. // As such, we need a means of piping the fine data read in here back to the data feed // so that it can be properly emitted via a TimeSlice.Create call. There isn't a mechanism // in place for this function to return such data. The following lines are tightly coupled // to the universeData dictionaries in SubscriptionSynchronizer and LiveTradingDataFeed and // rely on reference semantics to work. universeData.Data = new List <BaseData>(); foreach (var fine in fineCollection.Data.OfType <FineFundamental>()) { var fundamentals = new Fundamentals { Symbol = fine.Symbol, Time = fine.Time, EndTime = fine.EndTime, DataType = fine.DataType, AssetClassification = fine.AssetClassification, CompanyProfile = fine.CompanyProfile, CompanyReference = fine.CompanyReference, EarningReports = fine.EarningReports, EarningRatios = fine.EarningRatios, FinancialStatements = fine.FinancialStatements, OperationRatios = fine.OperationRatios, SecurityReference = fine.SecurityReference, ValuationRatios = fine.ValuationRatios, Market = fine.Symbol.ID.Market }; CoarseFundamental coarse; if (coarseData.TryGetValue(fine.Symbol, out coarse)) { // the only time the coarse data won't exist is if the selection function // doesn't use the data provided, and instead returns a constant list of // symbols -- coupled with a potential hole in the data fundamentals.Value = coarse.Value; fundamentals.Volume = coarse.Volume; fundamentals.DollarVolume = coarse.DollarVolume; fundamentals.HasFundamentalData = coarse.HasFundamentalData; // set the fine fundamental price property to yesterday's closing price fine.Value = coarse.Value; } universeData.Data.Add(fundamentals); } // END -- HACK ATTACK -- END // perform the fine fundamental universe selection selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection); } } else { // perform initial filtering and limit the result selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData); } // materialize the enumerable into a set for processing var selections = selectSymbolsResult.ToHashSet(); var additions = new List <Security>(); var removals = new List <Security>(); // first check for no pending removals, even if the universe selection // didn't change we might need to remove a security because a position was closed RemoveSecurityFromUniverse( _pendingRemovalsManager.CheckPendingRemovals(selections, universe), removals, dateTimeUtc, algorithmEndDateUtc); // check for no changes second if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged)) { return(SecurityChanges.None); } // determine which data subscriptions need to be removed from this universe foreach (var member in universe.Securities.Values.OrderBy(member => member.Security.Symbol.SecurityType)) { var security = member.Security; // if we've selected this subscription again, keep it if (selections.Contains(security.Symbol)) { continue; } // don't remove if the universe wants to keep him in if (!universe.CanRemoveMember(dateTimeUtc, security)) { continue; } // remove the member - this marks this member as not being // selected by the universe, but it may remain in the universe // until open orders are closed and the security is liquidated removals.Add(security); RemoveSecurityFromUniverse(_pendingRemovalsManager.TryRemoveMember(security, universe), removals, dateTimeUtc, algorithmEndDateUtc); } Dictionary <Symbol, Security> pendingAdditions; if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions)) { // if the frontier moved forward then we've added these securities to the algorithm _pendingSecurityAdditions.Clear(); // keep track of created securities so we don't create the same security twice, leads to bad things :) pendingAdditions = new Dictionary <Symbol, Security>(); _pendingSecurityAdditions[dateTimeUtc] = pendingAdditions; } // find new selections and add them to the algorithm foreach (var symbol in selections) { if (universe.Securities.ContainsKey(symbol)) { // if its already part of the universe no need to re add it continue; } // create the new security, the algorithm thread will add this at the appropriate time Security security; if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security)) { // For now this is required for retro compatibility with usages of security.Subscriptions var configs = _algorithm.SubscriptionManager.SubscriptionDataConfigService.Add(symbol, universe.UniverseSettings.Resolution, universe.UniverseSettings.FillForward, universe.UniverseSettings.ExtendedMarketHours, dataNormalizationMode: universe.UniverseSettings.DataNormalizationMode); security = _securityService.CreateSecurity(symbol, configs, universe.UniverseSettings.Leverage, (symbol.ID.SecurityType == SecurityType.Option || symbol.ID.SecurityType == SecurityType.FutureOption)); pendingAdditions.Add(symbol, security); } var addedSubscription = false; var dataFeedAdded = false; foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc, _algorithm.SubscriptionManager.SubscriptionDataConfigService)) { if (security.Symbol == request.Configuration.Symbol && // Just in case check its the same symbol, else AddData will throw. !security.Subscriptions.Contains(request.Configuration)) { // For now this is required for retro compatibility with usages of security.Subscriptions security.AddData(request.Configuration); } var toRemove = _currencySubscriptionDataConfigManager.GetSubscriptionDataConfigToRemove(request.Configuration.Symbol); if (toRemove != null) { Log.Trace($"UniverseSelection.ApplyUniverseSelection(): Removing internal currency data feed {toRemove}"); _dataManager.RemoveSubscription(toRemove); } // 'dataFeedAdded' will help us notify the user for security changes only once per non internal subscription // for example two universes adding the sample configuration, we don't want two notifications dataFeedAdded = _dataManager.AddSubscription(request); // only update our security changes if we actually added data if (!request.IsUniverseSubscription) { addedSubscription = true; _internalSubscriptionManager.AddedSubscriptionRequest(request); } } if (addedSubscription) { var addedMember = universe.AddMember(dateTimeUtc, security); if (addedMember && dataFeedAdded) { additions.Add(security); } } } // return None if there's no changes, otherwise return what we've modified var securityChanges = additions.Count + removals.Count != 0 ? new SecurityChanges(additions, removals) : SecurityChanges.None; // Add currency data feeds that weren't explicitly added in Initialize if (additions.Count > 0) { EnsureCurrencyDataFeeds(securityChanges); } if (securityChanges != SecurityChanges.None && Log.DebuggingEnabled) { // for performance lets not create the message string if debugging is not enabled // this can be executed many times and its in the algorithm thread Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges); } return(securityChanges); }
public void CoarseFundamentalDataGetsPipedCorrectly() { var lck = new object(); BaseDataCollection list = null; var timer = new Timer(state => { var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork); lock (state) { list = new BaseDataCollection { Symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false) }; list.Data.Add(new CoarseFundamental { Symbol = Symbols.SPY, Time = currentTime - Time.OneDay, // hard-coded coarse period of one day }); } }, lck, TimeSpan.FromSeconds(0.5), TimeSpan.FromMilliseconds(-1)); var yieldedUniverseData = false; var feed = RunDataFeed(getNextTicksFunction: fdqh => { lock (lck) { if (list != null) { try { var tmp = list; return(new List <BaseData> { tmp }); } finally { list = null; yieldedUniverseData = true; } } } return(Enumerable.Empty <BaseData>()); }); _algorithm.AddUniverse(coarse => coarse.Take(10).Select(x => x.Symbol)); var receivedCoarseData = false; ConsumeBridge(feed, TimeSpan.FromSeconds(2), ts => { if (ts.UniverseData.Count > 0 && ts.UniverseData.First().Value.Data.First() is CoarseFundamental) { receivedCoarseData = true; } }, sendUniverseData: true); timer.Dispose(); Assert.IsTrue(yieldedUniverseData); Assert.IsTrue(receivedCoarseData); }