Пример #1
0
        /// <summary>
        /// Performs universe selection using the data specified
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="data">The symbols to remain in the universe</param>
        /// <returns>The data that passes the filter</returns>
        public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
        {
            var futuresUniverseDataCollection = data as FuturesChainUniverseDataCollection;

            if (futuresUniverseDataCollection == null)
            {
                throw new ArgumentException($"Expected data of type '{typeof(FuturesChainUniverseDataCollection).Name}'");
            }

            var underlying = new Tick {
                Time = utcTime
            };

            // date change detection needs to be done in exchange time zone
            if (_cacheDate == data.Time.ConvertFromUtc(Future.Exchange.TimeZone).Date)
            {
                return(Unchanged);
            }

            var availableContracts = futuresUniverseDataCollection.Data.Select(x => x.Symbol);
            var results            = Future.ContractFilter.Filter(new FutureFilterUniverse(availableContracts, underlying));

            // if results are not dynamic, we cache them and won't call filtering till the end of the day
            if (!results.IsDynamic)
            {
                _cacheDate = data.Time.ConvertFromUtc(Future.Exchange.TimeZone).Date;
            }

            var resultingSymbols = results.ToHashSet();

            futuresUniverseDataCollection.FilteredContracts = resultingSymbols;

            return(resultingSymbols);
        }
        /// <summary>
        /// Performs universe selection using the data specified
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="data">The symbols to remain in the universe</param>
        /// <returns>The data that passes the filter</returns>
        public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
        {
            var optionsUniverseDataCollection = data as OptionChainUniverseDataCollection;

            if (optionsUniverseDataCollection == null)
            {
                throw new ArgumentException($"Expected data of type '{typeof(OptionChainUniverseDataCollection).Name}'");
            }

            // date change detection needs to be done in exchange time zone
            if (_cacheDate == data.Time.ConvertFromUtc(Option.Exchange.TimeZone).Date)
            {
                return(Unchanged);
            }

            var availableContracts = optionsUniverseDataCollection.Data.Select(x => x.Symbol);
            var results            = Option.ContractFilter.Filter(new OptionFilterUniverse(availableContracts, optionsUniverseDataCollection.Underlying));

            // if results are not dynamic, we cache them and won't call filtering till the end of the day
            if (!results.IsDynamic)
            {
                _cacheDate = data.Time.ConvertFromUtc(Option.Exchange.TimeZone).Date;
            }

            // always prepend the underlying symbol
            var resultingSymbols = _underlyingSymbol.Concat(results).ToHashSet();

            // we save off the filtered results to the universe data collection for later
            // population into the OptionChain. This is non-ideal and could be remedied by
            // the universe subscription emitting a special type after selection that could
            // be checked for in TimeSlice.Create, but for now this will do
            optionsUniverseDataCollection.FilteredContracts = resultingSymbols;

            return(resultingSymbols);
        }
Пример #3
0
        /// <summary>
        /// Performs universe selection using the data specified
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="data">The symbols to remain in the universe</param>
        /// <returns>The data that passes the filter</returns>
        public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
        {
            // date change detection needs to be done in exchange time zone
            var exchangeDate = data.Time.ConvertFromUtc(Option.Exchange.TimeZone).Date;

            if (_cacheDate == exchangeDate)
            {
                return(Unchanged);
            }

            var availableContracts = data.Data.Select(x => x.Symbol);

            // we will only update unique strikes when there is an exchange date change
            _optionFilterUniverse.Refresh(availableContracts, data.Underlying, _lastExchangeDate != exchangeDate);
            _lastExchangeDate = exchangeDate;

            var results = Option.ContractFilter.Filter(_optionFilterUniverse);

            // if results are not dynamic, we cache them and won't call filtering till the end of the day
            if (!results.IsDynamic)
            {
                _cacheDate = data.Time.ConvertFromUtc(Option.Exchange.TimeZone).Date;
            }

            // always prepend the underlying symbol
            return(_underlyingSymbol.Concat(results));
        }
Пример #4
0
        /// <summary>
        /// Performs universe selection based on the symbol mapping
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="data">Empty data</param>
        /// <returns>The symbols to use</returns>
        public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
        {
            yield return(_security.Symbol.Canonical);

            var mapFile = _mapFileProvider.ResolveMapFile(new SubscriptionDataConfig(Configuration,
                                                                                     dataMappingMode: UniverseSettings.DataMappingMode,
                                                                                     symbol: _security.Symbol.Canonical));

            var mappedSymbol = mapFile.GetMappedSymbol(utcTime.ConvertFromUtc(_security.Exchange.TimeZone));

            if (!string.IsNullOrEmpty(mappedSymbol) && mappedSymbol != _mappedSymbol)
            {
                if (_currentSymbol != null)
                {
                    // let's emit the old and new for the mapping date
                    yield return(_currentSymbol);
                }
                _mappedSymbol = mappedSymbol;

                _currentSymbol = _security.Symbol.Canonical
                                 .UpdateMappedSymbol(mappedSymbol, Configuration.ContractDepthOffset)
                                 .Underlying;
            }

            if (_currentSymbol != null)
            {
                ((IContinuousSecurity)_security).Mapped = _currentSymbol;
                yield return(_currentSymbol);
            }
        }
Пример #5
0
        /// <summary>
        /// Performs universe selection using the data specified
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="data">The symbols to remain in the universe</param>
        /// <returns>The data that passes the filter</returns>
        public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
        {
            var optionsUniverseDataCollection = data as OptionChainUniverseDataCollection;
            if (optionsUniverseDataCollection == null)
            {
                throw new ArgumentException(string.Format("Expected data of type '{0}'", typeof (OptionChainUniverseDataCollection).Name));
            }

            _underlying = optionsUniverseDataCollection.Underlying ?? _underlying;
            optionsUniverseDataCollection.Underlying = _underlying;

            if (_underlying == null || data.Data.Count == 0)
            {
                return Unchanged;
            }

            var availableContracts = optionsUniverseDataCollection.Data.Select(x => x.Symbol);
            var results = _option.ContractFilter.Filter(availableContracts, _underlying).ToHashSet();

            // we save off the filtered results to the universe data collection for later
            // population into the OptionChain. This is non-ideal and could be remedied by
            // the universe subscription emitting a special type after selection that could
            // be checked for in TimeSlice.Create, but for now this will do
            optionsUniverseDataCollection.FilteredContracts = results;

            return results;
        }
Пример #6
0
        /// <summary>
        /// Performs universe selection using the data specified
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="data">The symbols to remain in the universe</param>
        /// <returns>The data that passes the filter</returns>
        public IEnumerable <Symbol> PerformSelection(DateTime utcTime, BaseDataCollection data)
        {
            // select empty set of symbols after dispose requested
            if (DisposeRequested)
            {
                OnSelectionChanged();
                return(Enumerable.Empty <Symbol>());
            }

            var result = SelectSymbols(utcTime, data);

            if (ReferenceEquals(result, Unchanged))
            {
                return(Unchanged);
            }

            var selections = result.ToHashSet();
            var hasDiffs   = _previousSelections.AreDifferent(selections);

            _previousSelections = selections;
            if (!hasDiffs)
            {
                return(Unchanged);
            }

            OnSelectionChanged(selections);
            return(selections);
        }
Пример #7
0
        /// <summary>
        /// Performs universe selection using the data specified
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="data">The symbols to remain in the universe</param>
        /// <returns>The data that passes the filter</returns>
        public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
        {
            var optionsUniverseDataCollection = data as OptionChainUniverseDataCollection;

            if (optionsUniverseDataCollection == null)
            {
                throw new ArgumentException(string.Format("Expected data of type '{0}'", typeof(OptionChainUniverseDataCollection).Name));
            }

            _underlying = optionsUniverseDataCollection.Underlying ?? _underlying;
            optionsUniverseDataCollection.Underlying = _underlying;

            if (_underlying == null || data.Data.Count == 0)
            {
                return(Unchanged);
            }

            var availableContracts = optionsUniverseDataCollection.Data.Select(x => x.Symbol);
            var results            = _option.ContractFilter.Filter(availableContracts, _underlying).ToHashSet();

            // we save off the filtered results to the universe data collection for later
            // population into the OptionChain. This is non-ideal and could be remedied by
            // the universe subscription emitting a special type after selection that could
            // be checked for in TimeSlice.Create, but for now this will do
            optionsUniverseDataCollection.FilteredContracts = results;

            return(results);
        }
Пример #8
0
 /// <summary>
 /// Performs universe selection using the data specified
 /// </summary>
 /// <param name="utcTime">The current utc time</param>
 /// <param name="data">The symbols to remain in the universe</param>
 /// <returns>The data that passes the filter</returns>
 public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
 {
     using (Py.GIL())
     {
         var symbols  = _universe.SelectSymbols(utcTime, data) as PyObject;
         var iterator = symbols.GetIterator();
         foreach (PyObject symbol in iterator)
         {
             yield return(symbol.GetAndDispose <Symbol>());
         }
         iterator.Dispose();
         symbols.Dispose();
     }
 }
        /// <summary>
        /// Reads the specified <paramref name="source"/>
        /// </summary>
        /// <param name="source">The source to be read</param>
        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
        public IEnumerable<BaseData> Read(SubscriptionDataSource source)
        {
            IStreamReader reader = null;
            var instances = new BaseDataCollection();
            try
            {
                switch (source.TransportMedium)
                {
                    default:
                    case SubscriptionTransportMedium.Rest:
                        reader = new RestSubscriptionStreamReader(source.Source);
                        break;
                    case SubscriptionTransportMedium.LocalFile:
                        reader = new LocalFileSubscriptionStreamReader(source.Source);
                        break;
                    case SubscriptionTransportMedium.RemoteFile:
                        reader = new RemoteFileSubscriptionStreamReader(source.Source, Globals.Cache);
                        break;
                }

                var raw = "";
                try
                {
                    raw = reader.ReadLine();
                    var result = _factory.Reader(_config, raw, _date, _isLiveMode);
                    instances = result as BaseDataCollection;
                    if (instances == null)
                    {
                        OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
                    }
                }
                catch (Exception err)
                {
                    OnReaderError(raw, err);
                }

                foreach (var instance in instances.Data)
                {
                    yield return instance;
                }
            }
            finally
            {
                if (reader != null)
                    reader.Dispose();
            }
        }
Пример #10
0
        /// <summary>
        /// Performs universe selection using the data specified
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="data">The symbols to remain in the universe</param>
        /// <returns>The data that passes the filter</returns>
        public IEnumerable <Symbol> PerformSelection(DateTime utcTime, BaseDataCollection data)
        {
            var result = SelectSymbols(utcTime, data);

            if (ReferenceEquals(result, Unchanged))
            {
                return(Unchanged);
            }

            var selections = result.ToHashSet();
            var hasDiffs   = _previousSelections.Except(selections).Union(selections.Except(_previousSelections)).Any();

            _previousSelections = selections;
            if (!hasDiffs)
            {
                return(Unchanged);
            }
            return(selections);
        }
Пример #11
0
        /// <summary>
        /// Performs universe selection using the data specified
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="data">The symbols to remain in the universe</param>
        /// <returns>The data that passes the filter</returns>
        public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
        {
            var underlying = new Tick {
                Time = utcTime
            };

            // date change detection needs to be done in exchange time zone
            if (_cacheDate == data.Time.ConvertFromUtc(Future.Exchange.TimeZone).Date)
            {
                return(Unchanged);
            }

            var availableContracts = data.Data.Select(x => x.Symbol);
            var results            = Future.ContractFilter.Filter(new FutureFilterUniverse(availableContracts, underlying));

            // if results are not dynamic, we cache them and won't call filtering till the end of the day
            if (!results.IsDynamic)
            {
                _cacheDate = data.Time.ConvertFromUtc(Future.Exchange.TimeZone).Date;
            }

            return(results);
        }
 /// <summary>
 /// Returns the symbols defined by the user for this universe
 /// </summary>
 /// <param name="utcTime">The curren utc time</param>
 /// <param name="data">The symbols to remain in the universe</param>
 /// <returns>The data that passes the filter</returns>
 public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
 {
     return(_selector(utcTime));
 }
Пример #13
0
 /// <summary>
 /// Performs universe selection using the data specified
 /// </summary>
 /// <param name="utcTime">The current utc time</param>
 /// <param name="data">The symbols to remain in the universe</param>
 /// <returns>The data that passes the filter</returns>
 public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
 {
     return _selector(data.Data.OfType<FineFundamental>());
 }
Пример #14
0
 /// <summary>
 /// Performs universe selection using the data specified
 /// </summary>
 /// <param name="utcTime">The current utc time</param>
 /// <param name="data">The symbols to remain in the universe</param>
 /// <returns>The data that passes the filter</returns>
 public abstract IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data);
Пример #15
0
 /// <summary>
 /// Performs an initial, coarse filter
 /// </summary>
 /// <param name="utcTime">The current utc time</param>
 /// <param name="data">The coarse fundamental data</param>
 /// <returns>The data that passes the filter</returns>
 public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
 {
     return _universeSelector(data.Data);
 }
Пример #16
0
        public void HandlesCoarseFundamentalData()
        {
            var algorithm = new AlgorithmStub();
            Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
            algorithm.AddUniverse(new FuncUniverse(
                new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
                new UniverseSettings(Resolution.Second, 1, true, false, TimeSpan.Zero),
                coarse => coarse.Take(10).Select(x => x.Symbol) 
                ));

            var lck = new object();
            BaseDataCollection list = null;
            const int coarseDataPointCount = 100000;
            var timer = new Timer(state =>
            {
                var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
                Console.WriteLine(currentTime + ": timer.Elapsed");

                lock (state)
                {
                    list = new BaseDataCollection {Symbol = symbol};
                    list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
                    {
                        Symbol = SymbolCache.GetSymbol(x.ToString()),
                        Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
                    }));
                }
            }, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));

            bool yieldedUniverseData = false;
            var feed = RunDataFeed(algorithm, fdqh =>
            {
                lock (lck)
                {
                    if (list != null)
                        try
                        {
                            var tmp = list;
                            return new List<BaseData> { tmp };
                        }
                        finally
                        {
                            list = null;
                            yieldedUniverseData = true;
                        }
                }
                return Enumerable.Empty<BaseData>();
            });

            Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));

            var universeSelectionHadAllData = false;


            ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
            {
            });

            Assert.IsTrue(yieldedUniverseData);
            Assert.IsTrue(universeSelectionHadAllData);
        }
Пример #17
0
 /// <summary>
 /// Performs universe selection using the data specified
 /// </summary>
 /// <param name="utcTime">The current utc time</param>
 /// <param name="data">The symbols to remain in the universe</param>
 /// <returns>The data that passes the filter</returns>
 public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
 {
     return Universe.SelectSymbols(utcTime, data);
 }
Пример #18
0
        /// <summary>
        /// Applies universe selection the the data feed and algorithm
        /// </summary>
        /// <param name="universe">The universe to perform selection on</param>
        /// <param name="dateTimeUtc">The current date time in utc</param>
        /// <param name="universeData">The data provided to perform selection with</param>
        public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
        {
            var settings = universe.SubscriptionSettings;

            var limit = 1000; //daily/hourly limit
            var resolution = settings.Resolution;
            switch (resolution)
            {
                case Resolution.Tick:
                    limit = _algorithm.Securities.TickLimit;
                    break;
                case Resolution.Second:
                    limit = _algorithm.Securities.SecondLimit;
                    break;
                case Resolution.Minute:
                    limit = _algorithm.Securities.MinuteLimit;
                    break;
            }

            // subtract current subscriptions that can't be removed
            limit -= _algorithm.Securities.Count(x => x.Value.Resolution == resolution && x.Value.HoldStock);

            if (limit < 1)
            {
                // if we don't have room for more securities then we can't really do anything here.
                _algorithm.Error("Unable to add  more securities from universe selection due to holding stock.");
                return SecurityChanges.None;
            }

            // perform initial filtering and limit the result
            var selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData.Data);

            // check for no changes first
            if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
            {
                return SecurityChanges.None;
            }

            var selections = selectSymbolsResult.Take(limit).ToHashSet();

            // create a hash set of our existing subscriptions by sid
            var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol);

            var additions = new List<Security>();
            var removals = new List<Security>();

            // determine which data subscriptions need to be removed for this market
            foreach (var subscription in _dataFeed.Subscriptions)
            {
                // universes can only remove members of their own
                if (!universe.ContainsMember(subscription.Security)) continue;

                // never remove universe selection subscriptions
                if (subscription.IsUniverseSelectionSubscription) continue;

                var config = subscription.Configuration;

                // never remove internal feeds
                if (config.IsInternalFeed) continue;

                // if we've selected this subscription again, keep it
                if (selections.Contains(config.Symbol)) continue;

                // let the algorithm know this security has been removed from the universe
                removals.Add(subscription.Security);

                // but don't physically remove it from the algorithm if we hold stock or have open orders against it
                var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol);
                if (!subscription.Security.HoldStock && !openOrders.Any())
                {
                    // we need to mark this security as untradeable while it has no data subscription
                    // it is expected that this function is called while in sync with the algo thread,
                    // so we can make direct edits to the security here
                    subscription.Security.Cache.Reset();

                    if (_dataFeed.RemoveSubscription(subscription))
                    {
                        universe.RemoveMember(subscription.Security);
                    }
                }
            }

            // find new selections and add them to the algorithm
            foreach (var symbol in selections)
            {
                // we already have a subscription for this symbol so don't re-add it
                if (existingSubscriptions.Contains(symbol)) continue;
                
                // create the new security, the algorithm thread will add this at the appropriate time
                Security security;
                if (!_algorithm.Securities.TryGetValue(symbol, out security))
                {
                    security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase,
                        symbol,
                        settings.Resolution,
                        settings.FillForward,
                        settings.Leverage,
                        settings.ExtendedMarketHours,
                        false,
                        false,
                        false);
                }

                additions.Add(security);

                // add the new subscriptions to the data feed
                if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)))
                {
                    universe.AddMember(security);
                }
            }

            // return None if there's no changes, otherwise return what we've modified
            return additions.Count + removals.Count != 0
                ? new SecurityChanges(additions, removals)
                : SecurityChanges.None;
        }
Пример #19
0
 /// <summary>
 /// Performs universe selection using the data specified
 /// </summary>
 /// <param name="utcTime">The current utc time</param>
 /// <param name="data">The symbols to remain in the universe</param>
 /// <returns>The data that passes the filter</returns>
 public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
 {
     return(_selector(data.Data.OfType <CoarseFundamental>()));
 }
Пример #20
0
        /// <summary>
        /// Applies universe selection the the data feed and algorithm
        /// </summary>
        /// <param name="universe">The universe to perform selection on</param>
        /// <param name="dateTimeUtc">The current date time in utc</param>
        /// <param name="universeData">The data provided to perform selection with</param>
        public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
        {
            var settings = universe.UniverseSettings;

            // perform initial filtering and limit the result
            var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);

            // check for no changes first
            if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
            {
                return SecurityChanges.None;
            }

            // materialize the enumerable into a set for processing
            var selections = selectSymbolsResult.ToHashSet();

            // create a hash set of our existing subscriptions by sid
            var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol);

            var additions = new List<Security>();
            var removals = new List<Security>();

            // determine which data subscriptions need to be removed from this universe
            foreach (var member in universe.Members.Values)
            {
                var config = member.SubscriptionDataConfig;

                // if we've selected this subscription again, keep it
                if (selections.Contains(config.Symbol)) continue;

                // don't remove if the universe wants to keep him in
                if (!universe.CanRemoveMember(dateTimeUtc, member)) continue;

                // remove the member - this marks this member as not being
                // selected by the universe, but it may remain in the universe
                // until open orders are closed and the security is liquidated
                removals.Add(member);

                // but don't physically remove it from the algorithm if we hold stock or have open orders against it
                var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol);
                if (!member.HoldStock && !openOrders.Any())
                {
                    // safe to remove the member from the universe
                    universe.RemoveMember(dateTimeUtc, member);

                    // we need to mark this security as untradeable while it has no data subscription
                    // it is expected that this function is called while in sync with the algo thread,
                    // so we can make direct edits to the security here
                    member.Cache.Reset();
                    _dataFeed.RemoveSubscription(member.Symbol);
                }
            }

            // find new selections and add them to the algorithm
            foreach (var symbol in selections)
            {
                // we already have a subscription for this symbol so don't re-add it
                if (existingSubscriptions.Contains(symbol)) continue;

                // ask the limiter if we can add another subscription at that resolution
                string reason;
                if (!_limiter.CanAddSubscription(settings.Resolution, out reason))
                {
                    _algorithm.Error(reason);
                    Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscriptions: " + reason);
                    break;
                }
                
                // create the new security, the algorithm thread will add this at the appropriate time
                Security security;
                if (!_algorithm.Securities.TryGetValue(symbol, out security))
                {
                    security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, universe.SecurityInitializer,
                        symbol,
                        settings.Resolution,
                        settings.FillForward,
                        settings.Leverage,
                        settings.ExtendedMarketHours,
                        false,
                        false,
                        false);
                }

                additions.Add(security);

                // add the new subscriptions to the data feed
                if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)))
                {
                    universe.AddMember(dateTimeUtc, security);
                }
            }

            // Add currency data feeds that weren't explicitly added in Initialize
            if (additions.Count > 0)
            {
                var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
                foreach (var security in addedSecurities)
                {
                    _dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone));
                }
            }

            // return None if there's no changes, otherwise return what we've modified
            return additions.Count + removals.Count != 0
                ? new SecurityChanges(additions, removals)
                : SecurityChanges.None;
        }
Пример #21
0
        /// <summary>
        /// Creates the correct enumerator factory for the given request
        /// </summary>
        private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request)
        {
            if (request.IsUniverseSubscription)
            {
                if (request.Universe is UserDefinedUniverse)
                {
                    // Trigger universe selection when security added/removed after Initialize
                    var universe = (UserDefinedUniverse) request.Universe;
                    universe.CollectionChanged += (sender, args) =>
                    {
                        var items =
                            args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
                            args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;

                        if (items == null || _frontierUtc == DateTime.MinValue) return;

                        var symbol = items.OfType<Symbol>().FirstOrDefault();
                        if (symbol == null) return;

                        var collection = new BaseDataCollection(_frontierUtc, symbol);
                        var changes = _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection);
                        _algorithm.OnSecuritiesChanged(changes);
                    };

                    return new UserDefinedUniverseSubscriptionEnumeratorFactory(request.Universe as UserDefinedUniverse, MarketHoursDatabase.FromDataFolder());
                }
                if (request.Configuration.Type == typeof (CoarseFundamental))
                {
                    return new BaseDataCollectionSubscriptionEnumeratorFactory();
                }
                if (request.Configuration.Type == typeof(FineFundamental))
                {
                    return new FineFundamentalSubscriptionEnumeratorFactory();
                }
                if (request.Universe is OptionChainUniverse)
                {
                    return new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e));
                }
            }

            var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity
                ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) 
                : MapFileResolver.Empty;

            return new PostCreateConfigureSubscriptionEnumeratorFactory(
                new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, mapFileResolver, _factorFileProvider, false, true),
                enumerator => ConfigureEnumerator(request, false, enumerator)
                );
        }
Пример #22
0
 /// <summary>
 /// Performs an initial, coarse filter
 /// </summary>
 /// <param name="utcTime">The current utc time</param>
 /// <param name="data">The coarse fundamental data</param>
 /// <returns>The data that passes the filter</returns>
 public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
 {
     return(_universeSelector(data.Data.Cast <T>()));
 }
Пример #23
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = request.Universe.Configuration;

            var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

            IEnumerator<BaseData> enumerator;
            
            var userDefined = request.Universe as UserDefinedUniverse;
            if (userDefined != null)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());

                // spoof a tick on the requested interval to trigger the universe selection function
                var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
                enumerator = enumeratorFactory.CreateEnumerator(request);

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator<BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;

                // Trigger universe selection when security added/removed after Initialize
                userDefined.CollectionChanged += (sender, args) =>
                {
                    var items =
                           args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
                           args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;

                    if (items == null || _frontierUtc == DateTime.MinValue) return;

                    var symbol = items.OfType<Symbol>().FirstOrDefault();
                    if (symbol == null) return;

                    var collection = new BaseDataCollection(_frontierUtc, symbol);
                    var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
                    _algorithm.OnSecuritiesChanged(changes);
                };
            }
            else if (config.Type == typeof (CoarseFundamental))
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());

                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol});

                var enqueable = new EnqueueableEnumerator<BaseData>();
                _exchange.SetDataHandler(config.Symbol, data =>
                {
                    enqueable.Enqueue(data);
                });
                enumerator = enqueable;
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());

                // each time we exhaust we'll new up this enumerator stack
                var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
                {
                    var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
                    var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                    var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                    var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
                    var factorEnumerator = factory.Read(source).GetEnumerator();
                    var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
                    var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
                    return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
                });
                
                // rate limit the refreshing of the stack to the requested interval
                var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                var enqueueable = new EnqueueableEnumerator<BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);

            return subscription;
        }
Пример #24
0
        /// <summary>
        /// Applies universe selection the the data feed and algorithm
        /// </summary>
        /// <param name="universe">The universe to perform selection on</param>
        /// <param name="dateTimeUtc">The current date time in utc</param>
        /// <param name="universeData">The data provided to perform selection with</param>
        public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
        {
            // perform initial filtering and limit the result
            var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);

            // check for no changes first
            if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
            {
                return SecurityChanges.None;
            }

            // materialize the enumerable into a set for processing
            var selections = selectSymbolsResult.ToHashSet();

            var additions = new List<Security>();
            var removals = new List<Security>();
            var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);

            // determine which data subscriptions need to be removed from this universe
            foreach (var member in universe.Members.Values)
            {
                // if we've selected this subscription again, keep it
                if (selections.Contains(member.Symbol)) continue;

                // don't remove if the universe wants to keep him in
                if (!universe.CanRemoveMember(dateTimeUtc, member)) continue;

                // remove the member - this marks this member as not being
                // selected by the universe, but it may remain in the universe
                // until open orders are closed and the security is liquidated
                removals.Add(member);

                // but don't physically remove it from the algorithm if we hold stock or have open orders against it
                var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
                if (!member.HoldStock && !openOrders.Any())
                {
                    // safe to remove the member from the universe
                    universe.RemoveMember(dateTimeUtc, member);

                    // we need to mark this security as untradeable while it has no data subscription
                    // it is expected that this function is called while in sync with the algo thread,
                    // so we can make direct edits to the security here
                    member.Cache.Reset();
                    foreach (var subscription in universe.GetSubscriptionRequests(member, dateTimeUtc, algorithmEndDateUtc))
                    {
                        if (subscription.IsUniverseSubscription)
                        {
                            removals.Remove(member);
                        }
                        else
                        {
                            _dataFeed.RemoveSubscription(subscription.Configuration);
                        }
                    }

                    // remove symbol mappings for symbols removed from universes // TODO : THIS IS BAD!
                    SymbolCache.TryRemove(member.Symbol);
                }
            }

            // find new selections and add them to the algorithm
            foreach (var symbol in selections)
            {
                // create the new security, the algorithm thread will add this at the appropriate time
                Security security;
                if (!_algorithm.Securities.TryGetValue(symbol, out security))
                {
                    security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
                }

                var addedMember = universe.AddMember(dateTimeUtc, security);

                var addedSubscription = false;
                foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
                {
                    // ask the limiter if we can add another subscription at that resolution
                    string reason;
                    if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason))
                    {
                        // should we be counting universe subscriptions against user subscriptions limits?

                        _algorithm.Error(reason);
                        Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason);
                        continue;
                    }

                    // add the new subscriptions to the data feed
                    _dataFeed.AddSubscription(request);

                    // only update our security changes if we actually added data
                    if (!request.IsUniverseSubscription)
                    {
                        addedSubscription = addedMember;
                    }
                }

                if (addedSubscription)
                {
                    additions.Add(security);
                }
            }

            // Add currency data feeds that weren't explicitly added in Initialize
            if (additions.Count > 0)
            {
                var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
                foreach (var security in addedSecurities)
                {
                    // assume currency feeds are always one subscription per, these are typically quote subscriptions
                    _dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc));
                }
            }

            // return None if there's no changes, otherwise return what we've modified
            var securityChanges = additions.Count + removals.Count != 0
                ? new SecurityChanges(additions, removals)
                : SecurityChanges.None;

            if (securityChanges != SecurityChanges.None)
            {
                Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
            }

            return securityChanges;
        }
Пример #25
0
 /// <summary>
 /// Returns the symbols defined by the user for this universe
 /// </summary>
 /// <param name="utcTime">The curren utc time</param>
 /// <param name="data">The symbols to remain in the universe</param>
 /// <returns>The data that passes the filter</returns>
 public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
 {
     return _selector(utcTime);
 }
Пример #26
0
 /// <summary>
 /// Performs universe selection using the data specified
 /// </summary>
 /// <param name="utcTime">The current utc time</param>
 /// <param name="data">The symbols to remain in the universe</param>
 /// <returns>The data that passes the filter</returns>
 public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
 {
     return(Universe.SelectSymbols(utcTime, data));
 }
Пример #27
0
        /// <summary>
        /// Syncs the specifies subscriptions at the frontier time
        /// </summary>
        /// <param name="frontier">The time used for syncing, data in the future won't be included in this time slice</param>
        /// <param name="subscriptions">The subscriptions to sync</param>
        /// <param name="sliceTimeZone">The time zone of the created slice object</param>
        /// <param name="cashBook">The cash book, used for creating the cash book updates</param>
        /// <param name="nextFrontier">The next frontier time as determined by the first piece of data in the future ahead of the frontier.
        /// This value will equal DateTime.MaxValue when the subscriptions are all finished</param>
        /// <returns>A time slice for the specified frontier time</returns>
        public TimeSlice Sync(DateTime frontier, IEnumerable<Subscription> subscriptions, DateTimeZone sliceTimeZone, CashBook cashBook, out DateTime nextFrontier)
        {
            var changes = SecurityChanges.None;
            nextFrontier = DateTime.MaxValue;
            var earlyBirdTicks = nextFrontier.Ticks;
            var data = new List<DataFeedPacket>();
            var universeData = new Dictionary<Universe, BaseDataCollection>();
            
            SecurityChanges newChanges;
            do
            {

                universeData.Clear();
                newChanges = SecurityChanges.None;
                foreach (var subscription in subscriptions)
                {
                    if (subscription.EndOfStream)
                    {
                        OnSubscriptionFinished(subscription);
                        continue;
                    }

                    // prime if needed
                    if (subscription.Current == null)
                    {
                        if (!subscription.MoveNext())
                        {
                            OnSubscriptionFinished(subscription);
                            continue;
                        }
                    }

                    var packet = new DataFeedPacket(subscription.Security, subscription.Configuration);
                    data.Add(packet);

                    var configuration = subscription.Configuration;
                    var offsetProvider = subscription.OffsetProvider;
                    var currentOffsetTicks = offsetProvider.GetOffsetTicks(frontier);
                    while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks)
                    {
                        // we want bars rounded using their subscription times, we make a clone
                        // so we don't interfere with the enumerator's internal logic
                        var clone = subscription.Current.Clone(subscription.Current.IsFillForward);
                        clone.Time = clone.Time.ExchangeRoundDown(configuration.Increment, subscription.Security.Exchange.Hours, configuration.ExtendedMarketHours);
                        packet.Add(clone);
                        if (!subscription.MoveNext())
                        {
                            OnSubscriptionFinished(subscription);
                            break;
                        }
                    }

                    // we have new universe data to select based on, store the subscription data until the end
                    if (subscription.IsUniverseSelectionSubscription && packet.Count > 0)
                    {
                        // assume that if the first item is a base data collection then the enumerator handled the aggregation,
                        // otherwise, load all the the data into a new collection instance
                        var packetBaseDataCollection = packet.Data[0] as BaseDataCollection;
                        var packetData = packetBaseDataCollection == null
                            ? packet.Data
                            : packetBaseDataCollection.Data;

                        BaseDataCollection collection;
                        if (!universeData.TryGetValue(subscription.Universe, out collection))
                        {
                            if (packetBaseDataCollection is OptionChainUniverseDataCollection)
                            {
                                var current = subscription.Current as OptionChainUniverseDataCollection;
                                var underlying = current != null ? current.Underlying : null;
                                collection = new OptionChainUniverseDataCollection(frontier, subscription.Configuration.Symbol, packetData, underlying);
                            }
                            else
                            {
                                collection = new BaseDataCollection(frontier, subscription.Configuration.Symbol, packetData);
                            }

                            universeData[subscription.Universe] = collection;
                        }
                        else
                        {
                            collection.Data.AddRange(packetData);
                        }
                    }

                    if (subscription.Current != null)
                    {
                        // take the earliest between the next piece of data or the next tz discontinuity
                        earlyBirdTicks = Math.Min(earlyBirdTicks, Math.Min(subscription.Current.EndTime.Ticks - currentOffsetTicks, offsetProvider.GetNextDiscontinuity()));
                    }
                }

                foreach (var kvp in universeData)
                {
                    var universe = kvp.Key;
                    var baseDataCollection = kvp.Value;
                    newChanges += _universeSelection.ApplyUniverseSelection(universe, frontier, baseDataCollection);
                }

                changes += newChanges;
            }
            while (newChanges != SecurityChanges.None);

            nextFrontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc);

            return TimeSlice.Create(frontier, sliceTimeZone, cashBook, data, changes);
        }