public void EmitsData()
        {
            var algorithm = new AlgorithmStub(forex: new List<string> {"EURUSD"});

            // job is used to send into DataQueueHandler
            var job = new LiveNodePacket();
            // result handler is used due to dependency in SubscriptionDataReader
            var resultHandler = new BacktestingResultHandler();
            var dataFileProvider = new DefaultDataFileProvider();

            var lastTime = DateTime.MinValue;
            var timeProvider = new RealTimeProvider();
            var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
            {
                var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.EasternStandard);
                if (time == lastTime) return Enumerable.Empty<BaseData>();
                lastTime = time;
                 return Enumerable.Range(0, 9).Select(x => new Tick(time.AddMilliseconds(x*100), Symbols.EURUSD, 1.3m, 1.2m, 1.3m));
            });

            var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
            var mapFileProvider = new LocalDiskMapFileProvider();
            feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), dataFileProvider);

            var feedThreadStarted = new ManualResetEvent(false);
            Task.Factory.StartNew(() =>
            {
                feedThreadStarted.Set();
                feed.Run();
            });

            // wait for feed.Run to actually begin
            feedThreadStarted.WaitOne();

            var emittedData = false;
            ConsumeBridge(feed, TimeSpan.FromSeconds(10), true, ts =>
            {
                if (ts.Slice.Count != 0)
                {
                    emittedData = true;
                    Console.WriteLine("HasData: " + ts.Slice.Bars[Symbols.EURUSD].EndTime);
                    Console.WriteLine();
                }
            });

            Assert.IsTrue(emittedData);
        }
        public void ReadsZipEntryNames()
        {
            var time = new DateTime(2016, 03, 03, 12, 48, 15);
            var source = Path.Combine("TestData", "20151224_quote_american.zip");
            var config = new SubscriptionDataConfig(typeof (ZipEntryName), Symbol.Create("XLRE", SecurityType.Option, Market.USA), Resolution.Tick,
                TimeZones.NewYork, TimeZones.NewYork, false, false, false);
            var dataFileProvider = new DefaultDataFileProvider();
            var factory = new ZipEntryNameSubscriptionDataSourceReader(dataFileProvider, config, time, false);
            var expected = new[]
            {
                Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 21m, new DateTime(2016, 08, 19)),
                Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 22m, new DateTime(2016, 08, 19)),
                Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Put, 37m, new DateTime(2016, 08, 19)),
            };

            var actual = factory.Read(new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName)).ToList();

            // we only really care about the symbols
            CollectionAssert.AreEqual(expected, actual.Select(x => x.Symbol));
            Assert.IsTrue(actual.All(x => x is ZipEntryName));
        }
        public void ReadsFineFundamental(FineFundamentalTestParameters parameters)
        {
            var stopwatch = Stopwatch.StartNew();
            var rows = new List<FineFundamental>();

            var config = new SubscriptionDataConfig(typeof(FineFundamental), parameters.Symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
            var security = new Security(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), config, new Cash(CashBook.AccountCurrency, 0, 1), SymbolProperties.GetDefault(CashBook.AccountCurrency));
            var request = new SubscriptionRequest(false, null, security, config, parameters.StartDate, parameters.EndDate);
            var fileProvider = new DefaultDataFileProvider();

            var factory = new FineFundamentalSubscriptionEnumeratorFactory();
            var enumerator = factory.CreateEnumerator(request, fileProvider);
            while (enumerator.MoveNext())
            {
                var current = enumerator.Current as FineFundamental;
                rows.Add(current);
            }

            stopwatch.Stop();
            Console.WriteLine("Total rows: {0}, elapsed time: {1}", rows.Count, stopwatch.Elapsed);

            Assert.AreEqual(parameters.RowCount, rows.Count);

            if (parameters.RowCount != 1) return;

            var row = rows[0];
            Assert.AreEqual(parameters.CompanyShortName, row.CompanyReference.ShortName);
            Assert.AreEqual(parameters.Symbol, row.Symbol);
            Assert.AreEqual(parameters.Symbol != Symbol.Empty ? parameters.Symbol.Value : null, row.CompanyReference.PrimarySymbol);
            Assert.AreEqual(parameters.Symbol != Symbol.Empty ? parameters.Symbol.Value : null, row.SecurityReference.SecuritySymbol);
            Assert.AreEqual(parameters.Ebitda3M, row.FinancialStatements.IncomeStatement.EBITDA.ThreeMonths);
            Assert.AreEqual(parameters.Ebitda12M, row.FinancialStatements.IncomeStatement.EBITDA.TwelveMonths);
            Assert.AreEqual(parameters.Ebitda12M, row.FinancialStatements.IncomeStatement.EBITDA);
            Assert.AreEqual(parameters.CostOfRevenue3M, row.FinancialStatements.IncomeStatement.CostOfRevenue.ThreeMonths);
            Assert.AreEqual(parameters.CostOfRevenue12M, row.FinancialStatements.IncomeStatement.CostOfRevenue.TwelveMonths);
            Assert.AreEqual(parameters.CostOfRevenue12M, row.FinancialStatements.IncomeStatement.CostOfRevenue);
            Assert.AreEqual(parameters.EquityPerShareGrowth1Y, row.EarningRatios.EquityPerShareGrowth.OneYear);
            Assert.AreEqual(parameters.EquityPerShareGrowth1Y, row.EarningRatios.EquityPerShareGrowth);
            Assert.AreEqual(parameters.PeRatio, row.ValuationRatios.PERatio);
        }
        public void TestsFileSystemDataFeedSpeed()
        {
            var job = new BacktestNodePacket();
            var resultHandler = new BacktestingResultHandler();
            var mapFileProvider = new LocalDiskMapFileProvider();
            var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
            var dataFileProvider = new DefaultDataFileProvider();

            var algorithm = new BenchmarkTest();
            var feed = new FileSystemDataFeed();

            feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataFileProvider);
            algorithm.Initialize();

            var feedThreadStarted = new ManualResetEvent(false);
            Task.Factory.StartNew(() =>
            {
                feedThreadStarted.Set();
                feed.Run();
            });
            feedThreadStarted.WaitOne();

            var stopwatch = Stopwatch.StartNew();
            var lastMonth = -1;
            var count = 0;
            foreach (var timeSlice in feed)
            {
                if (timeSlice.Time.Month != lastMonth)
                {
                    Console.WriteLine(DateTime.Now + " - Time: " + timeSlice.Time);
                    lastMonth = timeSlice.Time.Month;
                }
                count++;
            }
            Console.WriteLine("Count: " + count);

            stopwatch.Stop();
            Console.WriteLine("Elapsed time: " + stopwatch.Elapsed);
        }
        private IDataFeed RunDataFeed(IAlgorithm algorithm, out FuncDataQueueHandler dataQueueHandler, ITimeProvider timeProvider = null, Func<FuncDataQueueHandler, IEnumerable<BaseData>> getNextTicksFunction = null)
        {
            getNextTicksFunction = getNextTicksFunction ?? (fdqh => fdqh.Subscriptions.Select(symbol => new Tick(DateTime.Now, symbol, 1, 2){Quantity = 1}));

            // job is used to send into DataQueueHandler
            var job = new LiveNodePacket();
            // result handler is used due to dependency in SubscriptionDataReader
            var resultHandler = new BacktestingResultHandler(); // new ResultHandlerStub();

            dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction);

            var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
            var mapFileProvider = new LocalDiskMapFileProvider();
            var fileProvider = new DefaultDataFileProvider();
            feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider);

            var feedThreadStarted = new ManualResetEvent(false);
            Task.Factory.StartNew(() =>
            {
                feedThreadStarted.Set();
                feed.Run();
            });

            // wait for feed.Run to actually begin
            feedThreadStarted.WaitOne();

            return feed;
        }
Esempio n. 6
0
        /// <summary>
        /// Applies universe selection the the data feed and algorithm
        /// </summary>
        /// <param name="universe">The universe to perform selection on</param>
        /// <param name="dateTimeUtc">The current date time in utc</param>
        /// <param name="universeData">The data provided to perform selection with</param>
        public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
        {
            IEnumerable <Symbol> selectSymbolsResult;

            // check if this universe must be filtered with fine fundamental data
            var fineFiltered = universe as FineFundamentalFilteredUniverse;

            if (fineFiltered != null)
            {
                // perform initial filtering and limit the result
                selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);

                // prepare a BaseDataCollection of FineFundamental instances
                var fineCollection   = new BaseDataCollection();
                var dataFileProvider = new DefaultDataFileProvider();

                foreach (var symbol in selectSymbolsResult)
                {
                    var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { dateTimeUtc });
                    var config  = FineFundamentalUniverse.CreateConfiguration(symbol);

                    Security security;
                    if (!_algorithm.Securities.TryGetValue(symbol, out security))
                    {
                        security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
                        _algorithm.Securities.Add(security);
                    }

                    var request    = new SubscriptionRequest(true, universe, security, config, dateTimeUtc, dateTimeUtc);
                    var enumerator = factory.CreateEnumerator(request, dataFileProvider);
                    if (enumerator.MoveNext())
                    {
                        fineCollection.Data.Add(enumerator.Current);
                    }
                }

                // perform the fine fundamental universe selection
                selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
            }
            else
            {
                // perform initial filtering and limit the result
                selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
            }

            // check for no changes first
            if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
            {
                return(SecurityChanges.None);
            }

            // materialize the enumerable into a set for processing
            var selections = selectSymbolsResult.ToHashSet();

            var additions           = new List <Security>();
            var removals            = new List <Security>();
            var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);

            // remove previously deselected members which were kept in the universe because of holdings or open orders
            foreach (var member in _pendingRemovals.ToList())
            {
                var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
                if (!member.HoldStock && !openOrders.Any())
                {
                    RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);

                    _pendingRemovals.Remove(member);
                }
            }

            // determine which data subscriptions need to be removed from this universe
            foreach (var member in universe.Members.Values)
            {
                // if we've selected this subscription again, keep it
                if (selections.Contains(member.Symbol))
                {
                    continue;
                }

                // don't remove if the universe wants to keep him in
                if (!universe.CanRemoveMember(dateTimeUtc, member))
                {
                    continue;
                }

                // remove the member - this marks this member as not being
                // selected by the universe, but it may remain in the universe
                // until open orders are closed and the security is liquidated
                removals.Add(member);

                // but don't physically remove it from the algorithm if we hold stock or have open orders against it
                var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
                if (!member.HoldStock && !openOrders.Any())
                {
                    RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
                }
                else
                {
                    _pendingRemovals.Add(member);
                }
            }

            // find new selections and add them to the algorithm
            foreach (var symbol in selections)
            {
                // create the new security, the algorithm thread will add this at the appropriate time
                Security security;
                if (!_algorithm.Securities.TryGetValue(symbol, out security))
                {
                    security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
                }

                var addedSubscription = false;

                foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
                {
                    // ask the limiter if we can add another subscription at that resolution
                    string reason;
                    if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason))
                    {
                        // should we be counting universe subscriptions against user subscriptions limits?

                        _algorithm.Error(reason);
                        Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason);
                        continue;
                    }
                    // add the new subscriptions to the data feed
                    _dataFeed.AddSubscription(request);

                    // only update our security changes if we actually added data
                    if (!request.IsUniverseSubscription)
                    {
                        addedSubscription = true;
                    }
                }

                if (addedSubscription)
                {
                    var addedMember = universe.AddMember(dateTimeUtc, security);

                    if (addedMember)
                    {
                        additions.Add(security);
                    }
                }
            }

            // Add currency data feeds that weren't explicitly added in Initialize
            if (additions.Count > 0)
            {
                var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
                foreach (var security in addedSecurities)
                {
                    // assume currency feeds are always one subscription per, these are typically quote subscriptions
                    _dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc));
                }
            }

            // return None if there's no changes, otherwise return what we've modified
            var securityChanges = additions.Count + removals.Count != 0
                ? new SecurityChanges(additions, removals)
                : SecurityChanges.None;

            if (securityChanges != SecurityChanges.None)
            {
                Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
            }

            return(securityChanges);
        }