private IDataFeed RunDataFeed(IAlgorithm algorithm, out FuncDataQueueHandler dataQueueHandler, ITimeProvider timeProvider = null, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null) { getNextTicksFunction = getNextTicksFunction ?? (fdqh => fdqh.Subscriptions.Select(symbol => new Tick(DateTime.Now, symbol, 1, 2) { Quantity = 1 })); // job is used to send into DataQueueHandler var job = new LiveNodePacket(); // result handler is used due to dependency in SubscriptionDataReader var resultHandler = new BacktestingResultHandler(); // new ResultHandlerStub(); dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction); var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider); var mapFileProvider = new LocalDiskMapFileProvider(); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider)); var feedThreadStarted = new ManualResetEvent(false); Task.Factory.StartNew(() => { feedThreadStarted.Set(); feed.Run(); }); // wait for feed.Run to actually begin feedThreadStarted.WaitOne(); return(feed); }
public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecondDataPoint, Resolution dataResolution) { var start = new DateTime(2019, 12, 9); var end = new DateTime(2019, 12, 12); var mapFileProvider = new LocalDiskMapFileProvider(); var mapFileResolver = new MapFileResolver(mapFileProvider.Get(Market.USA)); var dataReader = new SubscriptionDataReader( new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, dataResolution, TimeZones.NewYork, TimeZones.NewYork, false, false, false), start, end, mapFileResolver, new LocalDiskFactorFileProvider(mapFileProvider), LinqExtensions.Range(start, end, time => time + TimeSpan.FromDays(1)), false, new TestDataCacheProvider { Data = data } ); Assert.IsTrue(dataReader.MoveNext()); Assert.AreEqual(shouldEmitSecondDataPoint, dataReader.MoveNext()); }
public void ChecksMapFileFirstDate() { var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; algorithm.Initialize(); algorithm.PostInitialize(); var dataProvider = new DefaultDataProvider(); var resultHandler = new TestResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, true, enablePriceScaling: false); var universe = algorithm.UniverseManager.Single().Value; var security = algorithm.AddEquity("AAA", Resolution.Daily); var securityConfig = security.Subscriptions.First(); // start date is before the first date in the map file var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, new DateTime(2001, 12, 1), new DateTime(2016, 11, 1)); var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider); // should initialize the data source reader enumerator.MoveNext(); enumerator.Dispose(); factory.DisposeSafely(); resultHandler.Exit(); var message = ((DebugPacket)resultHandler.Messages.Single()).Message; Assert.IsTrue(message.Equals( "The starting date for symbol AAA, 2001-11-30, has been adjusted to match map file first date 2002-05-22.")); }
public override void Initialize() { SetStartDate(2014, 3, 25); //Set Start Date SetEndDate(2014, 4, 7); //Set End Date SetCash(100000); //Set Strategy Cash // Set our DataNormalizationMode to raw UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw; _googl = AddEquity(Ticker, Resolution.Daily).Symbol; // Get our factor file for this regression var dataProvider = Composer.Instance.GetExportedValueByTypeName <IDataProvider>(Config.Get("data-provider", "DefaultDataProvider")); var mapFileProvider = new LocalDiskMapFileProvider(); mapFileProvider.Initialize(dataProvider); var factorFileProvider = new LocalDiskFactorFileProvider(); factorFileProvider.Initialize(mapFileProvider, dataProvider); _factorFile = factorFileProvider.Get(_googl) as CorporateFactorProvider; // Prime our expected values _expectedRawPrices.MoveNext(); }
public void RetrievesFromDisk() { var provider = new LocalDiskMapFileProvider(); var mapFiles = provider.Get(QuantConnect.Market.USA); Assert.IsNotEmpty(mapFiles); }
public void DividendsDistribution(string exDividendDateStr, decimal expectedDistribution) { var dividendProvider = new DividendEventProvider(); var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false); var factorFile = new LocalDiskFactorFileProvider(); var mapFile = new LocalDiskMapFileProvider(); var start = new DateTime(1998, 01, 02); dividendProvider.Initialize(config, factorFile.Get(Symbols.AAPL), mapFile.Get(Market.USA).ResolveMapFile(Symbols.AAPL, start), start); var exDividendDate = DateTime.ParseExact(exDividendDateStr, DateFormat.EightCharacter, CultureInfo.InvariantCulture); var events = dividendProvider .GetEvents(new NewTradableDateEventArgs(exDividendDate, null, Symbols.AAPL, null)) .ToList(); // ex dividend date does not emit anything Assert.AreEqual(0, events.Count); events = dividendProvider .GetEvents(new NewTradableDateEventArgs(exDividendDate.AddDays(1), null, Symbols.AAPL, null)) .ToList(); Assert.AreEqual(1, events.Count); var dividend = events[0] as Dividend; Assert.IsNotNull(dividend); Assert.AreEqual(expectedDistribution, dividend.Distribution); }
private static IDataFeed RunLiveDataFeed( IAlgorithm algorithm, DateTime startDate, IEnumerable <Symbol> symbols, ITimeProvider timeProvider, DataManager dataManager, FuncDataQueueHandler funcDataQueueHandler = null) { var feed = new TestableLiveTradingDataFeed(funcDataQueueHandler ?? new FuncDataQueueHandler(x => Enumerable.Empty <BaseData>()), timeProvider); var mapFileProvider = new LocalDiskMapFileProvider(); feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(), mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), dataManager); foreach (var symbol in symbols) { var config = algorithm.Securities[symbol].SubscriptionDataConfig; var request = new SubscriptionRequest(false, null, algorithm.Securities[symbol], config, startDate, Time.EndOfTime); feed.AddSubscription(request); } var feedThreadStarted = new ManualResetEvent(false); Task.Factory.StartNew(() => { feedThreadStarted.Set(); feed.Run(); }); feedThreadStarted.WaitOne(); return(feed); }
public void RetrievesFromDisk() { var provider = new LocalDiskMapFileProvider(); var mapFiles = provider.Get(VigiothCapital.QuantTrader.Market.USA); Assert.IsNotEmpty(mapFiles); }
public void CachesValueAndReturnsSameReference() { var provider = new LocalDiskMapFileProvider(); var factorFile1 = provider.Get(Symbols.SPY); var factorFile2 = provider.Get(Symbols.SPY); Assert.IsTrue(ReferenceEquals(factorFile1, factorFile2)); }
public void CachesValueAndReturnsSameReference() { var provider = new LocalDiskMapFileProvider(); var mapFiles1 = provider.Get(VigiothCapital.QuantTrader.Market.USA); var mapFiles2 = provider.Get(VigiothCapital.QuantTrader.Market.USA); Assert.IsTrue(ReferenceEquals(mapFiles1, mapFiles2)); }
private IDataFeed RunDataFeed(out FuncDataQueueHandler dataQueueHandler, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null, Resolution resolution = Resolution.Second, List <string> equities = null, List <string> forex = null, List <string> crypto = null) { _algorithm.SetStartDate(_startDate); var lastTime = _manualTimeProvider.GetUtcNow(); getNextTicksFunction = getNextTicksFunction ?? (fdqh => { var time = _manualTimeProvider.GetUtcNow(); if (time == lastTime) { return(Enumerable.Empty <BaseData>()); } lastTime = time; var tickTime = lastTime.AddMinutes(-1).ConvertFromUtc(TimeZones.NewYork); return(fdqh.Subscriptions.Where(symbol => !_algorithm.UniverseManager.ContainsKey(symbol)) // its not a universe .Select(symbol => new Tick(tickTime, symbol, 1, 2) { Quantity = 1, // Symbol could not be in the Securities collections for the custom Universe tests. AlgorithmManager is in charge of adding them, and we are not executing that code here. TickType = _algorithm.Securities.ContainsKey(symbol) ? _algorithm.Securities[symbol].SubscriptionDataConfig.TickType : TickType.Trade }).ToList()); }); // job is used to send into DataQueueHandler var job = new LiveNodePacket(); // result handler is used due to dependency in SubscriptionDataReader var resultHandler = new BacktestingResultHandler(); dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction); var feed = new TestableLiveTradingDataFeed(dataQueueHandler); var mapFileProvider = new LocalDiskMapFileProvider(); var fileProvider = new DefaultDataProvider(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var securityService = new SecurityService(_algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, _algorithm); _algorithm.Securities.SetSecurityService(securityService); _dataManager = new DataManager(feed, new UniverseSelection(_algorithm, securityService), _algorithm, _algorithm.TimeKeeper, marketHoursDatabase, true); _algorithm.SubscriptionManager.SetDataManager(_dataManager); _algorithm.AddSecurities(resolution, equities, forex, crypto); _synchronizer = new TestableLiveSynchronizer(_manualTimeProvider); _synchronizer.Initialize(_algorithm, _dataManager); feed.Initialize(_algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, _dataManager, _synchronizer); _algorithm.PostInitialize(); Thread.Sleep(150); // small handicap for the data to be pumped so TimeSlices have data of all subscriptions return(feed); }
private IDataFeed RunDataFeed(out FuncDataQueueHandler dataQueueHandler, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null, Resolution resolution = Resolution.Second, List <string> equities = null, List <string> forex = null) { _algorithm.SetStartDate(_startDate); var lastTime = _manualTimeProvider.GetUtcNow(); getNextTicksFunction = getNextTicksFunction ?? (fdqh => { var time = _manualTimeProvider.GetUtcNow(); if (time == lastTime) { return(Enumerable.Empty <BaseData>()); } lastTime = time; return(fdqh.Subscriptions.Where(symbol => !_algorithm.UniverseManager.ContainsKey(symbol)) // its not a universe .Select(symbol => new Tick(lastTime.ConvertFromUtc(TimeZones.NewYork), symbol, 1, 2) { Quantity = 1, // Symbol could not be in the Securities collections for the custom Universe tests. AlgorithmManager is in charge of adding them, and we are not executing that code here. TickType = _algorithm.Securities.ContainsKey(symbol) ? _algorithm.Securities[symbol].SubscriptionDataConfig.TickType : TickType.Trade })); }); // job is used to send into DataQueueHandler var job = new LiveNodePacket(); // result handler is used due to dependency in SubscriptionDataReader var resultHandler = new BacktestingResultHandler(); dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction); var feed = new TestableLiveTradingDataFeed(dataQueueHandler, _manualTimeProvider); var mapFileProvider = new LocalDiskMapFileProvider(); var fileProvider = new DefaultDataProvider(); var dataManager = new DataManager(feed, new UniverseSelection(feed, _algorithm), _algorithm.Settings, _algorithm.TimeKeeper); _algorithm.SubscriptionManager.SetDataManager(dataManager); _algorithm.AddSecurities(resolution, equities, forex); feed.Initialize(_algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager); _algorithm.PostInitialize(); Thread.Sleep(150); // small handicap for the data to be pumped so TimeSlices have data of all subscriptions var feedThreadStarted = new ManualResetEvent(false); Task.Factory.StartNew(() => { feedThreadStarted.Set(); feed.Run(); }); // wait for feed.Run to actually begin feedThreadStarted.WaitOne(); return(feed); }
public void RetrievesFromDisk() { var provider = new LocalDiskMapFileProvider(); provider.Initialize(TestGlobals.DataProvider); var mapFiles = provider.Get(AuxiliaryDataKey.EquityUsa); Assert.IsNotEmpty(mapFiles); }
public void InitializationSpeedTest() { var mapFileProvider = new LocalDiskMapFileProvider(); var sw = Stopwatch.StartNew(); var mapFileresolver = mapFileProvider.Get(Market.USA); sw.Stop(); Console.WriteLine($"elapsed: {sw.Elapsed.TotalSeconds} seconds"); }
public void TestsFileSystemDataFeedSpeed() { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var dataProvider = new DefaultDataProvider(); var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; var feed = new FileSystemDataFeed(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var dataPermissionManager = new DataPermissionManager(); var dataManager = new DataManager(feed, new UniverseSelection( algorithm, new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)), dataPermissionManager, new DefaultDataProvider()), algorithm, algorithm.TimeKeeper, marketHoursDatabase, false, RegisteredSecurityDataTypesProvider.Null, dataPermissionManager); algorithm.SubscriptionManager.SetDataManager(dataManager); var synchronizer = new Synchronizer(); synchronizer.Initialize(algorithm, dataManager); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager, synchronizer, dataPermissionManager.DataChannelProvider); algorithm.Initialize(); algorithm.PostInitialize(); var cancellationTokenSource = new CancellationTokenSource(); var count = 0; var stopwatch = Stopwatch.StartNew(); var lastMonth = algorithm.StartDate.Month; foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token)) { if (timeSlice.Time.Month != lastMonth) { var elapsed = stopwatch.Elapsed.TotalSeconds; var thousands = count / 1000d; Log.Trace($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands / elapsed}"); lastMonth = timeSlice.Time.Month; } count++; } Log.Trace("Count: " + count); stopwatch.Stop(); feed.Exit(); dataManager.RemoveAllSubscriptions(); Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}"); }
public void CachesValueAndReturnsSameReference() { var provider = new LocalDiskMapFileProvider(); provider.Initialize(TestGlobals.DataProvider); var mapFiles1 = provider.Get(AuxiliaryDataKey.EquityUsa); var mapFiles2 = provider.Get(AuxiliaryDataKey.EquityUsa); Assert.IsTrue(ReferenceEquals(mapFiles1, mapFiles2)); }
public void EmitsData() { var algorithm = new AlgorithmStub(forex: new List <string> { "EURUSD" }); // job is used to send into DataQueueHandler var job = new LiveNodePacket(); // result handler is used due to dependency in SubscriptionDataReader var resultHandler = new BacktestingResultHandler(); var dataProvider = new DefaultDataProvider(); var lastTime = DateTime.MinValue; var timeProvider = new RealTimeProvider(); var dataQueueHandler = new FuncDataQueueHandler(fdqh => { var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.EasternStandard); if (time == lastTime) { return(Enumerable.Empty <BaseData>()); } lastTime = time; return(Enumerable.Range(0, 9).Select(x => new Tick(time.AddMilliseconds(x * 100), Symbols.EURUSD, 1.3m, 1.2m, 1.3m))); }); var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider); var mapFileProvider = new LocalDiskMapFileProvider(); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), dataProvider); var feedThreadStarted = new ManualResetEvent(false); Task.Factory.StartNew(() => { feedThreadStarted.Set(); feed.Run(); }); // wait for feed.Run to actually begin feedThreadStarted.WaitOne(); var emittedData = false; ConsumeBridge(feed, TimeSpan.FromSeconds(10), true, ts => { if (ts.Slice.Count != 0) { emittedData = true; Console.WriteLine("HasData: " + ts.Slice.Bars[Symbols.EURUSD].EndTime); Console.WriteLine(); } }); Assert.IsTrue(emittedData); }
public void EmitsBasedOnData() { var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, true, false); var time = new DateTime(2021, 2, 17); var timeProvider = new ManualTimeProvider(time); Delisting delisting = null; var autoResetEvent = new AutoResetEvent(false); var dataQueueHandler = new FuncDataQueueHandler(handler => { autoResetEvent.Set(); if (delisting != null) { var result = new[] { delisting }; delisting = null; return(result); } return(Enumerable.Empty <BaseData>()); }, timeProvider); var provider = new LiveDataBasedDelistingEventProvider(config, dataQueueHandler); var mapFile = new LocalDiskMapFileProvider().Get(config.Symbol.ID.Market).ResolveMapFile(config.Symbol, config.Type); provider.Initialize(config, null, mapFile, time); Assert.IsTrue(autoResetEvent.WaitOne(TimeSpan.FromMilliseconds(100))); var events = provider.GetEvents(new NewTradableDateEventArgs(time, null, Symbols.AAPL, null)).ToList(); Assert.AreEqual(0, events.Count); delisting = new Delisting(Symbols.AAPL, time, 1, DelistingType.Warning); Thread.Sleep(100); events = provider.GetEvents(new NewTradableDateEventArgs(time.AddDays(-1), null, Symbols.AAPL, null)).ToList(); Assert.AreEqual(0, events.Count); events = provider.GetEvents(new NewTradableDateEventArgs(time, null, Symbols.AAPL, null)).ToList(); Assert.AreEqual(1, events.Count); Assert.AreEqual(time, (events[0] as Delisting).Time); Assert.AreEqual(DelistingType.Warning, (events[0] as Delisting).Type); events = provider.GetEvents(new NewTradableDateEventArgs(time, null, Symbols.AAPL, null)).ToList(); Assert.AreEqual(0, events.Count); events = provider.GetEvents(new NewTradableDateEventArgs(time.AddDays(1), null, Symbols.AAPL, null)).ToList(); Assert.AreEqual(1, events.Count); Assert.AreEqual(time.AddDays(1), (events[0] as Delisting).Time); Assert.AreEqual(DelistingType.Delisted, (events[0] as Delisting).Type); }
/// <summary> /// Runs the Coarse universe generator with default values. /// </summary> /// <returns></returns> public static bool CoarseUniverseGenerator() { var dailyDataFolder = new DirectoryInfo(Path.Combine(Globals.DataFolder, SecurityType.Equity.SecurityTypeToLower(), Market.USA, Resolution.Daily.ResolutionToLower())); var destinationFolder = new DirectoryInfo(Path.Combine(Globals.DataFolder, SecurityType.Equity.SecurityTypeToLower(), Market.USA, "fundamental", "coarse")); var blackListedTickersFile = new FileInfo("blacklisted-tickers.txt"); var reservedWordPrefix = Config.Get("reserved-words-prefix", "quantconnect-"); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var generator = new CoarseUniverseGeneratorProgram(dailyDataFolder, destinationFolder, Market.USA, blackListedTickersFile, reservedWordPrefix, mapFileProvider, factorFileProvider); return generator.Run(); }
private void LaunchLean() { ConfigMerger.Merge(_config, _id, this.GetType()); var systemHandlers = new LeanEngineSystemHandlers( new JobQueue(), new EmptyApiHandler(), new Messaging(), new LocalLeanManager()); systemHandlers.Initialize(); //separate log uniquely named var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff") + "_" + _id + ".txt"; using (Log.LogHandler = new FileLogHandler(logFileName, true)) { var map = new LocalDiskMapFileProvider(); var leanEngineAlgorithmHandlers = new LeanEngineAlgorithmHandlers( new OptimizerResultHandler(), new ConsoleSetupHandler(), new FileSystemDataFeed(), new BacktestingTransactionHandler(), new BacktestingRealTimeHandler(), map, new LocalDiskFactorFileProvider(map), new DefaultDataProvider(), new OptimizerAlphaHandler(), new EmptyObjectStore(), new DataPermissionManager()); _resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results; var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out var algorithmPath); //todo: pass period through job //job.PeriodStart = _config.StartDate; //job.PeriodFinish = _config.EndDate; try { var algorithmManager = new AlgorithmManager(false); systemHandlers.LeanManager.Initialize(systemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager); var engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, false); engine.Run(job, algorithmManager, algorithmPath, WorkerThread.Instance); } finally { // clean up resources systemHandlers.Dispose(); leanEngineAlgorithmHandlers.Dispose(); } } }
public void FastExitsDoNotThrowUnhandledExceptions() { var algorithm = new AlgorithmStub(); // job is used to send into DataQueueHandler var job = new LiveNodePacket(); // result handler is used due to dependency in SubscriptionDataReader var resultHandler = new BacktestingResultHandler(); var feed = new TestableLiveTradingDataFeed(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var securityService = new SecurityService( algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm); algorithm.Securities.SetSecurityService(securityService); var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm, securityService), algorithm.Settings, algorithm.TimeKeeper, marketHoursDatabase); algorithm.SubscriptionManager.SetDataManager(dataManager); algorithm.AddSecurities(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToString()).ToList()); var getNextTicksFunction = Enumerable.Range(0, 20).Select(x => new Tick { Symbol = SymbolCache.GetSymbol(x.ToString()) }).ToList(); feed.DataQueueHandler = new FuncDataQueueHandler(handler => getNextTicksFunction); var mapFileProvider = new LocalDiskMapFileProvider(); var fileProvider = new DefaultDataProvider(); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager); var unhandledExceptionWasThrown = false; try { feed.Exit(); } catch (Exception ex) { QuantConnect.Logging.Log.Error(ex.ToString()); unhandledExceptionWasThrown = true; } Thread.Sleep(500); Assert.IsFalse(unhandledExceptionWasThrown); }
public void ChecksFirstDate() { var mapFileProvider = new LocalDiskMapFileProvider(); var mapFileResolver = mapFileProvider.Get(Market.USA); // QQQ started trading on 19990310 var mapFile = mapFileResolver.ResolveMapFile("QQQ", new DateTime(1999, 3, 9)); Assert.IsTrue(mapFile.IsNullOrEmpty()); var mapFile2 = mapFileResolver.ResolveMapFile("QQQ", new DateTime(2015, 3, 10)); Assert.IsFalse(mapFile2.IsNullOrEmpty()); }
public void FastExitsDoNotThrowUnhandledExceptions() { var algorithm = new AlgorithmStub(); // job is used to send into DataQueueHandler var job = new LiveNodePacket(); // result handler is used due to dependency in SubscriptionDataReader var resultHandler = new BacktestingResultHandler(); var feed = new TestableLiveTradingDataFeed(); var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm), algorithm.Settings, algorithm.TimeKeeper); algorithm.SubscriptionManager.SetDataManager(dataManager); algorithm.AddSecurities(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToString()).ToList()); var getNextTicksFunction = Enumerable.Range(0, 20).Select(x => new Tick { Symbol = SymbolCache.GetSymbol(x.ToString()) }).ToList(); feed.DataQueueHandler = new FuncDataQueueHandler(handler => getNextTicksFunction); var mapFileProvider = new LocalDiskMapFileProvider(); var fileProvider = new DefaultDataProvider(); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager); var feedThreadStarted = new ManualResetEvent(false); var unhandledExceptionWasThrown = false; Task.Run(() => { try { feedThreadStarted.Set(); feed.Run(); } catch (Exception ex) { QuantConnect.Logging.Log.Error(ex.ToString()); unhandledExceptionWasThrown = true; } }); feedThreadStarted.WaitOne(); feed.Exit(); Thread.Sleep(1000); Assert.IsFalse(unhandledExceptionWasThrown); }
public void TestsFileSystemDataFeedSpeed() { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var dataProvider = new DefaultDataProvider(); var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; var feed = new FileSystemDataFeed(); var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm), algorithm.Settings, algorithm.TimeKeeper); algorithm.SubscriptionManager.SetDataManager(dataManager); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager); algorithm.Initialize(); algorithm.PostInitialize(); var feedThreadStarted = new ManualResetEvent(false); var dataFeedThread = new Thread(() => { feedThreadStarted.Set(); feed.Run(); }) { IsBackground = true }; dataFeedThread.Start(); feedThreadStarted.WaitOne(); var count = 0; var stopwatch = Stopwatch.StartNew(); var lastMonth = algorithm.StartDate.Month; foreach (var timeSlice in feed) { if (timeSlice.Time.Month != lastMonth) { var elapsed = stopwatch.Elapsed.TotalSeconds; var thousands = count / 1000d; Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}"); lastMonth = timeSlice.Time.Month; } count++; } Console.WriteLine("Count: " + count); stopwatch.Stop(); Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}"); }
public void TestDataFeedEnumeratorStackSpeed() { var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; algorithm.Initialize(); algorithm.PostInitialize(); var dataProvider = new DefaultDataProvider(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, true, enablePriceScaling: false); var universe = algorithm.UniverseManager.Single().Value; var security = algorithm.Securities.Single().Value; var securityConfig = security.Subscriptions.First(); var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, algorithm.StartDate, algorithm.EndDate); var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider); var count = 0; var stopwatch = Stopwatch.StartNew(); var lastMonth = algorithm.StartDate.Month; while (enumerator.MoveNext()) { var current = enumerator.Current; if (current == null) { Log.Trace("ERROR: Current is null"); continue; } if (current.Time.Month != lastMonth) { var elapsed = stopwatch.Elapsed.TotalSeconds; var thousands = count / 1000d; Log.Trace($"{DateTime.Now} - Time: {current.Time}: KPS: {thousands / elapsed}"); lastMonth = current.Time.Month; } count++; } Log.Trace("Count: " + count); stopwatch.Stop(); enumerator.Dispose(); factory.DisposeSafely(); Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}"); }
public void TestsFileSystemDataFeedSpeed() { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var dataProvider = new DefaultDataProvider(); var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; var feed = new FileSystemDataFeed(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm, new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm)), algorithm.Settings, algorithm.TimeKeeper, marketHoursDatabase); algorithm.SubscriptionManager.SetDataManager(dataManager); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager); algorithm.Initialize(); algorithm.PostInitialize(); var count = 0; var stopwatch = Stopwatch.StartNew(); var lastMonth = algorithm.StartDate.Month; foreach (var timeSlice in feed) { if (timeSlice.Time.Month != lastMonth) { var elapsed = stopwatch.Elapsed.TotalSeconds; var thousands = count / 1000d; Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}"); lastMonth = timeSlice.Time.Month; } count++; } Console.WriteLine("Count: " + count); stopwatch.Stop(); feed.Exit(); Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}"); }
public void DoesNotLeakMemory() { var symbol = Symbols.AAPL; var config = new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false); var security = new Security( SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), config, new Cash(Currencies.USD, 0, 1), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var mapFileResolver = mapFileProvider.Get(security.Symbol.ID.Market); var fileProvider = new DefaultDataProvider(); var factory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, factorFileProvider); GC.Collect(); var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed; var date = new DateTime(1998, 1, 1); const int iterations = 1000; for (var i = 0; i < iterations; i++) { var request = new SubscriptionRequest(false, null, security, config, date, date); using (var enumerator = factory.CreateEnumerator(request, fileProvider)) { enumerator.MoveNext(); } date = date.AddDays(1); } GC.Collect(); var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed; Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB"); Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10); }
public virtual void SetUp() { _algorithm = new QCAlgorithm(); _dataProvider = new DefaultDataProvider(); var mapFileProvider = new LocalDiskMapFileProvider(); mapFileProvider.Initialize(_dataProvider); var factorFileProvider = new LocalZipFactorFileProvider(); factorFileProvider.Initialize(mapFileProvider, _dataProvider); var historyProvider = new SubscriptionDataReaderHistoryProvider(); _cacheProvider = new ZipDataCacheProvider(_dataProvider); historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, _dataProvider, _cacheProvider, mapFileProvider, factorFileProvider, null, true, new DataPermissionManager())); _algorithm.SetHistoryProvider(historyProvider); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); _algorithm.Settings.FreePortfolioValue = 0; }
public void ResolvesCorrectlyReUsedTicker() { var mapFileProvider = new LocalDiskMapFileProvider(); var mapFileResolver = mapFileProvider.Get(Market.USA); // FB.1 started trading on 19990929 and ended on 20030328 var mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(1999, 9, 28)); Assert.IsTrue(mapFile.IsNullOrEmpty()); mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(1999, 9, 29)); Assert.IsFalse(mapFile.IsNullOrEmpty()); // FB started trading on 20120518 mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(2012, 5, 17)); Assert.IsTrue(mapFile.IsNullOrEmpty()); mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(2015, 5, 18)); Assert.IsFalse(mapFile.IsNullOrEmpty()); }
public void TestsFileSystemDataFeedSpeed() { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var dataProvider = new DefaultDataProvider(); var algorithm = new BenchmarkTest(); var feed = new FileSystemDataFeed(); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider); algorithm.Initialize(); var feedThreadStarted = new ManualResetEvent(false); Task.Factory.StartNew(() => { feedThreadStarted.Set(); feed.Run(); }); feedThreadStarted.WaitOne(); var stopwatch = Stopwatch.StartNew(); var lastMonth = -1; var count = 0; foreach (var timeSlice in feed) { if (timeSlice.Time.Month != lastMonth) { Console.WriteLine(DateTime.Now + " - Time: " + timeSlice.Time); lastMonth = timeSlice.Time.Month; } count++; } Console.WriteLine("Count: " + count); stopwatch.Stop(); Console.WriteLine("Elapsed time: " + stopwatch.Elapsed); }