/// <summary> /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// </summary> /// <filterpriority>2</filterpriority> public void Dispose() { // Log our numerical precision limited warnings if any if (!_numericalPrecisionLimitedWarnings.IsNullOrEmpty()) { var message = "Due to numerical precision issues in the factor file, data for the following" + $" symbols was adjust to a later starting date: {string.Join(", ", _numericalPrecisionLimitedWarnings.Values.Take(_numericalPrecisionLimitedWarningsMaxCount))}"; // If we reached our max warnings count suggest that more may have been left out if (_numericalPrecisionLimitedWarnings.Count >= _numericalPrecisionLimitedWarningsMaxCount) { message += "..."; } _resultHandler.DebugMessage(message); } // Log our start date adjustments because of map files if (!_startDateLimitedWarnings.IsNullOrEmpty()) { var message = "The starting dates for the following symbols have been adjusted to match their" + $" map files first date: {string.Join(", ", _startDateLimitedWarnings.Values.Take(_startDateLimitedWarningsMaxCount))}"; // If we reached our max warnings count suggest that more may have been left out if (_startDateLimitedWarnings.Count >= _startDateLimitedWarningsMaxCount) { message += "..."; } _resultHandler.DebugMessage(message); } _zipDataCacheProvider?.DisposeSafely(); }
public void TickResolutionOpenInterestHistoryRequestIsNotFilteredWhenRequestedExplicitly() { _algorithm = new QCAlgorithm(); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); _algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider(); var zipCacheProvider = new ZipDataCacheProvider(_dataProvider); _algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, _dataProvider, zipCacheProvider, _mapFileProvider, _factorFileProvider, null, false, new DataPermissionManager())); var start = new DateTime(2014, 6, 05); _algorithm.SetStartDate(start); _algorithm.SetDateTime(start.AddDays(2)); _algorithm.UniverseSettings.FillForward = false; var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17)); var openInterests = _algorithm.History <OpenInterest>(optionSymbol, start, start.AddDays(2), Resolution.Minute).ToList(); zipCacheProvider.DisposeSafely(); Assert.IsNotEmpty(openInterests); Assert.AreEqual(2, openInterests.Count); Assert.AreEqual(new DateTime(2014, 06, 05, 6, 32, 0), openInterests[0].Time); Assert.AreEqual(optionSymbol, openInterests[0].Symbol); Assert.AreEqual(new DateTime(2014, 06, 06, 6, 32, 0), openInterests[1].Time); Assert.AreEqual(optionSymbol, openInterests[1].Symbol); }
public void TickResolutionHistoryRequest() { _algorithm = new QCAlgorithm(); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); _algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider(); var zipCacheProvider = new ZipDataCacheProvider(_dataProvider); _algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, _dataProvider, zipCacheProvider, _mapFileProvider, _factorFileProvider, null, false, new DataPermissionManager())); _algorithm.SetStartDate(2013, 10, 08); var start = new DateTime(2013, 10, 07); // Trades and quotes var result = _algorithm.History(new [] { Symbols.SPY }, start.AddHours(9.8), start.AddHours(10), Resolution.Tick).ToList(); // Just Trades var result2 = _algorithm.History <Tick>(Symbols.SPY, start.AddHours(9.8), start.AddHours(10), Resolution.Tick).ToList(); zipCacheProvider.DisposeSafely(); Assert.IsNotEmpty(result); Assert.IsNotEmpty(result2); Assert.IsTrue(result2.All(tick => tick.TickType == TickType.Trade)); // (Trades and quotes).Count > Trades * 2 Assert.Greater(result.Count, result2.Count * 2); }
public void TickResolutionOpenInterestHistoryRequestIsFilteredByDefault_SingleSymbol() { _algorithm = new QCAlgorithm(); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); _algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider(); var dataProvider = new DefaultDataProvider(); var zipCacheProvider = new ZipDataCacheProvider(dataProvider); _algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, dataProvider, zipCacheProvider, new LocalDiskMapFileProvider(), new LocalDiskFactorFileProvider(), null, false, new DataPermissionManager())); var start = new DateTime(2014, 6, 05); _algorithm.SetStartDate(start); _algorithm.SetDateTime(start.AddDays(2)); var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17)); var result = _algorithm.History(new[] { optionSymbol }, start, start.AddDays(2), Resolution.Minute, fillForward: false).ToList(); zipCacheProvider.DisposeSafely(); Assert.IsNotEmpty(result); Assert.IsTrue(result.Any(slice => slice.ContainsKey(optionSymbol))); var openInterests = result.Select(slice => slice.Get(typeof(OpenInterest)) as DataDictionary <OpenInterest>).Where(dataDictionary => dataDictionary.Count > 0).ToList(); Assert.AreEqual(0, openInterests.Count); }
public void GetLastKnownPriceOfIlliquidAsset_RealData() { var algorithm = new QCAlgorithm(); algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm)); algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider(); var cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider()); algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, new DefaultDataProvider(), cacheProvider, new LocalDiskMapFileProvider(), new LocalDiskFactorFileProvider(), null, false, new DataPermissionManager())); algorithm.SetDateTime(new DateTime(2014, 6, 6, 15, 0, 0)); //20140606_twx_minute_quote_american_call_230000_20150117.csv var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17)); var option = algorithm.AddOptionContract(optionSymbol); var lastKnownPrice = algorithm.GetLastKnownPrice(option); Assert.IsNotNull(lastKnownPrice); // Data gap of more than 15 minutes Assert.Greater((algorithm.Time - lastKnownPrice.EndTime).TotalMinutes, 15); cacheProvider.DisposeSafely(); }
public void ZipDataCacheProviderEphemeralDataIsRespected(bool isDataEphemeral) { var cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), isDataEphemeral: isDataEphemeral); var remoteReader = new RemoteFileSubscriptionStreamReader( cacheProvider, @"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm", Globals.Cache, null); Assert.IsFalse(remoteReader.EndOfStream); Assert.AreEqual(1, TestDownloadProvider.DownloadCount); var remoteReader2 = new RemoteFileSubscriptionStreamReader( cacheProvider, @"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm", Globals.Cache, null); Assert.IsFalse(remoteReader.EndOfStream); Assert.AreEqual(isDataEphemeral ? 2 : 1, TestDownloadProvider.DownloadCount); remoteReader.Dispose(); remoteReader2.Dispose(); cacheProvider.DisposeSafely(); }
public void GetLastKnownPriceOption() { var cacheProvider = new ZipDataCacheProvider(_dataProvider); var algorithm = GetAlgorithm(cacheProvider, new DateTime(2014, 06, 09)); var option = algorithm.AddOptionContract(Symbols.CreateOptionSymbol("AAPL", OptionRight.Call, 250m, new DateTime(2016, 01, 15))); var lastKnownPrice = algorithm.GetLastKnownPrice(option); Assert.AreEqual(typeof(QuoteBar), lastKnownPrice.GetType()); cacheProvider.DisposeSafely(); }
public void OptionsAreMappedCorrectly() { var historyProvider = new SubscriptionDataReaderHistoryProvider(); var zipCache = new ZipDataCacheProvider(new DefaultDataProvider()); historyProvider.Initialize(new HistoryProviderInitializeParameters( null, null, TestGlobals.DataProvider, zipCache, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, null, false, new DataPermissionManager())); var symbol = Symbol.CreateOption( "FOXA", Market.USA, OptionStyle.American, OptionRight.Call, 32, new DateTime(2013, 07, 20)); var result = historyProvider.GetHistory( new[] { new HistoryRequest(new DateTime(2013, 06, 28), new DateTime(2013, 07, 03), typeof(QuoteBar), symbol, Resolution.Minute, SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), TimeZones.NewYork, null, false, false, DataNormalizationMode.Raw, TickType.Quote) }, TimeZones.NewYork).ToList(); Assert.IsNotEmpty(result); // assert we fetch the data for the previous and new symbol var firstBar = result.First().Values.Single(); var lastBar = result.Last().Values.Single(); Assert.IsTrue(firstBar.Symbol.Value.Contains("NWSA")); Assert.AreEqual(28, firstBar.Time.Date.Day); Assert.IsTrue(lastBar.Symbol.Value.Contains("FOXA")); Assert.AreEqual(2, lastBar.Time.Date.Day); zipCache.DisposeSafely(); }
public void GetLastKnownPriceOfCustomData() { var cacheProvider = new ZipDataCacheProvider(_dataProvider); var algorithm = GetAlgorithm(cacheProvider, new DateTime(2018, 4, 4)); var alpha = algorithm.AddData <AlphaStreamsPortfolioState>("9fc8ef73792331b11dbd5429a"); var lastKnownPrice = algorithm.GetLastKnownPrice(alpha); Assert.IsNotNull(lastKnownPrice); cacheProvider.DisposeSafely(); }
public void GetLastKnownPriceFuture() { var cacheProvider = new ZipDataCacheProvider(_dataProvider); var algorithm = GetAlgorithm(cacheProvider, new DateTime(2013, 10, 8)); var future = algorithm.AddSecurity(Symbols.CreateFutureSymbol(Futures.Indices.SP500EMini, new DateTime(2013, 12, 20))); var lastKnownPrice = algorithm.GetLastKnownPrice(future); Assert.AreEqual(typeof(QuoteBar), lastKnownPrice.GetType()); cacheProvider.DisposeSafely(); }
public void GetLastKnownPriceEquity() { var cacheProvider = new ZipDataCacheProvider(_dataProvider); var algorithm = GetAlgorithm(cacheProvider, new DateTime(2013, 10, 8)); var equity = algorithm.AddEquity("SPY"); var lastKnownPrice = algorithm.GetLastKnownPrice(equity); Assert.AreEqual(typeof(TradeBar), lastKnownPrice.GetType()); cacheProvider.DisposeSafely(); }
public void GetLastKnownPricesOption() { var cacheProvider = new ZipDataCacheProvider(_dataProvider); var algorithm = GetAlgorithm(cacheProvider, new DateTime(2014, 06, 09)); var option = algorithm.AddOptionContract(Symbols.CreateOptionSymbol("AAPL", OptionRight.Call, 250m, new DateTime(2016, 01, 15))); var lastKnownPrices = algorithm.GetLastKnownPrices(option).ToList();; Assert.AreEqual(2, lastKnownPrices.Count); Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(TradeBar))); Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(QuoteBar))); cacheProvider.DisposeSafely(); }
public void GetLastKnownPricesEquity() { var cacheProvider = new ZipDataCacheProvider(_dataProvider); var algorithm = GetAlgorithm(cacheProvider, new DateTime(2013, 10, 8)); var equity = algorithm.AddEquity("SPY"); var lastKnownPrices = algorithm.GetLastKnownPrices(equity.Symbol).ToList(); Assert.AreEqual(2, lastKnownPrices.Count); Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(TradeBar))); Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(QuoteBar))); cacheProvider.DisposeSafely(); }
public void WarmUpPythonIndicatorProperly() { var algo = new AlgorithmStub { HistoryProvider = new SubscriptionDataReaderHistoryProvider() }; var zipCacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider); algo.HistoryProvider.Initialize(new HistoryProviderInitializeParameters( null, null, TestGlobals.DataProvider, zipCacheProvider, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, null, false, new DataPermissionManager())); algo.SetStartDate(2013, 10, 08); algo.AddEquity("SPY", Resolution.Minute); // Different types of indicators var indicatorDataPoint = new SimpleMovingAverage("SPY", 10); var indicatorDataBar = new AverageTrueRange("SPY", 10); var indicatorTradeBar = new VolumeWeightedAveragePriceIndicator("SPY", 10); using (Py.GIL()) { var sma = indicatorDataPoint.ToPython(); var atr = indicatorTradeBar.ToPython(); var vwapi = indicatorDataBar.ToPython(); Assert.DoesNotThrow(() => algo.WarmUpIndicator("SPY", sma, Resolution.Minute)); Assert.DoesNotThrow(() => algo.WarmUpIndicator("SPY", atr, Resolution.Minute)); Assert.DoesNotThrow(() => algo.WarmUpIndicator("SPY", vwapi, Resolution.Minute)); var smaIsReady = ((dynamic)sma).IsReady; var atrIsReady = ((dynamic)atr).IsReady; var vwapiIsReady = ((dynamic)vwapi).IsReady; Assert.IsTrue(smaIsReady.IsTrue()); Assert.IsTrue(atrIsReady.IsTrue()); Assert.IsTrue(vwapiIsReady.IsTrue()); } zipCacheProvider.DisposeSafely(); }
public void GetLastKnownPriceOfIlliquidAsset_RealData() { var cacheProvider = new ZipDataCacheProvider(_dataProvider); var algorithm = GetAlgorithm(cacheProvider, new DateTime(2014, 6, 6, 11, 0, 0)); //20140606_twx_minute_quote_american_call_230000_20150117.csv var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17)); var option = algorithm.AddOptionContract(optionSymbol); var lastKnownPrice = algorithm.GetLastKnownPrice(option); Assert.IsNotNull(lastKnownPrice); // Data gap of more than 15 minutes Assert.Greater((algorithm.Time - lastKnownPrice.EndTime).TotalMinutes, 15); cacheProvider.DisposeSafely(); }
public void CurrencyConversionRateResolved() { // Unit test to prove that in the event that default resolution (minute) history request returns // no data for our currency conversion that BaseSetupHandler will use a daily history request // to determine the the conversion rate if possible. // Setup history provider and algorithm var historyProvider = new SubscriptionDataReaderHistoryProvider(); var zipCache = new ZipDataCacheProvider(new DefaultDataProvider()); historyProvider.Initialize(new HistoryProviderInitializeParameters( null, null, TestGlobals.DataProvider, zipCache, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, null, false, new DataPermissionManager())); var algorithm = new BrokerageSetupHandlerTests.TestAlgorithm { UniverseSettings = { Resolution = Resolution.Minute } }; algorithm.SetHistoryProvider(historyProvider); // Pick a date range where we do NOT have BTCUSD minute data algorithm.SetStartDate(2015, 1, 24); algorithm.SetCash("USD", 0); algorithm.SetCash("BTC", 10); // Have BaseSetupHandler resolve the currency conversion BaseSetupHandler.SetupCurrencyConversions(algorithm, algorithm.DataManager.UniverseSelection); // Assert that our portfolio has some value and that value is bitcoin Assert.IsTrue(algorithm.Portfolio.Cash > 0); Assert.IsTrue(algorithm.Portfolio.CashBook["BTC"].ValueInAccountCurrency > 0); zipCache.DisposeSafely(); }
public void EquitiesAreMappedCorrectly() { var historyProvider = new SubscriptionDataReaderHistoryProvider(); var zipCache = new ZipDataCacheProvider(new DefaultDataProvider()); historyProvider.Initialize(new HistoryProviderInitializeParameters( null, null, new DefaultDataProvider(), zipCache, new LocalDiskMapFileProvider(), new LocalDiskFactorFileProvider(), null, false, new DataPermissionManager())); var symbol = Symbol.Create("WM", SecurityType.Equity, Market.USA); var result = historyProvider.GetHistory( new[] { new HistoryRequest(new DateTime(2008, 01, 01), new DateTime(2008, 01, 05), typeof(TradeBar), symbol, Resolution.Daily, SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), TimeZones.NewYork, null, false, false, DataNormalizationMode.Raw, TickType.Trade) }, TimeZones.NewYork).ToList(); var firstBar = result.First().Values.Single(); Assert.AreEqual("WMI", firstBar.Symbol.Value); Assert.IsNotEmpty(result); zipCache.DisposeSafely(); }
public void Performance(Type streamReaderType, Type readLineReaderType, TickType tickType) { var streamReaderMilliSeconds = 0L; var streamReaderCount = 0; var getLineReaderMilliSeconds = 0L; var getLineReaderCount = 0; var stopWatch = new Stopwatch(); { var config = new SubscriptionDataConfig( streamReaderType, Symbols.SPY, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork, false, true, false, tickType: tickType ); var zipCache = new ZipDataCacheProvider(new DefaultDataProvider()); var date = new DateTime(2013, 10, 07); var reader = new TextSubscriptionDataSourceReader( zipCache, config, date, false); var source = streamReaderType.GetBaseDataInstance().GetSource(config, date, false); // warmup streamReaderCount = reader.Read(source).Count(); streamReaderCount = 0; // start test stopWatch.Start(); for (int i = 0; i < 200; i++) { streamReaderCount += reader.Read(source).Count(); } stopWatch.Stop(); streamReaderMilliSeconds = stopWatch.ElapsedMilliseconds; zipCache.DisposeSafely(); } { var config = new SubscriptionDataConfig( readLineReaderType, Symbols.SPY, Resolution.Minute, TimeZones.NewYork, TimeZones.NewYork, false, true, false, tickType: tickType ); var zipCache = new ZipDataCacheProvider(new DefaultDataProvider()); var date = new DateTime(2013, 10, 07); var reader = new TextSubscriptionDataSourceReader( zipCache, config, date, false); var source = readLineReaderType.GetBaseDataInstance().GetSource(config, date, false); // warmup getLineReaderCount = reader.Read(source).Count(); getLineReaderCount = 0; // start test stopWatch.Start(); for (int i = 0; i < 200; i++) { getLineReaderCount += reader.Read(source).Count(); } stopWatch.Stop(); getLineReaderMilliSeconds = stopWatch.ElapsedMilliseconds; zipCache.DisposeSafely(); } Console.WriteLine($"StreamReader: {streamReaderMilliSeconds}ms. Count {streamReaderCount}"); Console.WriteLine($"GetLine Reader: {getLineReaderMilliSeconds}ms. Count {getLineReaderCount}"); // its 50% faster but lets leave some room to avoid noise Assert.IsTrue((streamReaderMilliSeconds * 1.85d) < getLineReaderMilliSeconds); Assert.AreEqual(getLineReaderCount, streamReaderCount); }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="manager">The algorithm manager instance</param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> /// <param name="workerThread">The worker thread instance</param> public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath, WorkerThread workerThread) { var marketHoursDatabaseTask = Task.Run(() => StaticInitializations()); var algorithm = default(IAlgorithm); var algorithmManager = manager; try { //Reset thread holders. var initializeComplete = false; //-> Initialize messaging system SystemHandlers.Notify.SetAuthentication(job); //-> Set the result handler type for this algorithm job, and launch the associated result thread. AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions); IBrokerage brokerage = null; DataManager dataManager = null; IDataCacheProvider historyDataCacheProvider = null; var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer(); try { // we get the mhdb before creating the algorithm instance, // since the algorithm constructor will use it var marketHoursDatabase = marketHoursDatabaseTask.Result; AlgorithmHandlers.Setup.WorkerThread = workerThread; // Save algorithm to cache, load algorithm instance: algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath); // Set algorithm in ILeanManager SystemHandlers.LeanManager.SetAlgorithm(algorithm); // initialize the alphas handler with the algorithm instance AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions); // initialize the object store AlgorithmHandlers.ObjectStore.Initialize(algorithm.Name, job.UserId, job.ProjectId, job.UserToken, job.Controls); // initialize the data permission manager AlgorithmHandlers.DataPermissionsManager.Initialize(job); // notify the user of any errors w/ object store persistence AlgorithmHandlers.ObjectStore.ErrorRaised += (sender, args) => algorithm.Debug($"ObjectStore Persistence Error: {args.Error.Message}"); // Initialize the brokerage IBrokerageFactory factory; brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory); var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); var registeredTypesProvider = new RegisteredSecurityDataTypesProvider(); var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDatabase, algorithm, registeredTypesProvider, new SecurityCacheProvider(algorithm.Portfolio)); algorithm.Securities.SetSecurityService(securityService); dataManager = new DataManager(AlgorithmHandlers.DataFeed, new UniverseSelection( algorithm, securityService, AlgorithmHandlers.DataPermissionsManager), algorithm, algorithm.TimeKeeper, marketHoursDatabase, _liveMode, registeredTypesProvider, AlgorithmHandlers.DataPermissionsManager); AlgorithmHandlers.Results.SetDataManager(dataManager); algorithm.SubscriptionManager.SetDataManager(dataManager); synchronizer.Initialize(algorithm, dataManager); // Initialize the data feed before we initialize so he can intercept added securities/universes via events AlgorithmHandlers.DataFeed.Initialize( algorithm, job, AlgorithmHandlers.Results, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, AlgorithmHandlers.DataProvider, dataManager, (IDataFeedTimeProvider)synchronizer, AlgorithmHandlers.DataPermissionsManager.DataChannelProvider); // set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider) algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions); // set the history provider before setting up the algorithm var historyProvider = GetHistoryProvider(job.HistoryProvider); if (historyProvider is BrokerageHistoryProvider) { (historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage); } historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode); historyProvider.Initialize( new HistoryProviderInitializeParameters( job, SystemHandlers.Api, AlgorithmHandlers.DataProvider, historyDataCacheProvider, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History, Invariant($"Processing history {progress}%...")); } }, // disable parallel history requests for live trading parallelHistoryRequestsEnabled: !_liveMode, dataPermissionManager: AlgorithmHandlers.DataPermissionsManager ) ); historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); }; historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); }; historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); }; historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); }; algorithm.HistoryProvider = historyProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore)); // set this again now that we've actually added securities AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue); // alpha handler needs start/end dates to determine sample step sizes AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = string.Join(",", algorithm.ErrorMessages); errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e => { var message = e.Message; if (e.InnerException != null) { var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value); message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err); } return(message); })); Log.Error("Engine.Run(): " + errorMessage); AlgorithmHandlers.Results.RuntimeError(errorMessage); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { Log.Error(err); var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err; AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString()); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } // log the job endpoints Log.Trace("JOB HANDLERS: "); Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName); Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName); Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName); Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName); Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName); Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName); Log.Trace(" ObjectStore: " + AlgorithmHandlers.ObjectStore.GetType().FullName); if (algorithm?.HistoryProvider != null) { Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName); } if (job is LiveNodePacket) { Log.Trace(" Brokerage: " + brokerage?.GetType().FullName); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { // notify the LEAN manager that the algorithm is initialized and starting SystemHandlers.LeanManager.OnAlgorithmStart(); //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.UtcNow; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Load the associated handlers for transaction and realtime events: AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results); AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api, algorithmManager.TimeLimit); // wire up the brokerage message handler brokerage.Message += (sender, message) => { algorithm.BrokerageMessageHandler.Handle(message); // fire brokerage message events algorithm.OnBrokerageMessage(message); switch (message.Type) { case BrokerageMessageType.Disconnect: algorithm.OnBrokerageDisconnect(); break; case BrokerageMessageType.Reconnect: algorithm.OnBrokerageReconnect(); break; } }; //Send status to user the algorithm is now executing. AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running); // Result manager scanning message queue: (started earlier) AlgorithmHandlers.Results.DebugMessage( $"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}"); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLimit.IsWithinLimit, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.Controls.RamAllocation, workerThread: workerThread); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F")); throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { HandleAlgorithmError(job, algorithm.RunTimeError); } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. algorithm.RunTimeError = err; algorithm.SetStatus(AlgorithmStatus.RuntimeError); HandleAlgorithmError(job, err); } // notify the LEAN manager that the algorithm has finished SystemHandlers.LeanManager.OnAlgorithmEnd(); try { var csvTransactionsFileName = Config.Get("transaction-log"); if (!string.IsNullOrEmpty(csvTransactionsFileName)) { SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName); } if (!_liveMode) { //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.UtcNow - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount; var kps = dataPoints / (double)1000 / totalSeconds; AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points."); } } catch (Exception err) { Log.Error(err, "Error sending analysis results"); } //Before we return, send terminate commands to close up the threads AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.RealTime.Exit(); dataManager?.RemoveAllSubscriptions(); workerThread?.Dispose(); } // Close data feed, alphas. Could be running even if algorithm initialization failed AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.Alphas.Exit(); //Close result handler: AlgorithmHandlers.Results.Exit(); //Wait for the threads to complete: var millisecondInterval = 10; var millisecondTotalWait = 0; while ((AlgorithmHandlers.Results.IsActive || (AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) || (AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) || (AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) || (AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) && millisecondTotalWait < 30 * 1000) { Thread.Sleep(millisecondInterval); if (millisecondTotalWait % (millisecondInterval * 10) == 0) { Log.Trace("Waiting for threads to exit..."); } millisecondTotalWait += millisecondInterval; } if (brokerage != null) { Log.Trace("Engine.Run(): Disconnecting from brokerage..."); brokerage.Disconnect(); brokerage.Dispose(); } if (AlgorithmHandlers.Setup != null) { Log.Trace("Engine.Run(): Disposing of setup handler..."); AlgorithmHandlers.Setup.Dispose(); } historyDataCacheProvider.DisposeSafely(); Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error(err, "Error running algorithm"); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) { SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); } AlgorithmHandlers.Results.Exit(); AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// </summary> /// <filterpriority>2</filterpriority> public void Dispose() { _zipDataCacheProvider?.DisposeSafely(); }
public virtual void TearDown() { _cacheProvider.DisposeSafely(); _dataProvider.DisposeSafely(); }