public void TestsFileSystemDataFeedSpeed() { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var dataProvider = new DefaultDataProvider(); var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; var feed = new FileSystemDataFeed(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var dataManager = new DataManager(feed, new UniverseSelection( algorithm, new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm)), algorithm, algorithm.TimeKeeper, marketHoursDatabase); algorithm.SubscriptionManager.SetDataManager(dataManager); var synchronizer = new Synchronizer(); synchronizer.Initialize(algorithm, dataManager, false, algorithm.Portfolio.CashBook); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager, synchronizer); algorithm.Initialize(); algorithm.PostInitialize(); var cancellationTokenSource = new CancellationTokenSource(); var count = 0; var stopwatch = Stopwatch.StartNew(); var lastMonth = algorithm.StartDate.Month; foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token)) { if (timeSlice.Time.Month != lastMonth) { var elapsed = stopwatch.Elapsed.TotalSeconds; var thousands = count / 1000d; Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}"); lastMonth = timeSlice.Time.Month; } count++; } Console.WriteLine("Count: " + count); stopwatch.Stop(); feed.Exit(); Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}"); }
/// <summary> /// Sets the initial cash for the algorithm if set in the job packet. /// </summary> /// <remarks>Should be called after initialize <see cref="LoadBacktestJobAccountCurrency"/></remarks> public static void LoadBacktestJobCashAmount(IAlgorithm algorithm, BacktestNodePacket job) { // set initial cash, if present in the job if (job.CashAmount.HasValue) { // Zero the CashBook - we'll populate directly from job foreach (var kvp in algorithm.Portfolio.CashBook) { kvp.Value.SetAmount(0); } algorithm.SetCash(job.CashAmount.Value.Amount); } }
/// <summary> /// Initialize the result handler with this result packet. /// </summary> /// <param name="job">Algorithm job packet for this result handler</param> /// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param> /// <param name="api">The api instance used for handling logs</param> /// <param name="dataFeed"></param> /// <param name="setupHandler"></param> /// <param name="transactionHandler"></param> public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler) { _api = api; _messagingHandler = messagingHandler; _transactionHandler = transactionHandler; _setupHandler = setupHandler; _job = (BacktestNodePacket)job; if (_job == null) { throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid."); } _compileId = _job.CompileId; _backtestId = _job.BacktestId; }
/// <summary> /// Initialize the result handler with this result packet. /// </summary> /// <param name="job">Algorithm job packet for this result handler</param> /// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param> /// <param name="api">The api instance used for handling logs</param> /// <param name="transactionHandler">The transaction handler used to get the algorithms <see cref="Order"/> information</param> public virtual void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, ITransactionHandler transactionHandler) { _algorithmId = job.AlgorithmId; _projectId = job.ProjectId; MessagingHandler = messagingHandler; TransactionHandler = transactionHandler; _job = (BacktestNodePacket)job; if (_job == null) { throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid."); } JobId = _job.BacktestId; CompileId = _job.CompileId; }
/// <summary> /// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution. /// </summary> /// <returns></returns> public AlgorithmNodePacket NextJob(out string location) { location = AlgorithmLocation; //If this isn't a backtesting mode/request, attempt a live job. if (_liveMode) { var liveJob = new LiveNodePacket { Type = PacketType.LiveNode, DataEndpoint = DataFeedEndpoint.LiveTrading, RealTimeEndpoint = RealTimeEndpoint.LiveTrading, ResultEndpoint = ResultHandlerEndpoint.Console, SetupEndpoint = SetupHandlerEndpoint.Brokerage, TransactionEndpoint = TransactionHandlerEndpoint.Brokerage, Algorithm = File.ReadAllBytes(AlgorithmLocation), Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName), Channel = Config.Get("job-channel"), UserId = Config.GetInt("job-user-id") }; try { // import the brokerage data for the configured brokerage var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); liveJob.BrokerageData = brokerageFactory.BrokerageData; } catch (Exception err) { Log.Error(string.Format("JobQueue.NextJob(): Error resoliving BrokerageData for live job for brokerage {0}. {1}", liveJob.Brokerage, err.Message)); } return(liveJob); } //Default run a backtesting job. var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local") { Type = PacketType.BacktestNode, DataEndpoint = DataFeedEndpoint.FileSystem, SetupEndpoint = SetupHandlerEndpoint.Console, ResultEndpoint = ResultHandlerEndpoint.Console, RealTimeEndpoint = RealTimeEndpoint.Backtesting, TransactionEndpoint = TransactionHandlerEndpoint.Backtesting, Algorithm = File.ReadAllBytes(AlgorithmLocation) }; return(backtestJob); }
public void TestsFileSystemDataFeedSpeed() { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var dataProvider = new DefaultDataProvider(); var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; var feed = new FileSystemDataFeed(); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider); algorithm.Initialize(); algorithm.PostInitialize(); var feedThreadStarted = new ManualResetEvent(false); var dataFeedThread = new Thread(() => { feedThreadStarted.Set(); feed.Run(); }) { IsBackground = true }; dataFeedThread.Start(); feedThreadStarted.WaitOne(); var count = 0; var stopwatch = Stopwatch.StartNew(); var lastMonth = algorithm.StartDate.Month; foreach (var timeSlice in feed) { if (timeSlice.Time.Month != lastMonth) { var elapsed = stopwatch.Elapsed.TotalSeconds; var thousands = count / 1000d; Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}"); lastMonth = timeSlice.Time.Month; } count++; } Console.WriteLine("Count: " + count); stopwatch.Stop(); Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}"); }
/// <summary> /// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution. /// </summary> /// <returns></returns> public AlgorithmNodePacket NextJob(out string location) { location = AlgorithmLocation; Log.Trace("JobQueue.NextJob(): Selected " + location); //If this isn't a backtesting mode/request, attempt a live job. if (_liveMode) { var liveJob = new LiveNodePacket { Type = PacketType.LiveNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName), Channel = Config.Get("job-channel"), UserId = Config.GetInt("job-user-id"), Version = Constants.Version, DeployId = Config.Get("algorithm-type-name"), RamAllocation = int.MaxValue }; try { // import the brokerage data for the configured brokerage var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); liveJob.BrokerageData = brokerageFactory.BrokerageData; } catch (Exception err) { Log.Error(string.Format("JobQueue.NextJob(): Error resoliving BrokerageData for live job for brokerage {0}. {1}", liveJob.Brokerage, err.Message)); } return(liveJob); } //Default run a backtesting job. var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local") { Type = PacketType.BacktestNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), Version = Constants.Version, BacktestId = Config.Get("algorithm-type-name"), RamAllocation = int.MaxValue, Language = (Language)Enum.Parse(typeof(Language), Config.Get("algorithm-language")) }; return(backtestJob); }
public void TestAlgorithmManagerSpeed() { var algorithmManager = new AlgorithmManager(false); var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"{nameof(AlgorithmManagerTests)}.{nameof(TestAlgorithmManagerSpeed)}"); var feed = new MockDataFeed(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var dataManager = new DataManager(feed, new UniverseSelection( algorithm, new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null)), algorithm, algorithm.TimeKeeper, marketHoursDatabase, false, RegisteredSecurityDataTypesProvider.Null); algorithm.SubscriptionManager.SetDataManager(dataManager); var transactions = new BacktestingTransactionHandler(); var results = new BacktestingResultHandler(); var realtime = new BacktestingRealTimeHandler(); var leanManager = new NullLeanManager(); var alphas = new NullAlphaHandler(); var token = new CancellationToken(); var nullSynchronizer = new NullSynchronizer(algorithm); algorithm.Initialize(); algorithm.PostInitialize(); results.Initialize(job, new QuantConnect.Messaging.Messaging(), new Api.Api(), transactions); results.SetAlgorithm(algorithm, algorithm.Portfolio.TotalPortfolioValue); transactions.Initialize(algorithm, new BacktestingBrokerage(algorithm), results); feed.Initialize(algorithm, job, results, null, null, null, dataManager, null); Log.Trace("Starting algorithm manager loop to process " + nullSynchronizer.Count + " time slices"); var sw = Stopwatch.StartNew(); algorithmManager.Run(job, algorithm, nullSynchronizer, transactions, results, realtime, leanManager, alphas, token); sw.Stop(); var thousands = nullSynchronizer.Count / 1000d; var seconds = sw.Elapsed.TotalSeconds; Log.Trace("COUNT: " + nullSynchronizer.Count + " KPS: " + thousands / seconds); }
public void RoundTripNullJobDates() { var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"{nameof(BacktestNodePacketTests)}.Pepe"); var serialized = JsonConvert.SerializeObject(job); var job2 = JsonConvert.DeserializeObject <BacktestNodePacket>(serialized); Assert.AreEqual(job.BacktestId, job2.BacktestId); Assert.AreEqual(job.Name, job2.Name); Assert.IsNull(job.PeriodFinish); Assert.IsNull(job.PeriodStart); Assert.AreEqual(job.PeriodFinish, job2.PeriodFinish); Assert.AreEqual(job.PeriodStart, job2.PeriodStart); Assert.AreEqual(job.ProjectId, job2.ProjectId); Assert.AreEqual(job.SessionId, job2.SessionId); Assert.AreEqual(job.Language, job2.Language); }
/// <summary> /// Facade for the parent's NextJob() function. /// We're exposing this in order to change the BacktestId. /// </summary> /// <param name="location"></param> /// <returns></returns> public AlgorithmNodePacket NextJob(out string location) { location = AlgorithmLocation; Log.Trace("JobQueue.NextJob(): Selected " + location); // check for parameters in the config var parameters = new Dictionary <string, string>(); var parametersConfigString = Config.Get("parameters"); if (parametersConfigString != string.Empty) { parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString); } var controls = new Controls() { MinuteLimit = Config.GetInt("symbol-minute-limit", 10000), SecondLimit = Config.GetInt("symbol-second-limit", 10000), TickLimit = Config.GetInt("symbol-tick-limit", 10000), RamAllocation = int.MaxValue, MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000) }; // We currently offer no live job functionality //Default run a backtesting job. var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] { }, 10000, "local") { Type = PacketType.BacktestNode, Algorithm = File.ReadAllBytes(location), HistoryProvider = Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"), Channel = AccessToken, UserId = UserId, ProjectId = ProjectId, Version = Globals.Version, BacktestId = AlgorithmTypeName + "-" + Guid.NewGuid(), Language = Language.CSharp, Parameters = parameters, Controls = controls }; return(backtestJob); }
public void MessageHandler_WillSend_NewBackTestJob_ToCorrectRoute() { var backtest = new BacktestNodePacket(); using (var pullSocket = new PullSocket(">tcp://localhost:" + _port)) { _messageHandler.SetAuthentication(backtest); var message = pullSocket.ReceiveMultipartMessage(); var payload = message[0].ConvertToString(); var packet = JsonConvert.DeserializeObject <Packet>(payload); Assert.IsTrue(message.FrameCount == 1); Assert.IsTrue(PacketType.BacktestNode == packet.Type); Assert.IsTrue(payload == JsonConvert.SerializeObject(backtest)); } }
/******************************************************** * CLASS CONSTRUCTOR *********************************************************/ /// <summary> /// Create a new backtesting data feed. /// </summary> /// <param name="algorithm">Instance of the algorithm</param> /// <param name="job">Algorithm work task</param> public FileSystemDataFeed(IAlgorithm algorithm, BacktestNodePacket job) { Subscriptions = algorithm.SubscriptionManager.Subscriptions; _subscriptions = Subscriptions.Count; //Public Properties: DataFeed = DataFeedEndpoint.FileSystem; IsActive = true; Bridge = new ConcurrentQueue <List <BaseData> > [_subscriptions]; EndOfBridge = new bool[_subscriptions]; SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions]; FillForwardFrontiers = new DateTime[_subscriptions]; //Class Privates: _job = job; _algorithm = algorithm; _endOfStreams = false; _bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count: }
/// <summary> /// Initialize the result handler with this result packet. /// </summary> /// <param name="job">Algorithm job packet for this result handler</param> /// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param> /// <param name="api">The api instance used for handling logs</param> /// <param name="dataFeed"></param> /// <param name="setupHandler"></param> /// <param name="transactionHandler"></param> public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler) { _api = api; _messagingHandler = messagingHandler; _transactionHandler = transactionHandler; _job = (BacktestNodePacket)job; if (_job == null) { throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid."); } _compileId = _job.CompileId; _backtestId = _job.BacktestId; //Get the resample period: var totalMinutes = (_job.PeriodFinish - _job.PeriodStart).TotalMinutes; var resampleMinutes = (totalMinutes < (_minimumSamplePeriod * _samples)) ? _minimumSamplePeriod : (totalMinutes / _samples); // Space out the sampling every _resamplePeriod = TimeSpan.FromMinutes(resampleMinutes); Log.Trace("BacktestingResultHandler(): Sample Period Set: " + resampleMinutes.ToString("00.00")); }
/******************************************************** * PUBLIC CONSTRUCTOR *********************************************************/ /// <summary> /// Console result handler constructor. /// </summary> /// <remarks>Setup the default sampling and notification periods based on the backtest length.</remarks> public ConsoleResultHandler(BacktestNodePacket packet) { Log.Trace("Launching Console Result Handler: QuantConnect v2.0"); Messages = new ConcurrentQueue <Packet>(); Charts = new ConcurrentDictionary <string, Chart>(); _chartLock = new Object(); _isActive = true; _job = packet; //Get the resample period: const double samples = 4000; const double minimumSamplePeriod = 4 * 60; var totalMinutes = (_job.PeriodFinish - _job.PeriodStart).TotalMinutes; var resampleMinutes = (totalMinutes < (minimumSamplePeriod * samples)) ? minimumSamplePeriod : (totalMinutes / samples); // Space out the sampling every _resamplePeriod = TimeSpan.FromMinutes(resampleMinutes); //Notification Period for pushes: _notificationPeriod = TimeSpan.FromSeconds(5); }
public void TestsFileSystemDataFeedSpeed() { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var dataProvider = new DefaultDataProvider(); var algorithm = new BenchmarkTest(); var feed = new FileSystemDataFeed(); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider); algorithm.Initialize(); var feedThreadStarted = new ManualResetEvent(false); Task.Factory.StartNew(() => { feedThreadStarted.Set(); feed.Run(); }); feedThreadStarted.WaitOne(); var stopwatch = Stopwatch.StartNew(); var lastMonth = -1; var count = 0; foreach (var timeSlice in feed) { if (timeSlice.Time.Month != lastMonth) { Console.WriteLine(DateTime.Now + " - Time: " + timeSlice.Time); lastMonth = timeSlice.Time.Month; } count++; } Console.WriteLine("Count: " + count); stopwatch.Stop(); Console.WriteLine("Elapsed time: " + stopwatch.Elapsed); }
public void TestsFileSystemDataFeedSpeed() { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var dataProvider = new DefaultDataProvider(); var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; var feed = new FileSystemDataFeed(); var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm), algorithm.Settings, algorithm.TimeKeeper); algorithm.SubscriptionManager.SetDataManager(dataManager); feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager); algorithm.Initialize(); algorithm.PostInitialize(); var count = 0; var stopwatch = Stopwatch.StartNew(); var lastMonth = algorithm.StartDate.Month; foreach (var timeSlice in feed) { if (timeSlice.Time.Month != lastMonth) { var elapsed = stopwatch.Elapsed.TotalSeconds; var thousands = count / 1000d; Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}"); lastMonth = timeSlice.Time.Month; } count++; } Console.WriteLine("Count: " + count); stopwatch.Stop(); feed.Exit(); Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}"); }
/// <summary> /// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution. /// </summary> /// <returns></returns> public AlgorithmNodePacket NextJob(out string location) { location = GetAlgorithmLocation(); Log.Trace($"JobQueue.NextJob(): Selected {location}"); // check for parameters in the config var parameters = new Dictionary <string, string>(); var parametersConfigString = Config.Get("parameters"); if (parametersConfigString != string.Empty) { parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString); } var controls = new Controls() { MinuteLimit = Config.GetInt("symbol-minute-limit", 10000), SecondLimit = Config.GetInt("symbol-second-limit", 10000), TickLimit = Config.GetInt("symbol-tick-limit", 10000), RamAllocation = int.MaxValue, MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000) }; if ((Language)Enum.Parse(typeof(Language), Config.Get("algorithm-language")) == Language.Python) { // Set the python path for loading python algorithms. var pythonFile = new FileInfo(location); var pythonPath = new List <string> { pythonFile.Directory.FullName, new DirectoryInfo(Environment.CurrentDirectory).FullName, }; // Don't include an empty environment variable in pythonPath, otherwise the PYTHONPATH // environment variable won't be used in the module import process var pythonPathEnvironmentVariable = Environment.GetEnvironmentVariable("PYTHONPATH"); if (!string.IsNullOrEmpty(pythonPathEnvironmentVariable)) { pythonPath.Add(pythonPathEnvironmentVariable); } Environment.SetEnvironmentVariable("PYTHONPATH", string.Join(OS.IsLinux ? ":" : ";", pythonPath)); } //If this isn't a backtesting mode/request, attempt a live job. if (_liveMode) { var liveJob = new LiveNodePacket { Type = PacketType.LiveNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), DataQueueHandler = Config.Get("data-queue-handler", DefaultDataQueueHandler), DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, Version = Globals.Version, DeployId = AlgorithmTypeName, Parameters = parameters, Language = Language, Controls = controls }; try { // import the brokerage data for the configured brokerage var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); liveJob.BrokerageData = brokerageFactory.BrokerageData; } catch (Exception err) { Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}"); } return(liveJob); } //Default run a backtesting job. var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local") { Type = PacketType.BacktestNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, Version = Globals.Version, BacktestId = AlgorithmTypeName, Language = Language, Parameters = parameters, Controls = controls }; return(backtestJob); }
/// <summary> /// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution. /// </summary> /// <returns></returns> public AlgorithmNodePacket NextJob(out string location) { location = GetAlgorithmLocation(); Log.Trace($"JobQueue.NextJob(): Selected {location}"); // check for parameters in the config var parameters = new Dictionary <string, string>(); var parametersConfigString = Config.Get("parameters"); if (parametersConfigString != string.Empty) { parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString); } var controls = new Controls() { MinuteLimit = Config.GetInt("symbol-minute-limit", 10000), SecondLimit = Config.GetInt("symbol-second-limit", 10000), TickLimit = Config.GetInt("symbol-tick-limit", 10000), RamAllocation = int.MaxValue, MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000) }; if ((Language)Enum.Parse(typeof(Language), Config.Get("algorithm-language")) == Language.Python) { // Set the python path for loading python algorithms ("algorithm-location" config parameter) var pythonFile = new FileInfo(location); // PythonInitializer automatically adds the current working directory for us PythonInitializer.SetPythonPathEnvironmentVariable(new string[] { pythonFile.Directory.FullName }); } var algorithmId = Config.Get("algorithm-id", AlgorithmTypeName); //If this isn't a backtesting mode/request, attempt a live job. if (_liveMode) { var liveJob = new LiveNodePacket { Type = PacketType.LiveNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), DataQueueHandler = Config.Get("data-queue-handler", DefaultDataQueueHandler), DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, Version = Globals.Version, DeployId = algorithmId, Parameters = parameters, Language = Language, Controls = controls }; try { // import the brokerage data for the configured brokerage var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); liveJob.BrokerageData = brokerageFactory.BrokerageData; } catch (Exception err) { Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}"); } return(liveJob); } //Default run a backtesting job. var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, "local") { Type = PacketType.BacktestNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, Version = Globals.Version, BacktestId = algorithmId, Language = Language, Parameters = parameters, Controls = controls }; string periodStart = Config.Get("period-start"); string periodFinish = Config.Get("period-finish"); string cashAmount = Config.Get("cash-amount"); if (!string.IsNullOrEmpty(periodStart)) { backtestJob.PeriodStart = DateTime.Parse(periodStart, CultureInfo.InvariantCulture); } if (!string.IsNullOrEmpty(periodFinish)) { backtestJob.PeriodFinish = DateTime.Parse(periodFinish, CultureInfo.InvariantCulture); } if (!string.IsNullOrEmpty(cashAmount)) { decimal amount = Decimal.Parse(cashAmount, CultureInfo.InvariantCulture); if (amount > 0) { backtestJob.CashAmount = new CashAmount(amount, Currencies.USD); } } return(backtestJob); }
/// <summary> /// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution. /// </summary> /// <returns></returns> public AlgorithmNodePacket NextJob(out string location) { location = GetAlgorithmLocation(); Log.Trace($"JobQueue.NextJob(): Selected {location}"); // check for parameters in the config var parameters = new Dictionary <string, string>(); var parametersConfigString = Config.Get("parameters"); if (parametersConfigString != string.Empty) { parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString); } var controls = new Controls() { MinuteLimit = Config.GetInt("symbol-minute-limit", 10000), SecondLimit = Config.GetInt("symbol-second-limit", 10000), TickLimit = Config.GetInt("symbol-tick-limit", 10000), RamAllocation = int.MaxValue, MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000) }; var algorithmId = Config.Get("algorithm-id", AlgorithmTypeName); //If this isn't a backtesting mode/request, attempt a live job. if (_liveMode) { var liveJob = new LiveNodePacket { Type = PacketType.LiveNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), DataQueueHandler = Config.Get("data-queue-handler", DefaultDataQueueHandler), DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, OrganizationId = OrganizationId, Version = Globals.Version, DeployId = algorithmId, Parameters = parameters, Language = Language, Controls = controls }; try { // import the brokerage data for the configured brokerage var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); liveJob.BrokerageData = brokerageFactory.BrokerageData; } catch (Exception err) { Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}"); } return(liveJob); } //Default run a backtesting job. var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, "local") { Type = PacketType.BacktestNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, OrganizationId = OrganizationId, Version = Globals.Version, BacktestId = algorithmId, Language = Language, Parameters = parameters, Controls = controls }; return(backtestJob); }
/// <summary> /// Creates an instance of the PortfolioLooper class /// </summary> /// <param name="startingCash">Equity curve</param> /// <param name="orders">Order events</param> /// <param name="resolution">Optional parameter to override default resolution (Hourly)</param> private PortfolioLooper(double startingCash, List <Order> orders, Resolution resolution = _resolution) { // Initialize the providers that the HistoryProvider requires var factorFileProvider = Composer.Instance.GetExportedValueByTypeName <IFactorFileProvider>("LocalDiskFactorFileProvider"); var mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>("LocalDiskMapFileProvider"); _cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), false); var historyProvider = new SubscriptionDataReaderHistoryProvider(); var dataPermissionManager = new DataPermissionManager(); historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, _cacheProvider, mapFileProvider, factorFileProvider, (_) => { }, false, dataPermissionManager)); Algorithm = new PortfolioLooperAlgorithm((decimal)startingCash, orders); Algorithm.SetHistoryProvider(historyProvider); // Dummy LEAN datafeed classes and initializations that essentially do nothing var job = new BacktestNodePacket(1, 2, "3", null, 9m, $""); var feed = new MockDataFeed(); // Create MHDB and Symbol properties DB instances for the DataManager var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); _dataManager = new DataManager(feed, new UniverseSelection( Algorithm, new SecurityService(Algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, Algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(Algorithm.Portfolio)), dataPermissionManager, new DefaultDataProvider()), Algorithm, Algorithm.TimeKeeper, marketHoursDatabase, false, RegisteredSecurityDataTypesProvider.Null, dataPermissionManager); _securityService = new SecurityService(Algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, Algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(Algorithm.Portfolio)); var transactions = new BacktestingTransactionHandler(); _resultHandler = new BacktestingResultHandler(); // Initialize security services and other properties so that we // don't get null reference exceptions during our re-calculation Algorithm.Securities.SetSecurityService(_securityService); Algorithm.SubscriptionManager.SetDataManager(_dataManager); // Initializes all the proper Securities from the orders provided by the user Algorithm.FromOrders(orders); // Initialize the algorithm Algorithm.Initialize(); Algorithm.PostInitialize(); // More initialization, this time with Algorithm and other misc. classes _resultHandler.Initialize(job, new Messaging.Messaging(), new Api.Api(), transactions); _resultHandler.SetAlgorithm(Algorithm, Algorithm.Portfolio.TotalPortfolioValue); Algorithm.Transactions.SetOrderProcessor(transactions); transactions.Initialize(Algorithm, new BacktestingBrokerage(Algorithm), _resultHandler); feed.Initialize(Algorithm, job, _resultHandler, null, null, null, _dataManager, null, null); // Begin setting up the currency conversion feed if needed var coreSecurities = Algorithm.Securities.Values.ToList(); if (coreSecurities.Any(x => x.Symbol.SecurityType == SecurityType.Forex || x.Symbol.SecurityType == SecurityType.Crypto)) { BaseSetupHandler.SetupCurrencyConversions(Algorithm, _dataManager.UniverseSelection); var conversionSecurities = Algorithm.Securities.Values.Where(s => !coreSecurities.Contains(s)).ToList(); // Skip the history request if we don't need to convert anything if (conversionSecurities.Any()) { // Point-in-time Slices to convert FX and Crypto currencies to the portfolio currency _conversionSlices = GetHistory(Algorithm, conversionSecurities, resolution); } } }
/******************************************************** * CLASS PROPERTIES *********************************************************/ /******************************************************** * CLASS CONSTRUCTOR *********************************************************/ /// <summary> /// Create and connect to IQFeed. /// </summary> public IQFeedDataFeed(IAlgorithm algorithm, BacktestNodePacket job) : base(algorithm, job) { _algorithm = algorithm; _job = job; }
/// <summary> /// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution. /// </summary> /// <returns></returns> public AlgorithmNodePacket NextJob(out string location) { location = AlgorithmLocation; Log.Trace("JobQueue.NextJob(): Selected " + location); // check for parameters in the config var parameters = new Dictionary <string, string>(); var parametersConfigString = Config.Get("parameters"); if (parametersConfigString != string.Empty) { parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString); } var controls = new Controls() { MinuteLimit = Config.GetInt("symbol-minute-limit", 10000), SecondLimit = Config.GetInt("symbol-second-limit", 10000), TickLimit = Config.GetInt("symbol-tick-limit", 10000) }; //If this isn't a backtesting mode/request, attempt a live job. if (_liveMode) { var liveJob = new LiveNodePacket { Type = PacketType.LiveNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName), Channel = AccessToken, UserId = UserId, ProjectId = ProjectId, Version = Globals.Version, DeployId = AlgorithmTypeName, RamAllocation = int.MaxValue, Parameters = parameters, Language = Language, Controls = controls }; try { // import the brokerage data for the configured brokerage var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); liveJob.BrokerageData = brokerageFactory.BrokerageData; } catch (Exception err) { Log.Error(err, string.Format("Error resolving BrokerageData for live job for brokerage {0}:", liveJob.Brokerage)); } return(liveJob); } //Default run a backtesting job. var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local") { Type = PacketType.BacktestNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), Channel = AccessToken, UserId = UserId, ProjectId = ProjectId, Version = Globals.Version, BacktestId = AlgorithmTypeName, RamAllocation = int.MaxValue, Language = Language, Parameters = parameters, Controls = controls }; return(backtestJob); }
/******************************************************** * CLASS VARIABLES *********************************************************/ /******************************************************** * CLASS PROPERTIES *********************************************************/ /******************************************************** * CLASS CONSTRUCTOR *********************************************************/ /// <summary> /// Pass through the backtesting datafeed to the underlying file system datafeed implementation. /// </summary> /// <param name="algorithm">Algorithm we're operating with</param> /// <param name="job">Algorithm worker job</param> public BacktestingDataFeed(IAlgorithm algorithm, BacktestNodePacket job) : base(algorithm, job) { DataFeed = DataFeedEndpoint.Backtesting; }
/// <summary> /// Resolve max orders for this algorithm /// </summary> /// <param name="job"></param> protected override int GetMaximumOrders(BacktestNodePacket job) { // For local backtest MaxOrders is always max int return(int.MaxValue); }
public void FutureChainEnumerator(bool fillForward) { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var feed = new FileSystemDataFeed(); var algorithm = new AlgorithmStub(feed); algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor()); algorithm.SetStartDate(new DateTime(2013, 10, 07)); algorithm.SetEndDate(new DateTime(2013, 10, 08)); algorithm.SetFutureChainProvider(new BacktestingFutureChainProvider(TestGlobals.DataCacheProvider)); var dataPermissionManager = new DataPermissionManager(); using var synchronizer = new Synchronizer(); synchronizer.Initialize(algorithm, algorithm.DataManager); feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider); var future = algorithm.AddFuture("ES", fillDataForward: fillForward); future.SetFilter(0, 300); algorithm.PostInitialize(); using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); var count = 0L; var lastMonth = algorithm.StartDate.Month; foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token)) { if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0) { var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone); var currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(count % (24 * 60))); while (!future.Exchange.DateTimeIsOpen(nyTime.Date.Add(currentExpectedTime).AddMinutes(-1))) { // skip closed market times currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(++count % (24 * 60))); } var universeData = timeSlice.UniverseData.OrderBy(kvp => kvp.Key.Configuration.Symbol).ToList(); var chainData = universeData[0].Value; Log.Trace($"{nyTime}. Count: {count}. Universe Data Count {universeData.Count}"); Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}. Count: {count}"); Assert.IsTrue(timeSlice.UniverseData.All(kvp => kvp.Value.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay == nyTime.TimeOfDay)); if (chainData.FilteredContracts.IsNullOrEmpty()) { Assert.AreEqual(new DateTime(2013, 10, 09), nyTime, $"Unexpected chain FilteredContracts was empty on {nyTime}"); } if (universeData.Count == 1) { // the chain Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol)); } else { // we have 2 universe data, the chain and the continuous future Assert.AreEqual(2, universeData.Count); Assert.IsTrue(universeData.All(kvp => kvp.Key.Configuration.Symbol.SecurityType == SecurityType.Future)); Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol)); Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol.ID.Symbol.Contains("CONTINUOUS", StringComparison.InvariantCultureIgnoreCase))); var continuousData = universeData[1].Value; Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}"); Assert.IsTrue(!chainData.FilteredContracts.IsNullOrEmpty()); } count++; } } feed.Exit(); algorithm.DataManager.RemoveAllSubscriptions(); // 2 days worth of minute data Assert.AreEqual(24 * 2 * 60 + 1, count); }
/// <summary> /// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution. /// </summary> /// <returns></returns> public AlgorithmNodePacket NextJob(out string location) { location = GetAlgorithmLocation(); Log.Trace($"JobQueue.NextJob(): Selected {location}"); // check for parameters in the config var parameters = new Dictionary <string, string>(); var parametersConfigString = Config.Get("parameters"); if (parametersConfigString != string.Empty) { parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString); } var controls = new Controls() { MinuteLimit = Config.GetInt("symbol-minute-limit", 10000), SecondLimit = Config.GetInt("symbol-second-limit", 10000), TickLimit = Config.GetInt("symbol-tick-limit", 10000), RamAllocation = int.MaxValue, MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000) }; var algorithmId = Config.Get("algorithm-id", AlgorithmTypeName); //If this isn't a backtesting mode/request, attempt a live job. if (_liveMode) { var dataHandlers = Config.Get("data-queue-handler", DefaultDataQueueHandler); var liveJob = new LiveNodePacket { Type = PacketType.LiveNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), DataQueueHandler = dataHandlers, DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, OrganizationId = OrganizationId, Version = Globals.Version, DeployId = algorithmId, Parameters = parameters, Language = Language, Controls = controls }; Type brokerageName = null; try { // import the brokerage data for the configured brokerage var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); brokerageName = brokerageFactory.BrokerageType; liveJob.BrokerageData = brokerageFactory.BrokerageData; } catch (Exception err) { Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}"); } foreach (var dataHandlerName in dataHandlers.DeserializeList()) { var brokerageFactoryForDataHandler = GetFactoryFromDataQueueHandler(dataHandlerName); if (brokerageFactoryForDataHandler == null) { Log.Trace($"JobQueue.NextJob(): Not able to fetch data handler factory with name: {dataHandlerName}"); continue; } if (brokerageFactoryForDataHandler.BrokerageType == brokerageName) { //Don't need to add brokearageData again if added by brokerage continue; } foreach (var data in brokerageFactoryForDataHandler.BrokerageData) { if (data.Key == "live-holdings") { //live-holdings not required for data handler continue; } else if (!liveJob.BrokerageData.ContainsKey(data.Key)) { liveJob.BrokerageData.Add(data.Key, data.Value); } else { throw new ArgumentException($"JobQueue.NextJob(): Key already exists in BrokerageData -- {data.Key}"); } } } return(liveJob); } //Default run a backtesting job. var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] { }, "local") { Type = PacketType.BacktestNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, OrganizationId = OrganizationId, Version = Globals.Version, BacktestId = algorithmId, Language = Language, Parameters = parameters, Controls = controls }; return(backtestJob); }
public BacktestConsoleStatusHandler(BacktestNodePacket _job) { this._job = _job; }