Ejemplo n.º 1
0
 public void Initialize(AlgorithmNodePacket job,
     IMessagingHandler messagingHandler,
     IApi api,
     IDataFeed dataFeed,
     ISetupHandler setupHandler,
     ITransactionHandler transactionHandler)
 {
     _job = job;
 }
 /// <summary>
 /// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
 /// </summary>
 /// <param name="results">The result handler for communicating results from the algorithm</param>
 /// <param name="setup">The setup handler used to initialize algorithm state</param>
 /// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
 /// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
 /// <param name="realTime">The real time handler used to process real time events</param>
 /// <param name="historyProvider">The history provider used to process historical data requests</param>
 /// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param>
 /// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
 public LeanEngineAlgorithmHandlers(IResultHandler results,
     ISetupHandler setup,
     IDataFeed dataFeed,
     ITransactionHandler transactions,
     IRealTimeHandler realTime,
     IHistoryProvider historyProvider,
     ICommandQueueHandler commandQueue,
     IMapFileProvider mapFileProvider
     )
 {
     if (results == null)
     {
         throw new ArgumentNullException("results");
     }
     if (setup == null)
     {
         throw new ArgumentNullException("setup");
     }
     if (dataFeed == null)
     {
         throw new ArgumentNullException("dataFeed");
     }
     if (transactions == null)
     {
         throw new ArgumentNullException("transactions");
     }
     if (realTime == null)
     {
         throw new ArgumentNullException("realTime");
     }
     if (historyProvider == null)
     {
         throw new ArgumentNullException("realTime");
     }
     if (commandQueue == null)
     {
         throw new ArgumentNullException("commandQueue");
     }
     if (mapFileProvider == null)
     {
         throw new ArgumentNullException("mapFileProvider");
     }
     _results = results;
     _setup = setup;
     _dataFeed = dataFeed;
     _transactions = transactions;
     _realTime = realTime;
     _historyProvider = historyProvider;
     _commandQueue = commandQueue;
     _mapFileProvider = mapFileProvider;
 }
Ejemplo n.º 3
0
 /// <summary>
 /// Initialize the result handler with this result packet.
 /// </summary>
 /// <param name="job">Algorithm job packet for this result handler</param>
 /// <param name="messagingHandler"></param>
 /// <param name="api"></param>
 /// <param name="dataFeed"></param>
 /// <param name="setupHandler"></param>
 /// <param name="transactionHandler"></param>
 public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
 {
     //Redirect the log messages here:
     _job = job;
     var desktopLogging = new FunctionalLogHandler(DebugMessage, DebugMessage, ErrorMessage);
     Log.LogHandler = new CompositeLogHandler(new[] { desktopLogging, Log.LogHandler });
 }
Ejemplo n.º 4
0
 /// <summary>
 /// Initialize the result handler with this result packet.
 /// </summary>
 /// <param name="job">Algorithm job packet for this result handler</param>
 /// <param name="messagingHandler"></param>
 /// <param name="api"></param>
 /// <param name="dataFeed"></param>
 /// <param name="setupHandler"></param>
 /// <param name="transactionHandler"></param>
 public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
 {
     _api = api;
     _dataFeed = dataFeed;
     _messagingHandler = messagingHandler;
     _setupHandler = setupHandler;
     _transactionHandler = transactionHandler;
     _job = (LiveNodePacket)job;
     if (_job == null) throw new Exception("LiveResultHandler.Constructor(): Submitted Job type invalid."); 
     _deployId = _job.DeployId;
     _compileId = _job.CompileId;
 }
Ejemplo n.º 5
0
 /// <summary>
 /// Initialize the result handler with this result packet.
 /// </summary>
 /// <param name="job">Algorithm job packet for this result handler</param>
 /// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param>
 /// <param name="api">The api instance used for handling logs</param>
 /// <param name="dataFeed"></param>
 /// <param name="setupHandler"></param>
 /// <param name="transactionHandler"></param>
 public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
 {
     _api = api;
     _messagingHandler   = messagingHandler;
     _transactionHandler = transactionHandler;
     _setupHandler       = setupHandler;
     _job = (BacktestNodePacket)job;
     if (_job == null)
     {
         throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
     }
     _compileId  = _job.CompileId;
     _backtestId = _job.BacktestId;
 }
        /// <summary>
        /// Initialize the result handler with this result packet.
        /// </summary>
        /// <param name="job">Algorithm job packet for this result handler</param>
        /// <param name="messagingHandler"></param>
        /// <param name="api"></param>
        /// <param name="dataFeed"></param>
        /// <param name="setupHandler"></param>
        /// <param name="transactionHandler"></param>
        public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
        {
            //Redirect the log messages here:
            _job = job;
            var desktopLogging = new FunctionalLogHandler(DebugMessage, DebugMessage, ErrorMessage);

            Log.LogHandler = new CompositeLogHandler(new[] { desktopLogging, Log.LogHandler });
        }
Ejemplo n.º 7
0
        /********************************************************
         * CLASS METHODS
         *********************************************************/
        /// <summary>
        /// Launch the algorithm manager to run this strategy
        /// </summary>
        /// <param name="job">Algorithm job</param>
        /// <param name="algorithm">Algorithm instance</param>
        /// <param name="feed">Datafeed object</param>
        /// <param name="transactions">Transaction manager object</param>
        /// <param name="results">Result handler object</param>
        /// <param name="setup">Setup handler object</param>
        /// <param name="realtime">Realtime processing object</param>
        /// <remarks>Modify with caution</remarks>
        public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
        {
            //Initialize:
            _dataPointCount = 0;
            var backwardsCompatibilityMode = false;
            var tradebarsType          = typeof(TradeBars);
            var ticksType              = typeof(Ticks);
            var startingPortfolioValue = setup.StartingCapital;
            var backtestMode           = (job.Type == PacketType.BacktestNode);
            var methodInvokers         = new Dictionary <Type, MethodInvoker>();
            var marginCallFrequency    = TimeSpan.FromMinutes(5);
            var nextMarginCallTime     = DateTime.MinValue;

            //Initialize Properties:
            _frontier       = setup.StartingDate;
            _algorithmId    = job.AlgorithmId;
            _algorithmState = AlgorithmStatus.Running;
            _previousTime   = setup.StartingDate.Date;

            //Create the method accessors to push generic types into algorithm: Find all OnData events:

            //Algorithm 1.0 Data Accessors.
            //If the users defined these methods, add them in manually. This allows keeping backwards compatibility to algorithm 1.0.
            var oldTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnTradeBar", new[] { typeof(Dictionary <string, TradeBar>) });
            var oldTicksMethodInfo     = (algorithm.GetType()).GetMethod("OnTick", new[] { typeof(Dictionary <string, List <Tick> >) });

            //Algorithm 2.0 Data Generics Accessors.
            //New hidden access to tradebars with custom type.
            var newTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { tradebarsType });
            var newTicksMethodInfo     = (algorithm.GetType()).GetMethod("OnData", new[] { ticksType });

            if (newTradeBarsMethodInfo == null && newTicksMethodInfo == null)
            {
                backwardsCompatibilityMode = true;
                if (oldTradeBarsMethodInfo != null)
                {
                    methodInvokers.Add(tradebarsType, oldTradeBarsMethodInfo.DelegateForCallMethod());
                }
                if (oldTradeBarsMethodInfo != null)
                {
                    methodInvokers.Add(ticksType, oldTicksMethodInfo.DelegateForCallMethod());
                }
            }
            else
            {
                backwardsCompatibilityMode = false;
                if (newTradeBarsMethodInfo != null)
                {
                    methodInvokers.Add(tradebarsType, newTradeBarsMethodInfo.DelegateForCallMethod());
                }
                if (newTicksMethodInfo != null)
                {
                    methodInvokers.Add(ticksType, newTicksMethodInfo.DelegateForCallMethod());
                }
            }

            //Go through the subscription types and create invokers to trigger the event handlers for each custom type:
            foreach (var config in feed.Subscriptions)
            {
                //If type is a tradebar, combine tradebars and ticks into unified array:
                if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
                {
                    //Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
                    var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });

                    //If we already have this Type-handler then don't add it to invokers again.
                    if (methodInvokers.ContainsKey(config.Type))
                    {
                        continue;
                    }

                    //If we couldnt find the event handler, let the user know we can't fire that event.
                    if (genericMethod == null)
                    {
                        algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) {  }");
                        _algorithmState        = AlgorithmStatus.RuntimeError;
                        return;
                    }
                    methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
                }
            }

            //Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
            Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
            foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
            {
                //Check this backtest is still running:
                if (_algorithmState != AlgorithmStatus.Running)
                {
                    break;
                }

                //Go over each time stamp we've collected, pass it into the algorithm in order:
                foreach (var time in newData.Keys)
                {
                    //Set the time frontier:
                    _frontier = time;

                    //Execute with TimeLimit Monitor:
                    if (Isolator.IsCancellationRequested)
                    {
                        return;
                    }

                    //If we're in backtest mode we need to capture the daily performance. We do this here directly
                    //before updating the algorithm state with the new data from this time step, otherwise we'll
                    //produce incorrect samples (they'll take into account this time step's new price values)
                    if (backtestMode)
                    {
                        //Refresh the realtime event monitor:
                        //in backtest mode use the algorithms clock as realtime.
                        realtime.SetTime(time);

                        //On day-change sample equity and daily performance for statistics calculations
                        if (_previousTime.Date != time.Date)
                        {
                            //Sample the portfolio value over time for chart.
                            results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));

                            //Check for divide by zero
                            if (startingPortfolioValue == 0m)
                            {
                                results.SamplePerformance(_previousTime.Date, 0);
                            }
                            else
                            {
                                results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
                            }
                            startingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
                        }
                    }

                    //Update algorithm state after capturing performance from previous day

                    //On each time step push the real time prices to the cashbook so we can have updated conversion rates
                    algorithm.Portfolio.CashBook.Update(newData[time]);

                    //Update the securities properties: first before calling user code to avoid issues with data
                    algorithm.Securities.Update(time, newData[time]);

                    // perform margin calls
                    if (time >= nextMarginCallTime)
                    {
                        // determine if there are possible margin call orders to be executed
                        var marginCallOrders = algorithm.Portfolio.ScanForMarginCall();
                        if (marginCallOrders.Count != 0)
                        {
                            // execute the margin call orders
                            var executedOrders = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders);
                            foreach (var order in executedOrders)
                            {
                                algorithm.Error(string.Format("Executed MarginCallOrder: {0} - Quantity: {1} @ {2}", order.Symbol, order.Quantity, order.Price));
                            }
                        }

                        nextMarginCallTime = time + marginCallFrequency;
                    }


                    //Check if the user's signalled Quit: loop over data until day changes.
                    if (algorithm.GetQuit())
                    {
                        _algorithmState = AlgorithmStatus.Quit;
                        break;
                    }

                    //Pass in the new time first:
                    algorithm.SetDateTime(time);

                    //Trigger the data events: Invoke the types we have data for:
                    var oldBars  = new Dictionary <string, TradeBar>();
                    var oldTicks = new Dictionary <string, List <Tick> >();
                    var newBars  = new TradeBars(time);
                    var newTicks = new Ticks(time);

                    //Invoke all non-tradebars, non-ticks methods and build up the TradeBars and Ticks dictionaries
                    // --> i == Subscription Configuration Index, so we don't need to compare types.
                    foreach (var i in newData[time].Keys)
                    {
                        //Data point and config of this point:
                        var dataPoints = newData[time][i];
                        var config     = feed.Subscriptions[i];

                        //Keep track of how many data points we've processed
                        _dataPointCount += dataPoints.Count;

                        //We don't want to pump data that we added just for currency conversions
                        if (config.IsInternalFeed)
                        {
                            continue;
                        }

                        //Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
                        foreach (var dataPoint in dataPoints)
                        {
                            //Update registered consolidators for this symbol index
                            try
                            {
                                for (var j = 0; j < config.Consolidators.Count; j++)
                                {
                                    config.Consolidators[j].Update(dataPoint);
                                }
                            }
                            catch (Exception err)
                            {
                                algorithm.RunTimeError = err;
                                _algorithmState        = AlgorithmStatus.RuntimeError;
                                Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err.Message);
                                return;
                            }

                            switch (config.Type.Name)
                            {
                            case "TradeBar":
                                var bar = dataPoint as TradeBar;
                                try
                                {
                                    if (bar != null)
                                    {
                                        if (backwardsCompatibilityMode)
                                        {
                                            if (!oldBars.ContainsKey(bar.Symbol))
                                            {
                                                oldBars.Add(bar.Symbol, bar);
                                            }
                                        }
                                        else
                                        {
                                            if (!newBars.ContainsKey(bar.Symbol))
                                            {
                                                newBars.Add(bar.Symbol, bar);
                                            }
                                        }
                                    }
                                }
                                catch (Exception err)
                                {
                                    Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> " + bar.Value.ToString("C"));
                                    Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString() + ") Count:(" + newBars.Count + ") " + err.Message);
                                }
                                break;

                            case "Tick":
                                var tick = dataPoint as Tick;
                                if (tick != null)
                                {
                                    if (backwardsCompatibilityMode)
                                    {
                                        if (!oldTicks.ContainsKey(tick.Symbol))
                                        {
                                            oldTicks.Add(tick.Symbol, new List <Tick>());
                                        }
                                        oldTicks[tick.Symbol].Add(tick);
                                    }
                                    else
                                    {
                                        if (!newTicks.ContainsKey(tick.Symbol))
                                        {
                                            newTicks.Add(tick.Symbol, new List <Tick>());
                                        }
                                        newTicks[tick.Symbol].Add(tick);
                                    }
                                }
                                break;

                            default:
                                //Send data into the generic algorithm event handlers
                                try
                                {
                                    methodInvokers[config.Type](algorithm, dataPoint);
                                }
                                catch (Exception err)
                                {
                                    algorithm.RunTimeError = err;
                                    _algorithmState        = AlgorithmStatus.RuntimeError;
                                    Log.Debug("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
                                    return;
                                }
                                break;
                            }
                        }
                    }

                    //After we've fired all other events in this second, fire the pricing events:
                    if (backwardsCompatibilityMode)
                    {
                        //Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
                        try
                        {
                            if (oldTradeBarsMethodInfo != null && oldBars.Count > 0)
                            {
                                methodInvokers[tradebarsType](algorithm, oldBars);
                            }
                            if (oldTicksMethodInfo != null && oldTicks.Count > 0)
                            {
                                methodInvokers[ticksType](algorithm, oldTicks);
                            }
                        }
                        catch (Exception err)
                        {
                            algorithm.RunTimeError = err;
                            _algorithmState        = AlgorithmStatus.RuntimeError;
                            Log.Debug("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
                            return;
                        }
                    }
                    else
                    {
                        //Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
                        try
                        {
                            if (newTradeBarsMethodInfo != null && newBars.Count > 0)
                            {
                                methodInvokers[tradebarsType](algorithm, newBars);
                            }
                            if (newTicksMethodInfo != null && newTicks.Count > 0)
                            {
                                methodInvokers[ticksType](algorithm, newTicks);
                            }
                        }
                        catch (Exception err)
                        {
                            algorithm.RunTimeError = err;
                            _algorithmState        = AlgorithmStatus.RuntimeError;
                            Log.Debug("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
                            return;
                        }
                    }

                    //If its the historical/paper trading models, wait until market orders have been "filled"
                    // Manually trigger the event handler to prevent thread switch.
                    transactions.ProcessSynchronousEvents();

                    //Save the previous time for the sample calculations
                    _previousTime = time;

                    // Process any required events of the results handler such as sampling assets, equity, or stock prices.
                    results.ProcessSynchronousEvents();
                } // End of Time Loop
            }     // End of ForEach DataStream

            //Stream over:: Send the final packet and fire final events:
            Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
            try
            {
                algorithm.OnEndOfAlgorithm();
            }
            catch (Exception err)
            {
                _algorithmState        = AlgorithmStatus.RuntimeError;
                algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
                Log.Debug("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
                return;
            }

            // Process any required events of the results handler such as sampling assets, equity, or stock prices.
            results.ProcessSynchronousEvents(forceProcess: true);

            //Liquidate Holdings for Calculations:
            if (_algorithmState == AlgorithmStatus.Liquidated || !Engine.LiveMode)
            {
                Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
                algorithm.Liquidate();
                results.LogMessage("Algorithm Liquidated");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
            }

            //Manually stopped the algorithm
            if (_algorithmState == AlgorithmStatus.Stopped)
            {
                Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
                results.LogMessage("Algorithm Stopped");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
            }

            //Backtest deleted.
            if (_algorithmState == AlgorithmStatus.Deleted)
            {
                Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
                results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
            }

            //Algorithm finished, send regardless of commands:
            results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);

            //Take final samples:
            results.SampleRange(algorithm.GetChartUpdates());
            results.SampleEquity(_frontier, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
            results.SamplePerformance(_frontier, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
        } // End of Run();
 /// <summary>
 /// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
 /// </summary>
 /// <param name="results">The result handler for communicating results from the algorithm</param>
 /// <param name="setup">The setup handler used to initialize algorithm state</param>
 /// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
 /// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
 /// <param name="realTime">The real time handler used to process real time events</param>
 /// <param name="historyProvider">The history provider used to process historical data requests</param>
 /// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param>
 /// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
 /// <param name="factorFileProvider">Map file provider used as a map file source for the data feed</param>
 /// <param name="dataFileProvider">file provider used to retrieve security data if it is not on the file system</param>
 public LeanEngineAlgorithmHandlers(IResultHandler results,
                                    ISetupHandler setup,
                                    IDataFeed dataFeed,
                                    ITransactionHandler transactions,
                                    IRealTimeHandler realTime,
                                    IHistoryProvider historyProvider,
                                    ICommandQueueHandler commandQueue,
                                    IMapFileProvider mapFileProvider,
                                    IFactorFileProvider factorFileProvider,
                                    IDataFileProvider dataFileProvider
                                    )
 {
     if (results == null)
     {
         throw new ArgumentNullException("results");
     }
     if (setup == null)
     {
         throw new ArgumentNullException("setup");
     }
     if (dataFeed == null)
     {
         throw new ArgumentNullException("dataFeed");
     }
     if (transactions == null)
     {
         throw new ArgumentNullException("transactions");
     }
     if (realTime == null)
     {
         throw new ArgumentNullException("realTime");
     }
     if (historyProvider == null)
     {
         throw new ArgumentNullException("realTime");
     }
     if (commandQueue == null)
     {
         throw new ArgumentNullException("commandQueue");
     }
     if (mapFileProvider == null)
     {
         throw new ArgumentNullException("mapFileProvider");
     }
     if (factorFileProvider == null)
     {
         throw new ArgumentNullException("factorFileProvider");
     }
     if (dataFileProvider == null)
     {
         throw new ArgumentNullException("dataFileProvider");
     }
     _results            = results;
     _setup              = setup;
     _dataFeed           = dataFeed;
     _transactions       = transactions;
     _realTime           = realTime;
     _historyProvider    = historyProvider;
     _commandQueue       = commandQueue;
     _mapFileProvider    = mapFileProvider;
     _factorFileProvider = factorFileProvider;
     _dataFileProvider   = dataFileProvider;
 }
Ejemplo n.º 9
0
        /********************************************************
         * CLASS METHODS
         *********************************************************/
        /// <summary>
        /// Primary Analysis Thread:
        /// </summary>
        public static void Main(string[] args)
        {
            //Initialize:
            var    algorithmPath    = "";
            string mode             = "RELEASE";
            AlgorithmNodePacket job = null;
            var algorithm           = default(IAlgorithm);
            var startTime           = DateTime.Now;

            Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
            _version       = DateTime.ParseExact(Config.Get("version", DateTime.Now.ToString(DateFormat.UI)), DateFormat.UI, CultureInfo.InvariantCulture);

            #if DEBUG
            mode = "DEBUG";
            #endif

            //Name thread for the profiler:
            Thread.CurrentThread.Name = "Algorithm Analysis Thread";
            Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + _version + " Mode: " + mode);
            Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
            Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App  " + +OS.TotalPhysicalMemoryUsed + "Mb-Used  " + OS.TotalPhysicalMemory + "Mb-Total");

            //Import external libraries specific to physical server location (cloud/local)
            try
            {
                // grab the right export based on configuration
                Api      = Composer.Instance.GetExportedValueByTypeName <IApi>(Config.Get("api-handler"));
                Notify   = Composer.Instance.GetExportedValueByTypeName <IMessagingHandler>(Config.Get("messaging-handler"));
                JobQueue = Composer.Instance.GetExportedValueByTypeName <IJobQueueHandler>(Config.Get("job-queue-handler"));
            }
            catch (CompositionException compositionException)
            { Log.Error("Engine.Main(): Failed to load library: " + compositionException); }

            //Setup packeting, queue and controls system: These don't do much locally.
            Api.Initialize();
            Notify.Initialize();
            JobQueue.Initialize();

            //Start monitoring the backtest active status:
            var statusPingThread = new Thread(StateCheck.Ping.Run);
            statusPingThread.Start();

            do
            {
                try
                {
                    //Reset algo manager internal variables preparing for a new algorithm.
                    AlgorithmManager.ResetManager();

                    //Reset thread holders.
                    var    initializeComplete = false;
                    Thread threadFeed         = null;
                    Thread threadTransactions = null;
                    Thread threadResults      = null;
                    Thread threadRealTime     = null;

                    do
                    {
                        //-> Pull job from QuantConnect job queue, or, pull local build:
                        job = JobQueue.NextJob(out algorithmPath); // Blocking.

                        if (!IsLocal && LiveMode && (job.Version < Version || (job.Version == Version && job.Redelivered)))
                        {
                            //Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
                            //In this event kill the old algorithm and leave a message so the user can later review.
                            JobQueue.AcknowledgeJob(job);
                            Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
                            Notify.SetChannel(job.Channel);
                            Notify.RuntimeError(job.AlgorithmId, _collapseMessage);
                            job = null;
                        }
                    } while (job == null);


                    //-> Initialize messaging system
                    Notify.SetChannel(job.Channel);

                    //-> Create SetupHandler to configure internal algorithm state:
                    SetupHandler = GetSetupHandler(job.SetupEndpoint);

                    //-> Set the result handler type for this algorithm job, and launch the associated result thread.
                    ResultHandler = GetResultHandler(job);
                    threadResults = new Thread(ResultHandler.Run, 0)
                    {
                        Name = "Result Thread"
                    };
                    threadResults.Start();

                    try
                    {
                        // Save algorithm to cache, load algorithm instance:
                        algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath);

                        //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
                        initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job);

                        //If there are any reasons it failed, pass these back to the IDE.
                        if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0)
                        {
                            initializeComplete = false;
                            //Get all the error messages: internal in algorithm and external in setup handler.
                            var errorMessage = String.Join(",", algorithm.ErrorMessages);
                            errorMessage += String.Join(",", SetupHandler.Errors);
                            throw new Exception(errorMessage);
                        }
                    }
                    catch (Exception err)
                    {
                        var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
                        ResultHandler.RuntimeError(runtimeMessage, err.StackTrace);
                        Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
                    }

                    //-> Using the job + initialization: load the designated handlers:
                    if (initializeComplete)
                    {
                        //-> Reset the backtest stopwatch; we're now running the algorithm.
                        startTime = DateTime.Now;

                        //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
                        algorithm.SetAlgorithmId(job.AlgorithmId);
                        algorithm.SetLiveMode(LiveMode);
                        algorithm.SetLocked();

                        //Load the associated handlers for data, transaction and realtime events:
                        ResultHandler.SetAlgorithm(algorithm);
                        DataFeed           = GetDataFeedHandler(algorithm, job);
                        TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job);
                        RealTimeHandler    = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job);

                        //Set the error handlers for the brokerage asynchronous errors.
                        SetupHandler.SetupErrorHandler(ResultHandler, _brokerage);

                        //Send status to user the algorithm is now executing.
                        ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);

                        //Launch the data, transaction and realtime handlers into dedicated threads
                        threadFeed = new Thread(DataFeed.Run, 0)
                        {
                            Name = "DataFeed Thread"
                        };
                        threadTransactions = new Thread(TransactionHandler.Run, 0)
                        {
                            Name = "Transaction Thread"
                        };
                        threadRealTime = new Thread(RealTimeHandler.Run, 0)
                        {
                            Name = "RealTime Thread"
                        };

                        //Launch the data feed, result sending, and transaction models/handlers in separate threads.
                        threadFeed.Start();         // Data feed pushing data packets into thread bridge;
                        threadTransactions.Start(); // Transaction modeller scanning new order requests
                        threadRealTime.Start();     // RealTime scan time for time based events:
                        // Result manager scanning message queue: (started earlier)

                        try
                        {
                            // Execute the Algorithm Code:
                            var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, () =>
                            {
                                try
                                {
                                    //Run Algorithm Job:
                                    // -> Using this Data Feed,
                                    // -> Send Orders to this TransactionHandler,
                                    // -> Send Results to ResultHandler.
                                    AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler);
                                }
                                catch (Exception err)
                                {
                                    //Debugging at this level is difficult, stack trace needed.
                                    Log.Error("Engine.Run(): Error in Algo Manager: " + err.Message + " ST >> " + err.StackTrace);
                                }

                                Log.Trace("Engine.Run(): Exiting Algorithm Manager");
                            }, MaximumRamAllocation);

                            if (!complete)
                            {
                                Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F"));
                                throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster.");
                            }

                            // Algorithm runtime error:
                            if (algorithm.RunTimeError != null)
                            {
                                throw algorithm.RunTimeError;
                            }
                        }
                        catch (Exception err)
                        {
                            //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
                            Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
                            if (DataFeed != null)
                            {
                                DataFeed.Exit();
                            }
                            if (ResultHandler != null)
                            {
                                var message = "Runtime Error: " + err.Message;
                                Log.Trace("Engine.Run(): Sending runtime error to user...");
                                ResultHandler.LogMessage(message);
                                ResultHandler.RuntimeError(message, err.StackTrace);
                                Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
                            }
                        }

                        //Send result data back: this entire code block could be rewritten.
                        // todo: - Split up statistics class, its enormous.
                        // todo: - Make a dedicated Statistics.Benchmark class.
                        // todo: - Move all creation and transmission of statistics out of primary engine loop.
                        // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);

                        try
                        {
                            var charts     = new Dictionary <string, Chart>(ResultHandler.Charts);
                            var orders     = new Dictionary <int, Order>(algorithm.Transactions.Orders);
                            var holdings   = new Dictionary <string, Holding>();
                            var statistics = new Dictionary <string, string>();
                            var banner     = new Dictionary <string, string>();

                            try
                            {
                                //Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
                                const string strategyEquityKey   = "Strategy Equity";
                                const string equityKey           = "Equity";
                                const string dailyPerformanceKey = "Daily Performance";

                                // make sure we've taken samples for these series before just blindly requesting them
                                if (charts.ContainsKey(strategyEquityKey) &&
                                    charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
                                    charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
                                {
                                    var equity      = charts[strategyEquityKey].Series[equityKey].Values;
                                    var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
                                    var profitLoss  =
                                        new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
                                    statistics = Statistics.Statistics.Generate(equity, profitLoss, performance,
                                                                                SetupHandler.StartingCapital, 252);
                                }
                            }
                            catch (Exception err)
                            {
                                Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
                            }

                            //Diagnostics Completed, Send Result Packet:
                            var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
                            ResultHandler.DebugMessage(string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
                                                                     job.AlgorithmId, totalSeconds.ToString("F2"), ((AlgorithmManager.DataPoints / (double)1000) / totalSeconds).ToString("F0"), AlgorithmManager.DataPoints.ToString("N0")));

                            ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
                        }
                        catch (Exception err)
                        {
                            Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + "  ST >> " + err.StackTrace);
                        }

                        //Before we return, send terminate commands to close up the threads
                        TransactionHandler.Exit();
                        DataFeed.Exit();
                        RealTimeHandler.Exit();
                    }

                    //Close result handler:
                    ResultHandler.Exit();

                    //Wait for the threads to complete:
                    var ts = Stopwatch.StartNew();
                    while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000)
                    {
                        Thread.Sleep(100); Log.Trace("Waiting for threads to exit...");
                    }
                    if (threadFeed != null && threadFeed.IsAlive)
                    {
                        threadFeed.Abort();
                    }
                    if (threadTransactions != null && threadTransactions.IsAlive)
                    {
                        threadTransactions.Abort();
                    }
                    if (threadResults != null && threadResults.IsAlive)
                    {
                        threadResults.Abort();
                    }
                    Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
                }
                catch (Exception err)
                {
                    Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
                }
                finally
                {
                    //Delete the message from the job queue:
                    JobQueue.AcknowledgeJob(job);
                    Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);

                    //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
                    if (LiveMode && AlgorithmManager.State != AlgorithmStatus.Running && AlgorithmManager.State != AlgorithmStatus.RuntimeError)
                    {
                        Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmManager.State);
                    }

                    //Attempt to clean up ram usage:
                    GC.Collect();
                }
                //If we're running locally will execute just once.
            } while (!IsLocal);

            // Send the exit signal and then kill the thread
            StateCheck.Ping.Exit();

            // Make the console window pause so we can read log output before exiting and killing the application completely
            Console.Read();

            //Finally if ping thread still not complete, kill.
            if (statusPingThread != null && statusPingThread.IsAlive)
            {
                statusPingThread.Abort();
            }

            if (Log.LogHandler != null)
            {
                Log.LogHandler.Dispose();
            }
        }
Ejemplo n.º 10
0
        /// <summary>
        /// Initialize the result handler with this result packet.
        /// </summary>
        /// <param name="job">Algorithm job packet for this result handler</param>
        /// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param>
        /// <param name="api">The api instance used for handling logs</param>
        /// <param name="dataFeed"></param>
        /// <param name="setupHandler"></param>
        /// <param name="transactionHandler"></param>
        public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
        {
            _api = api;
            _messagingHandler = messagingHandler;
            _transactionHandler = transactionHandler;
            _job = (BacktestNodePacket)job;
            if (_job == null) throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
            _compileId = _job.CompileId;
            _backtestId = _job.BacktestId;

            //Get the resample period:
            var totalMinutes = (_job.PeriodFinish - _job.PeriodStart).TotalMinutes;
            var resampleMinutes = (totalMinutes < (_minimumSamplePeriod * _samples)) ? _minimumSamplePeriod : (totalMinutes / _samples); // Space out the sampling every
            _resamplePeriod = TimeSpan.FromMinutes(resampleMinutes);
            Log.Trace("BacktestingResultHandler(): Sample Period Set: " + resampleMinutes.ToString("00.00"));
        }
Ejemplo n.º 11
0
        /// <summary>
        /// Initialize the result handler with this result packet.
        /// </summary>
        /// <param name="job">Algorithm job packet for this result handler</param>
        /// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param>
        /// <param name="api">The api instance used for handling logs</param>
        /// <param name="dataFeed"></param>
        /// <param name="setupHandler"></param>
        /// <param name="transactionHandler"></param>
        public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
        {
            _api = api;
            _messagingHandler   = messagingHandler;
            _transactionHandler = transactionHandler;
            _job = (BacktestNodePacket)job;
            if (_job == null)
            {
                throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
            }
            _compileId  = _job.CompileId;
            _backtestId = _job.BacktestId;

            //Get the resample period:
            var totalMinutes    = (_job.PeriodFinish - _job.PeriodStart).TotalMinutes;
            var resampleMinutes = (totalMinutes < (_minimumSamplePeriod * _samples)) ? _minimumSamplePeriod : (totalMinutes / _samples); // Space out the sampling every

            _resamplePeriod = TimeSpan.FromMinutes(resampleMinutes);
            Log.Trace("BacktestingResultHandler(): Sample Period Set: " + resampleMinutes.ToString("00.00"));
        }
        /// <summary>
        /// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
        /// </summary>
        /// <param name="results">The result handler for communicating results from the algorithm</param>
        /// <param name="setup">The setup handler used to initialize algorithm state</param>
        /// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
        /// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
        /// <param name="realTime">The real time handler used to process real time events</param>
        /// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
        /// <param name="factorFileProvider">Map file provider used as a map file source for the data feed</param>
        /// <param name="dataProvider">file provider used to retrieve security data if it is not on the file system</param>
        /// <param name="alphas">The alpha handler used to process generated insights</param>
        /// <param name="objectStore">The object store used for persistence</param>
        /// <param name="dataPermissionsManager">The data permission manager to use</param>
        public LeanEngineAlgorithmHandlers(IResultHandler results,
                                           ISetupHandler setup,
                                           IDataFeed dataFeed,
                                           ITransactionHandler transactions,
                                           IRealTimeHandler realTime,
                                           IMapFileProvider mapFileProvider,
                                           IFactorFileProvider factorFileProvider,
                                           IDataProvider dataProvider,
                                           IAlphaHandler alphas,
                                           IObjectStore objectStore,
                                           IDataPermissionManager dataPermissionsManager
                                           )
        {
            if (results == null)
            {
                throw new ArgumentNullException(nameof(results));
            }
            if (setup == null)
            {
                throw new ArgumentNullException(nameof(setup));
            }
            if (dataFeed == null)
            {
                throw new ArgumentNullException(nameof(dataFeed));
            }
            if (transactions == null)
            {
                throw new ArgumentNullException(nameof(transactions));
            }
            if (realTime == null)
            {
                throw new ArgumentNullException(nameof(realTime));
            }
            if (mapFileProvider == null)
            {
                throw new ArgumentNullException(nameof(mapFileProvider));
            }
            if (factorFileProvider == null)
            {
                throw new ArgumentNullException(nameof(factorFileProvider));
            }
            if (dataProvider == null)
            {
                throw new ArgumentNullException(nameof(dataProvider));
            }
            if (alphas == null)
            {
                throw new ArgumentNullException(nameof(alphas));
            }
            if (objectStore == null)
            {
                throw new ArgumentNullException(nameof(objectStore));
            }
            if (dataPermissionsManager == null)
            {
                throw new ArgumentNullException(nameof(dataPermissionsManager));
            }

            Results                = results;
            Setup                  = setup;
            DataFeed               = dataFeed;
            Transactions           = transactions;
            RealTime               = realTime;
            MapFileProvider        = mapFileProvider;
            FactorFileProvider     = factorFileProvider;
            DataProvider           = dataProvider;
            Alphas                 = alphas;
            ObjectStore            = objectStore;
            DataPermissionsManager = dataPermissionsManager;
        }
Ejemplo n.º 13
0
Archivo: Engine.cs Proyecto: mdj/Lean
        /********************************************************
         * CLASS METHODS
         *********************************************************/
        /// <summary>
        /// Primary Analysis Thread:
        /// </summary>
        public static void Main(string[] args)
        {
            //Initialize:
            AlgorithmNodePacket job = null;
            var timer     = Stopwatch.StartNew();
            var algorithm = default(IAlgorithm);

            //Name thread for the profiler:
            Thread.CurrentThread.Name = "Algorithm Analysis Thread";
            Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
            Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App  " + +OS.TotalPhysicalMemoryUsed + "Mb-Used  " + OS.TotalPhysicalMemory + "Mb-Total");

            //Import external libraries specific to physical server location (cloud/local)
            var catalog = new AggregateCatalog();

            catalog.Catalogs.Add(new DirectoryCatalog(@"../../Extensions"));
            var container = new CompositionContainer(catalog);

            try
            {
                // grab the right export based on configuration
                Notify = container.GetExportedValueByTypeName <IMessagingHandler>(Config.Get("messaging-handler"));
                Queue  = container.GetExportedValueByTypeName <IQueueHandler>(Config.Get("queue-handler"));
                Api    = container.GetExportedValueByTypeName <IApi>(Config.Get("api-handler"));
            }
            catch (CompositionException compositionException)
            { Log.Error("Engine.Main(): Failed to load library: " + compositionException); }

            //Setup packeting, queue and controls system: These don't do much locally.
            Api.Initialize();
            Notify.Initialize();
            Queue.Initialize(_liveMode);

            //Start monitoring the backtest active status:
            var statusPingThread = new Thread(StateCheck.Ping.Run);

            statusPingThread.Start();

            do
            {
                try
                {
                    //Clean up cache directories:

                    //Reset algo manager internal variables preparing for a new algorithm.
                    AlgorithmManager.ResetManager();

                    //Reset thread holders.
                    var    initializeComplete = false;
                    Thread threadFeed         = null;
                    Thread threadTransactions = null;
                    Thread threadResults      = null;
                    Thread threadRealTime     = null;

                    //-> Pull job from QuantConnect job queue, or, pull local build:
                    var algorithmPath = "";
                    job = Queue.NextJob(out algorithmPath); // Blocking.

                    //-> Initialize messaging system
                    Notify.SetChannel(job.Channel);

                    //-> Reset the backtest stopwatch; we're now running the algorithm.
                    timer.Restart();

                    //-> Create SetupHandler to configure internal algorithm state:
                    SetupHandler = GetSetupHandler(job.SetupEndpoint);

                    //-> Set the result handler type for this algorithm job, and launch the associated result thread.
                    ResultHandler = GetResultHandler(job);
                    threadResults = new Thread(ResultHandler.Run, 0)
                    {
                        Name = "Result Thread"
                    };
                    threadResults.Start();

                    try
                    {
                        // Save algorithm to cache, load algorithm instance:
                        algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath);

                        //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
                        initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job);

                        //If there are any reasons it failed, pass these back to the IDE.
                        if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0)
                        {
                            initializeComplete = false;
                            //Get all the error messages: internal in algorithm and external in setup handler.
                            var errorMessage = String.Join(",", algorithm.ErrorMessages);
                            errorMessage += String.Join(",", SetupHandler.Errors);
                            throw new Exception(errorMessage);
                        }
                    }
                    catch (Exception err)
                    {
                        ResultHandler.RuntimeError("Algorithm.Initialize() Error: " + err.Message, err.StackTrace);
                    }

                    //-> Using the job + initialization: load the designated handlers:
                    if (initializeComplete)
                    {
                        //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
                        algorithm.SetAlgorithmId(job.AlgorithmId);
                        algorithm.SetLiveMode(LiveMode);
                        algorithm.SetLocked();

                        //Load the associated handlers for data, transaction and realtime events:
                        ResultHandler.SetAlgorithm(algorithm);
                        DataFeed           = GetDataFeedHandler(algorithm, job);
                        TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job);
                        RealTimeHandler    = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job);

                        //Set the error handlers for the brokerage asynchronous errors.
                        SetupHandler.SetupErrorHandler(ResultHandler, _brokerage);

                        //Send status to user the algorithm is now executing.
                        ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);

                        //Launch the data, transaction and realtime handlers into dedicated threads
                        threadFeed = new Thread(DataFeed.Run, 0)
                        {
                            Name = "DataFeed Thread"
                        };
                        threadTransactions = new Thread(TransactionHandler.Run, 0)
                        {
                            Name = "Transaction Thread"
                        };
                        threadRealTime = new Thread(RealTimeHandler.Run, 0)
                        {
                            Name = "RealTime Thread"
                        };

                        //Launch the data feed, result sending, and transaction models/handlers in separate threads.
                        threadFeed.Start();         // Data feed pushing data packets into thread bridge;
                        threadTransactions.Start(); // Transaction modeller scanning new order requests
                        threadRealTime.Start();     // RealTime scan time for time based events:
                                                    // Result manager scanning message queue: (started earlier)

                        try
                        {
                            // Execute the Algorithm Code:
                            var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, () =>
                            {
                                try
                                {
                                    //Run Algorithm Job:
                                    // -> Using this Data Feed,
                                    // -> Send Orders to this TransactionHandler,
                                    // -> Send Results to ResultHandler.
                                    AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler);
                                }
                                catch (Exception err)
                                {
                                    //Debugging at this level is difficult, stack trace needed.
                                    Log.Error("Engine.Run(): Error in Algo Manager: " + err.Message + " ST >> " + err.StackTrace);
                                }

                                Log.Trace("Engine.Run(): Exiting Algorithm Manager");
                            }, MaximumRamAllocation);

                            if (!complete)
                            {
                                Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F"));
                                throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster.");
                            }

                            // Algorithm runtime error:
                            if (AlgorithmManager.RunTimeError != null)
                            {
                                throw AlgorithmManager.RunTimeError;
                            }
                        }
                        catch (Exception err)
                        {
                            //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
                            Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
                            if (DataFeed != null)
                            {
                                DataFeed.Exit();
                            }
                            if (ResultHandler != null)
                            {
                                var runtimeMessage = "Runtime Error: " + err.Message + " Stack Trace: " + err.StackTrace;
                                ResultHandler.RuntimeError(runtimeMessage, err.StackTrace);
                                Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
                            }
                        }

                        //Send result data back: this entire code block could be rewritten.
                        // todo: - Split up statistics class, its enormous.
                        // todo: - Make a dedicated Statistics.Benchmark class.
                        // todo: - Elegently manage failure scenarios where no equity present.
                        // todo: - Move all creation and transmission of statistics out of primary engine loop.
                        // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);

                        try
                        {
                            var charts     = new Dictionary <string, Chart>(ResultHandler.Charts);
                            var orders     = new Dictionary <int, Order>(algorithm.Transactions.Orders);
                            var holdings   = new Dictionary <string, Holding>();
                            var statistics = new Dictionary <string, string>();
                            var banner     = new Dictionary <string, string>();

                            try
                            {
                                //Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
                                const string strategyEquityKey   = "Strategy Equity";
                                const string equityKey           = "Equity";
                                const string dailyPerformanceKey = "Daily Performance";

                                // make sure we've taken samples for these series before just blindly requesting them
                                if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
                                {
                                    var equity      = charts[strategyEquityKey].Series[equityKey].Values;
                                    var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
                                    var profitLoss  = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
                                    statistics = Statistics.Statistics.Generate(equity, profitLoss, performance, SetupHandler.StartingCapital, 252);
                                }
                            }
                            catch (Exception err) {
                                Log.Error("Algorithm.Node.Engine(): Error generating result packet: " + err.Message);
                            }

                            //Diagnostics Completed:
                            ResultHandler.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") completed analysis in " + timer.Elapsed.TotalSeconds.ToString("F2") + " seconds");

                            //Send the result packet:
                            ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
                        }
                        catch (Exception err)
                        {
                            Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + "  ST >> " + err.StackTrace);
                        }

                        //Before we return, send terminate commands to close up the threads
                        timer.Stop();               //Algorithm finished running.
                        TransactionHandler.Exit();
                        DataFeed.Exit();
                        RealTimeHandler.Exit();
                        AlgorithmManager.ResetManager();
                    }

                    //Close result handler:
                    ResultHandler.Exit();

                    //Wait for the threads to complete:
                    Log.Trace("Engine.Main(): Waiting for threads to deactivate...");
                    var ts = Stopwatch.StartNew();
                    while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 60 * 1000)
                    {
                        Thread.Sleep(100);
                        DataFeed.Exit();
                        Log.Trace("WAITING >> Result: " + ResultHandler.IsActive + " Transaction: " + TransactionHandler.IsActive + " DataFeed: " + DataFeed.IsActive + " RealTime: " + RealTimeHandler.IsActive);
                    }

                    Log.Trace("Engine.Main(): Closing Threads...");
                    if (threadFeed != null && threadFeed.IsAlive)
                    {
                        threadFeed.Abort();
                    }
                    if (threadTransactions != null && threadTransactions.IsAlive)
                    {
                        threadTransactions.Abort();
                    }
                    if (threadResults != null && threadResults.IsAlive)
                    {
                        threadResults.Abort();
                    }
                    Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
                }
                catch (Exception err)
                {
                    Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
                }
                finally
                {
                    //Delete the message from the queue before another worker picks it up:
                    Queue.AcknowledgeJob(job);
                    Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
                    GC.Collect();
                }

                //If we're running locally will execute just once.
            } while (!IsLocal);

            // Send the exit signal and then kill the thread
            StateCheck.Ping.Exit();

            // Make the console window pause so we can read log output before exiting and killing the application completely
            Console.ReadKey();

            //Finally if ping thread still not complete, kill.
            if (statusPingThread != null && statusPingThread.IsAlive)
            {
                statusPingThread.Abort();
            }
        }
Ejemplo n.º 14
0
        /********************************************************
         * CLASS METHODS
         *********************************************************/
        /// <summary>
        /// Launch the algorithm manager to run this strategy
        /// </summary>
        /// <param name="job">Algorithm job</param>
        /// <param name="algorithm">Algorithm instance</param>
        /// <param name="feed">Datafeed object</param>
        /// <param name="transactions">Transaction manager object</param>
        /// <param name="results">Result handler object</param>
        /// <param name="setup">Setup handler object</param>
        /// <param name="realtime">Realtime processing object</param>
        /// <remarks>Modify with caution</remarks>
        public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
        {
            //Initialize:
            var backwardsCompatibilityMode = false;
            var tradebarsType       = typeof(TradeBars);
            var ticksType           = typeof(Ticks);
            var startingPerformance = setup.StartingCapital;
            var backtestMode        = (job.Type == PacketType.BacktestNode);
            var methodInvokers      = new Dictionary <Type, MethodInvoker>();

            //Initialize Properties:
            _nextSample     = new DateTime();
            _frontier       = setup.StartingDate;
            _runtimeError   = null;
            _algorithmId    = job.AlgorithmId;
            _algorithmState = AlgorithmStatus.Running;
            _previousTime   = setup.StartingDate.Date;

            //Create the method accessors to push generic types into algorithm: Find all OnData events:

            //Algorithm 1.0 Data Accessors.
            //If the users defined these methods, add them in manually. This allows keeping backwards compatibility to algorithm 1.0.
            var oldTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnTradeBar", new[] { typeof(Dictionary <string, TradeBar>) });
            var oldTicksMethodInfo     = (algorithm.GetType()).GetMethod("OnTick", new[] { typeof(Dictionary <string, List <Tick> >) });

            //Algorithm 2.0 Data Generics Accessors.
            //New hidden access to tradebars with custom type.
            var newTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { tradebarsType });
            var newTicksMethodInfo     = (algorithm.GetType()).GetMethod("OnData", new[] { ticksType });

            if (newTradeBarsMethodInfo == null && newTicksMethodInfo == null)
            {
                backwardsCompatibilityMode = true;
                if (oldTradeBarsMethodInfo != null)
                {
                    methodInvokers.Add(tradebarsType, oldTradeBarsMethodInfo.DelegateForCallMethod());
                }
                if (oldTradeBarsMethodInfo != null)
                {
                    methodInvokers.Add(ticksType, oldTicksMethodInfo.DelegateForCallMethod());
                }
            }
            else
            {
                backwardsCompatibilityMode = false;
                if (newTradeBarsMethodInfo != null)
                {
                    methodInvokers.Add(tradebarsType, newTradeBarsMethodInfo.DelegateForCallMethod());
                }
                if (newTicksMethodInfo != null)
                {
                    methodInvokers.Add(ticksType, newTicksMethodInfo.DelegateForCallMethod());
                }
            }

            //Go through the subscription types and create invokers to trigger the event handlers for each custom type:
            foreach (var config in feed.Subscriptions)
            {
                //If type is a tradebar, combine tradebars and ticks into unified array:
                if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
                {
                    //Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
                    var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });

                    //Is we already have this Type-handler then don't add it to invokers again.
                    if (methodInvokers.ContainsKey(config.Type))
                    {
                        continue;
                    }

                    //If we couldnt find the event handler, let the user know we can't fire that event.
                    if (genericMethod == null)
                    {
                        _runtimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) {  }");
                        return;
                    }
                    methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
                }
            }

            //Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
            Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
            foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
            {
                //Check this backtest is still running:
                if (_algorithmState != AlgorithmStatus.Running)
                {
                    break;
                }

                //Go over each time stamp we've collected, pass it into the algorithm in order:
                foreach (var time in newData.Keys)
                {
                    //Set the time frontier:
                    _frontier = time;

                    //Execute with TimeLimit Monitor:
                    if (Isolator.IsCancellationRequested)
                    {
                        return;
                    }

                    //Refresh the realtime event monitor:
                    realtime.SetTime(time);

                    //Fire EOD if the time packet we just processed is greater
                    if (backtestMode && _previousTime.Date != time.Date)
                    {
                        //Sample the portfolio value over time for chart.
                        results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));

                        if (startingPerformance == 0)
                        {
                            results.SamplePerformance(_previousTime.Date, 0);
                        }
                        else
                        {
                            results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
                        }

                        startingPerformance = algorithm.Portfolio.TotalPortfolioValue;
                    }

                    //Check if the user's signalled Quit: loop over data until day changes.
                    if (algorithm.GetQuit())
                    {
                        _algorithmState = AlgorithmStatus.Quit;
                        break;
                    }

                    //Pass in the new time first:
                    algorithm.SetDateTime(time);

                    //Trigger the data events: Invoke the types we have data for:
                    var oldBars  = new Dictionary <string, TradeBar>();
                    var oldTicks = new Dictionary <string, List <Tick> >();
                    var newBars  = new TradeBars(time);
                    var newTicks = new Ticks(time);

                    //Invoke all non-tradebars, non-ticks methods:
                    // --> i == Subscription Configuration Index, so we don't need to compare types.
                    foreach (var i in newData[time].Keys)
                    {
                        //Data point and config of this point:
                        var dataPoints = newData[time][i];
                        var config     = feed.Subscriptions[i];

                        //Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
                        foreach (var dataPoint in dataPoints)
                        {
                            //Update the securities properties: first before calling user code to avoid issues with data
                            algorithm.Securities.Update(time, dataPoint);

                            //Update registered consolidators for this symbol index
                            for (var j = 0; j < config.Consolidators.Count; j++)
                            {
                                config.Consolidators[j].Update(dataPoint);
                            }

                            switch (config.Type.Name)
                            {
                            case "TradeBar":
                                var bar = dataPoint as TradeBar;
                                try
                                {
                                    if (bar != null)
                                    {
                                        if (backwardsCompatibilityMode)
                                        {
                                            if (!oldBars.ContainsKey(bar.Symbol))
                                            {
                                                oldBars.Add(bar.Symbol, bar);
                                            }
                                        }
                                        else
                                        {
                                            if (!newBars.ContainsKey(bar.Symbol))
                                            {
                                                newBars.Add(bar.Symbol, bar);
                                            }
                                        }
                                    }
                                }
                                catch (Exception err)
                                {
                                    Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> " + bar.Value.ToString("C"));
                                    Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString() + ") Count:(" + newBars.Count + ") " + err.Message);
                                }
                                break;

                            case "Tick":
                                var tick = dataPoint as Tick;
                                if (tick != null)
                                {
                                    if (backwardsCompatibilityMode)
                                    {
                                        if (!oldTicks.ContainsKey(tick.Symbol))
                                        {
                                            oldTicks.Add(tick.Symbol, new List <Tick>());
                                        }
                                        oldTicks[tick.Symbol].Add(tick);
                                    }
                                    else
                                    {
                                        if (!newTicks.ContainsKey(tick.Symbol))
                                        {
                                            newTicks.Add(tick.Symbol, new List <Tick>());
                                        }
                                        newTicks[tick.Symbol].Add(tick);
                                    }
                                }
                                break;

                            default:
                                //Send data into the generic algorithm event handlers
                                try
                                {
                                    methodInvokers[config.Type](algorithm, dataPoint);
                                }
                                catch (Exception err)
                                {
                                    _runtimeError = err;
                                    Log.Error("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
                                    return;
                                }
                                break;
                            }
                        }
                    }

                    //After we've fired all other events in this second, fire the pricing events:
                    if (backwardsCompatibilityMode)
                    {
                        //Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
                        try
                        {
                            if (oldTradeBarsMethodInfo != null && oldBars.Count > 0)
                            {
                                methodInvokers[tradebarsType](algorithm, oldBars);
                            }
                            if (oldTicksMethodInfo != null && oldTicks.Count > 0)
                            {
                                methodInvokers[ticksType](algorithm, oldTicks);
                            }
                        }
                        catch (Exception err)
                        {
                            _runtimeError = err;
                            Log.Error("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
                            return;
                        }
                    }
                    else
                    {
                        //Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
                        try
                        {
                            if (newTradeBarsMethodInfo != null && newBars.Count > 0)
                            {
                                methodInvokers[tradebarsType](algorithm, newBars);
                            }
                            if (newTicksMethodInfo != null && newTicks.Count > 0)
                            {
                                methodInvokers[ticksType](algorithm, newTicks);
                            }
                        }
                        catch (Exception err)
                        {
                            _runtimeError = err;
                            Log.Error("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
                            return;
                        }
                    }

                    //If its the historical/paper trading models, wait until its flagged as "ready"
                    if (job.TransactionEndpoint == TransactionHandlerEndpoint.Backtesting)
                    {
                        while (!transactions.Ready)
                        {
                            Thread.Yield();
                        }
                    }

                    if (time > _nextSample)
                    {
                        //Set next sample time: 4000 samples per backtest
                        _nextSample = time.Add(results.ResamplePeriod);

                        //Sample the portfolio value over time for chart.
                        results.SampleEquity(time, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));

                        //Also add the user samples / plots to the result handler tracking:
                        results.SampleRange(algorithm.GetChartUpdates());

                        //Sample the asset pricing:
                        foreach (var security in algorithm.Securities.Values)
                        {
                            results.SampleAssetPrices(security.Symbol, time, security.Price);
                        }
                    }

                    ProcessMessages(results, algorithm);

                    //Save the previous time for the sample calculations
                    _previousTime = time;
                } // End of Time Loop
            }     // End of ForEach DataStream

            //Stream over:: Send the final packet and fire final events:
            Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
            try
            {
                algorithm.OnEndOfAlgorithm();
            }
            catch (Exception err)
            {
                _runtimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
                return;
            }

            //Process the final messages from the algorithm
            ProcessMessages(results, algorithm);

            //Liquidate Holdings for Calculations:
            if (_algorithmState == AlgorithmStatus.Liquidated || Engine.IsLocal || job.TransactionEndpoint == TransactionHandlerEndpoint.Backtesting)
            {
                Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
                algorithm.Liquidate();
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
            }

            //Manually stopped the algorithm
            if (_algorithmState == AlgorithmStatus.Stopped)
            {
                Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
            }

            //Backtest deleted.
            if (_algorithmState == AlgorithmStatus.Deleted)
            {
                Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
                results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
            }

            //Algorithm finished, send regardless of commands:
            results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);

            //Take final samples:
            results.SampleRange(algorithm.GetChartUpdates());
            results.SampleEquity(_frontier, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
            results.SamplePerformance(_frontier, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
        } // End of Run();
Ejemplo n.º 15
0
        /********************************************************
        * CLASS METHODS
        *********************************************************/
        /// <summary>
        /// Primary Analysis Thread:
        /// </summary>
        public static void Main(string[] args)
        {
            //Initialize:
            var algorithmPath = "";
            string mode = "RELEASE";
            AlgorithmNodePacket job = null;
            var algorithm = default(IAlgorithm);
            var startTime = DateTime.Now;
            Log.LogHandler = Composer.Instance.GetExportedValueByTypeName<ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));

            #if DEBUG
                mode = "DEBUG";
            #endif

            //Name thread for the profiler:
            Thread.CurrentThread.Name = "Algorithm Analysis Thread";
            Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Constants.Version + " Mode: " + mode);
            Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
            Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App  " + +OS.TotalPhysicalMemoryUsed + "Mb-Used  " + OS.TotalPhysicalMemory + "Mb-Total");

            //Import external libraries specific to physical server location (cloud/local)
            try
            {
                // grab the right export based on configuration
                Api = Composer.Instance.GetExportedValueByTypeName<IApi>(Config.Get("api-handler"));
                Notify = Composer.Instance.GetExportedValueByTypeName<IMessagingHandler>(Config.Get("messaging-handler"));
                JobQueue = Composer.Instance.GetExportedValueByTypeName<IJobQueueHandler>(Config.Get("job-queue-handler"));
            }
            catch (CompositionException compositionException)
            { Log.Error("Engine.Main(): Failed to load library: " + compositionException);
            }

            //Setup packeting, queue and controls system: These don't do much locally.
            Api.Initialize();
            Notify.Initialize();
            JobQueue.Initialize();

            //Start monitoring the backtest active status:
            var statusPingThread = new Thread(StateCheck.Ping.Run);
            statusPingThread.Start();

            try
            {
                //Reset algo manager internal variables preparing for a new algorithm.
                AlgorithmManager.ResetManager();

                //Reset thread holders.
                var initializeComplete = false;
                Thread threadFeed = null;
                Thread threadTransactions = null;
                Thread threadResults = null;
                Thread threadRealTime = null;

                do
                {
                    //-> Pull job from QuantConnect job queue, or, pull local build:
                    job = JobQueue.NextJob(out algorithmPath); // Blocking.

                    // if the job version doesn't match this instance version then we can't process it
                    // we also don't want to reprocess redelivered live jobs
                    if (job.Version != Constants.Version || (LiveMode && job.Redelivered))
                    {
                        Log.Error("Engine.Run(): Job Version: " + job.Version + "  Deployed Version: " + Constants.Version);

                        //Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
                        //In this event kill the old algorithm and leave a message so the user can later review.
                        JobQueue.AcknowledgeJob(job);
                        Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
                        Notify.SetChannel(job.Channel);
                        Notify.RuntimeError(job.AlgorithmId, _collapseMessage);
                        job = null;
                    }
                } while (job == null);

                //-> Initialize messaging system
                Notify.SetChannel(job.Channel);

                //-> Create SetupHandler to configure internal algorithm state:
                SetupHandler = GetSetupHandler(job.SetupEndpoint);

                //-> Set the result handler type for this algorithm job, and launch the associated result thread.
                ResultHandler = GetResultHandler(job);
                threadResults = new Thread(ResultHandler.Run, 0) {Name = "Result Thread"};
                threadResults.Start();

                try
                {
                    // Save algorithm to cache, load algorithm instance:
                    algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath);

                    //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
                    initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job);

                    //If there are any reasons it failed, pass these back to the IDE.
                    if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0)
                    {
                        initializeComplete = false;
                        //Get all the error messages: internal in algorithm and external in setup handler.
                        var errorMessage = String.Join(",", algorithm.ErrorMessages);
                        errorMessage += String.Join(",", SetupHandler.Errors);
                        ResultHandler.RuntimeError(errorMessage);
                        Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError);
                    }
                }
                catch (Exception err)
                {
                    var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
                    ResultHandler.RuntimeError(runtimeMessage, err.StackTrace);
                    Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
                }

                //-> Using the job + initialization: load the designated handlers:
                if (initializeComplete)
                {
                    //-> Reset the backtest stopwatch; we're now running the algorithm.
                    startTime = DateTime.Now;

                    //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
                    algorithm.SetAlgorithmId(job.AlgorithmId);
                    algorithm.SetLiveMode(LiveMode);
                    algorithm.SetLocked();

                    //Load the associated handlers for data, transaction and realtime events:
                    ResultHandler.SetAlgorithm(algorithm);
                    DataFeed            = GetDataFeedHandler(algorithm, job);
                    TransactionHandler  = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job);
                    RealTimeHandler     = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job);

                    //Set the error handlers for the brokerage asynchronous errors.
                    SetupHandler.SetupErrorHandler(ResultHandler, _brokerage);

                    //Send status to user the algorithm is now executing.
                    ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);

                    //Launch the data, transaction and realtime handlers into dedicated threads
                    threadFeed = new Thread(DataFeed.Run) {Name = "DataFeed Thread"};
                    threadTransactions = new Thread(TransactionHandler.Run) {Name = "Transaction Thread"};
                    threadRealTime = new Thread(RealTimeHandler.Run) {Name = "RealTime Thread"};

                    //Launch the data feed, result sending, and transaction models/handlers in separate threads.
                    threadFeed.Start(); // Data feed pushing data packets into thread bridge;
                    threadTransactions.Start(); // Transaction modeller scanning new order requests
                    threadRealTime.Start(); // RealTime scan time for time based events:

                    // Result manager scanning message queue: (started earlier)
                    ResultHandler.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version));

                    try
                    {
                        // Execute the Algorithm Code:
                        var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, AlgorithmManager.TimeLoopWithinLimits, () =>
                        {
                            try
                            {
                                //Run Algorithm Job:
                                // -> Using this Data Feed,
                                // -> Send Orders to this TransactionHandler,
                                // -> Send Results to ResultHandler.
                                AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler);
                            }
                            catch (Exception err)
                            {
                                //Debugging at this level is difficult, stack trace needed.
                                Log.Error("Engine.Run", err);
                            }

                            Log.Trace("Engine.Run(): Exiting Algorithm Manager");

                            }, job.UserPlan == UserPlan.Free ? 1024 : MaximumRamAllocation);

                        if (!complete)
                        {
                            Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F"));
                            throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster.");
                        }

                        // Algorithm runtime error:
                        if (algorithm.RunTimeError != null)
                        {
                            throw algorithm.RunTimeError;
                        }
                    }
                    catch (Exception err)
                    {
                        //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
                        Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
                        if (DataFeed != null) DataFeed.Exit();
                        if (ResultHandler != null)
                        {
                            var message = "Runtime Error: " + err.Message;
                            Log.Trace("Engine.Run(): Sending runtime error to user...");
                            ResultHandler.LogMessage(message);
                            ResultHandler.RuntimeError(message, err.StackTrace);
                            Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
                        }
                    }

                    //Send result data back: this entire code block could be rewritten.
                    // todo: - Split up statistics class, its enormous.
                    // todo: - Make a dedicated Statistics.Benchmark class.
                    // todo: - Move all creation and transmission of statistics out of primary engine loop.
                    // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);

                    try
                    {
                        var charts = new Dictionary<string, Chart>(ResultHandler.Charts);
                        var orders = new Dictionary<int, Order>(algorithm.Transactions.Orders);
                        var holdings = new Dictionary<string, Holding>();
                        var statistics = new Dictionary<string, string>();
                        var banner = new Dictionary<string, string>();

                        try
                        {
                            //Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
                            const string strategyEquityKey = "Strategy Equity";
                            const string equityKey = "Equity";
                            const string dailyPerformanceKey = "Daily Performance";

                            // make sure we've taken samples for these series before just blindly requesting them
                            if (charts.ContainsKey(strategyEquityKey) &&
                                charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
                                charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
                            {
                                var equity = charts[strategyEquityKey].Series[equityKey].Values;
                                var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
                                var profitLoss =
                                    new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord);
                                statistics = Statistics.Statistics.Generate(equity, profitLoss, performance,
                                    SetupHandler.StartingPortfolioValue, algorithm.Portfolio.TotalFees, 252);
                            }
                        }
                        catch (Exception err)
                        {
                            Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
                        }

                        //Diagnostics Completed, Send Result Packet:
                        var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
                        ResultHandler.DebugMessage(string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
                            job.AlgorithmId, totalSeconds.ToString("F2"), ((AlgorithmManager.DataPoints / (double)1000) / totalSeconds).ToString("F0"), AlgorithmManager.DataPoints.ToString("N0")));

                        ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
                    }
                    catch (Exception err)
                    {
                        Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + "  ST >> " + err.StackTrace);
                    }

                    //Before we return, send terminate commands to close up the threads
                    TransactionHandler.Exit();
                    DataFeed.Exit();
                    RealTimeHandler.Exit();
                }

                //Close result handler:
                ResultHandler.Exit();
                StateCheck.Ping.Exit();

                //Wait for the threads to complete:
                var ts = Stopwatch.StartNew();
                while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000)
                {
                    Thread.Sleep(100); Log.Trace("Waiting for threads to exit...");
                }

                //Terminate threads still in active state.
                if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort();
                if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort();
                if (threadResults != null && threadResults.IsAlive) threadResults.Abort();
                if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort();

                if (_brokerage != null)
                {
                    _brokerage.Disconnect();
                }
                if (SetupHandler != null)
                {
                    SetupHandler.Dispose();
                }
                Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
            }
            catch (Exception err)
            {
                Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
            }
            finally
            {
                //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
                if (LiveMode && AlgorithmManager.State != AlgorithmStatus.Running && AlgorithmManager.State != AlgorithmStatus.RuntimeError)
                    Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmManager.State);

                //Delete the message from the job queue:
                JobQueue.AcknowledgeJob(job);
                Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);

                //Attempt to clean up ram usage:
                GC.Collect();
            }

            //Final disposals.
            Api.Dispose();

            // Make the console window pause so we can read log output before exiting and killing the application completely
            if (IsLocal)
            {
                Log.Trace("Engine.Main(): Analysis Complete. Press any key to continue.");
                Console.Read();
            }
            Log.LogHandler.Dispose();
        }
Ejemplo n.º 16
0
 public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
 {
     _shadow.Initialize(job, messagingHandler, api, dataFeed, setupHandler, transactionHandler);
 }
Ejemplo n.º 17
0
        /// <summary>
        /// Initialize the result handler with this result packet.
        /// </summary>
        /// <param name="packet">Algorithm job packet for this result handler</param>
        /// <param name="messagingHandler"></param>
        /// <param name="api"></param>
        /// <param name="dataFeed"></param>
        /// <param name="setupHandler"></param>
        /// <param name="transactionHandler"></param>
        public void Initialize(AlgorithmNodePacket packet, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
        {
            // we expect one of two types here, the backtest node packet or the live node packet
            var job = packet as BacktestNodePacket;
            if (job != null)
            {
                _algorithmNode = new BacktestConsoleStatusHandler(job);
            }
            else
            {
                var live = packet as LiveNodePacket;
                if (live == null)
                {
                    throw new ArgumentException("Unexpected AlgorithmNodeType: " + packet.GetType().Name);
                }
                _algorithmNode = new LiveConsoleStatusHandler(live);
            }
            _resamplePeriod = _algorithmNode.ComputeSampleEquityPeriod();

            var time = DateTime.Now.ToString("yyyy-MM-dd-HH-mm");
            _chartDirectory = Path.Combine("../../../Charts/", packet.AlgorithmId, time);
            if (Directory.Exists(_chartDirectory))
            {
                foreach (var file in Directory.EnumerateFiles(_chartDirectory, "*.csv", SearchOption.AllDirectories))
                {
                    File.Delete(file);
                }
                Directory.Delete(_chartDirectory, true);
            }
            Directory.CreateDirectory(_chartDirectory);
            _messagingHandler = messagingHandler; 

        }
Ejemplo n.º 18
0
        /// <summary>
        /// Initialize the result handler with this result packet.
        /// </summary>
        /// <param name="packet">Algorithm job packet for this result handler</param>
        /// <param name="messagingHandler"></param>
        /// <param name="api"></param>
        /// <param name="dataFeed"></param>
        /// <param name="setupHandler"></param>
        /// <param name="transactionHandler"></param>
        public void Initialize(AlgorithmNodePacket packet, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
        {
            // we expect one of two types here, the backtest node packet or the live node packet
            var job = packet as BacktestNodePacket;

            if (job != null)
            {
                _algorithmNode = new BacktestConsoleStatusHandler(job);
            }
            else
            {
                var live = packet as LiveNodePacket;
                if (live == null)
                {
                    throw new ArgumentException("Unexpected AlgorithmNodeType: " + packet.GetType().Name);
                }
                _algorithmNode = new LiveConsoleStatusHandler(live);
            }
            _resamplePeriod = _algorithmNode.ComputeSampleEquityPeriod();

            var time = DateTime.Now.ToString("yyyy-MM-dd-HH-mm");

            _chartDirectory = Path.Combine("../../../Charts/", packet.AlgorithmId, time);
            if (Directory.Exists(_chartDirectory))
            {
                foreach (var file in Directory.EnumerateFiles(_chartDirectory, "*.csv", SearchOption.AllDirectories))
                {
                    File.Delete(file);
                }
                Directory.Delete(_chartDirectory, true);
            }
            Directory.CreateDirectory(_chartDirectory);
            _messagingHandler = messagingHandler;
        }
Ejemplo n.º 19
0
        /********************************************************
        * CLASS METHODS
        *********************************************************/
        /// <summary>
        /// Launch the algorithm manager to run this strategy
        /// </summary>
        /// <param name="job">Algorithm job</param>
        /// <param name="algorithm">Algorithm instance</param>
        /// <param name="feed">Datafeed object</param>
        /// <param name="transactions">Transaction manager object</param>
        /// <param name="results">Result handler object</param>
        /// <param name="setup">Setup handler object</param>
        /// <param name="realtime">Realtime processing object</param>
        /// <remarks>Modify with caution</remarks>
        public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
        {
            //Initialize:
            var backwardsCompatibilityMode = false;
            var tradebarsType = typeof (TradeBars);
            var ticksType = typeof(Ticks);
            var startingPerformance = setup.StartingCapital;
            var backtestMode = (job.Type == PacketType.BacktestNode);
            var methodInvokers = new Dictionary<Type, MethodInvoker>();

            //Initialize Properties:
            _frontier = setup.StartingDate;
            _runtimeError = null;
            _algorithmId = job.AlgorithmId;
            _algorithmState = AlgorithmStatus.Running;
            _previousTime = setup.StartingDate.Date;

            //Create the method accessors to push generic types into algorithm: Find all OnData events:

            //Algorithm 1.0 Data Accessors.
            //If the users defined these methods, add them in manually. This allows keeping backwards compatibility to algorithm 1.0.
            var oldTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnTradeBar",   new[] { typeof(Dictionary<string, TradeBar>) });
            var oldTicksMethodInfo = (algorithm.GetType()).GetMethod("OnTick", new[] { typeof(Dictionary<string, List<Tick>>) });

            //Algorithm 2.0 Data Generics Accessors.
            //New hidden access to tradebars with custom type.
            var newTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { tradebarsType });
            var newTicksMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { ticksType });

            if (newTradeBarsMethodInfo == null && newTicksMethodInfo == null)
            {
                backwardsCompatibilityMode = true;
                if (oldTradeBarsMethodInfo != null) methodInvokers.Add(tradebarsType, oldTradeBarsMethodInfo.DelegateForCallMethod());
                if (oldTradeBarsMethodInfo != null) methodInvokers.Add(ticksType, oldTicksMethodInfo.DelegateForCallMethod());
            }
            else
            {
                backwardsCompatibilityMode = false;
                if (newTradeBarsMethodInfo != null) methodInvokers.Add(tradebarsType, newTradeBarsMethodInfo.DelegateForCallMethod());
                if (newTicksMethodInfo != null) methodInvokers.Add(ticksType, newTicksMethodInfo.DelegateForCallMethod());
            }

            //Go through the subscription types and create invokers to trigger the event handlers for each custom type:
            foreach (var config in feed.Subscriptions)
            {
                //If type is a tradebar, combine tradebars and ticks into unified array:
                if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
                {
                    //Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
                    var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });

                    //Is we already have this Type-handler then don't add it to invokers again.
                    if (methodInvokers.ContainsKey(config.Type)) continue;

                    //If we couldnt find the event handler, let the user know we can't fire that event.
                    if (genericMethod == null)
                    {
                        _runtimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) {  }");
                        _algorithmState = AlgorithmStatus.RuntimeError;
                        return;
                    }
                    methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
                }
            }

            //Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
            Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
            foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
            {
                //Check this backtest is still running:
                if (_algorithmState != AlgorithmStatus.Running) break;

                //Go over each time stamp we've collected, pass it into the algorithm in order:
                foreach (var time in newData.Keys)
                {
                    //Set the time frontier:
                    _frontier = time;

                    //Execute with TimeLimit Monitor:
                    if (Isolator.IsCancellationRequested) return;

                    //Refresh the realtime event monitor:
                    realtime.SetTime(time);

                    //Fire EOD if the time packet we just processed is greater
                    if (backtestMode && _previousTime.Date != time.Date)
                    {
                        //Sample the portfolio value over time for chart.
                        results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));

                        if (startingPerformance == 0)
                        {
                            results.SamplePerformance(_previousTime.Date, 0);
                        }
                        else
                        {
                            results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
                        }

                        startingPerformance = algorithm.Portfolio.TotalPortfolioValue;
                    }

                    //Check if the user's signalled Quit: loop over data until day changes.
                    if (algorithm.GetQuit())
                    {
                        _algorithmState = AlgorithmStatus.Quit;
                        break;
                    }

                    //Pass in the new time first:
                    algorithm.SetDateTime(time);

                    //Trigger the data events: Invoke the types we have data for:
                    var oldBars = new Dictionary<string, TradeBar>();
                    var oldTicks = new Dictionary<string, List<Tick>>();
                    var newBars = new TradeBars(time);
                    var newTicks = new Ticks(time);

                    //Invoke all non-tradebars, non-ticks methods:
                    // --> i == Subscription Configuration Index, so we don't need to compare types.
                    foreach (var i in newData[time].Keys)
                    {
                        //Data point and config of this point:
                        var dataPoints = newData[time][i];
                        var config = feed.Subscriptions[i];

                        //Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
                        foreach (var dataPoint in dataPoints)
                        {
                            //Update the securities properties: first before calling user code to avoid issues with data
                            algorithm.Securities.Update(time, dataPoint);

                            //Update registered consolidators for this symbol index
                            for (var j = 0; j < config.Consolidators.Count; j++)
                            {
                                config.Consolidators[j].Update(dataPoint);
                            }

                            switch (config.Type.Name)
                            {
                                case "TradeBar":
                                    var bar = dataPoint as TradeBar;
                                    try
                                    {
                                        if (bar != null)
                                        {
                                            if (backwardsCompatibilityMode)
                                            {
                                                if (!oldBars.ContainsKey(bar.Symbol)) oldBars.Add(bar.Symbol, bar);
                                            }
                                            else
                                            {
                                                if (!newBars.ContainsKey(bar.Symbol)) newBars.Add(bar.Symbol, bar);
                                            }
                                        }
                                    }
                                    catch (Exception err)
                                    {
                                        Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> " + bar.Value.ToString("C"));
                                        Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString() + ") Count:(" + newBars.Count + ") " + err.Message);
                                    }
                                    break;

                                case "Tick":
                                    var tick = dataPoint as Tick;
                                    if (tick != null)
                                    {
                                         if (backwardsCompatibilityMode) {
                                             if (!oldTicks.ContainsKey(tick.Symbol)) { oldTicks.Add(tick.Symbol, new List<Tick>()); }
                                             oldTicks[tick.Symbol].Add(tick);
                                         }
                                         else
                                         {
                                             if (!newTicks.ContainsKey(tick.Symbol)) { newTicks.Add(tick.Symbol, new List<Tick>()); }
                                             newTicks[tick.Symbol].Add(tick);
                                         }
                                    }
                                    break;

                                default:
                                    //Send data into the generic algorithm event handlers
                                    try
                                    {
                                        methodInvokers[config.Type](algorithm, dataPoint);
                                    }
                                    catch (Exception err)
                                    {
                                        _runtimeError = err;
                                        _algorithmState = AlgorithmStatus.RuntimeError;
                                        Log.Debug("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
                                        return;
                                    }
                                    break;
                            }
                        }
                    }

                    //After we've fired all other events in this second, fire the pricing events:
                    if (backwardsCompatibilityMode)
                    {
                        //Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
                        try
                        {
                            if (oldTradeBarsMethodInfo != null && oldBars.Count > 0) methodInvokers[tradebarsType](algorithm, oldBars);
                            if (oldTicksMethodInfo != null && oldTicks.Count > 0) methodInvokers[ticksType](algorithm, oldTicks);
                        }
                        catch (Exception err)
                        {
                            _runtimeError = err;
                            _algorithmState = AlgorithmStatus.RuntimeError;
                            Log.Debug("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
                            return;
                        }
                    }
                    else
                    {
                        //Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
                        try
                        {
                            if (newTradeBarsMethodInfo != null && newBars.Count > 0) methodInvokers[tradebarsType](algorithm, newBars);
                            if (newTicksMethodInfo != null && newTicks.Count > 0) methodInvokers[ticksType](algorithm, newTicks);
                        }
                        catch (Exception err)
                        {
                            _runtimeError = err;
                            _algorithmState = AlgorithmStatus.RuntimeError;
                            Log.Debug("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
                            return;
                        }
                    }

                    //If its the historical/paper trading models, wait until market orders have been "filled"
                    // Manually trigger the event handler to prevent thread switch.
                    transactions.ProcessSynchronousEvents();

                    //Save the previous time for the sample calculations
                    _previousTime = time;

                } // End of Time Loop

                // Process any required events of the results handler such as sampling assets, equity, or stock prices.
                results.ProcessSynchronousEvents();
            } // End of ForEach DataStream

            //Stream over:: Send the final packet and fire final events:
            Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
            try
            {
                algorithm.OnEndOfAlgorithm();
            }
            catch (Exception err)
            {
                _algorithmState = AlgorithmStatus.RuntimeError;
                _runtimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
                Log.Debug("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
                return;
            }

            // Process any required events of the results handler such as sampling assets, equity, or stock prices.
            results.ProcessSynchronousEvents(forceProcess: true);

            //Liquidate Holdings for Calculations:
            if (_algorithmState == AlgorithmStatus.Liquidated || !Engine.LiveMode)
            {
                Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
                algorithm.Liquidate();
                results.LogMessage("Algorithm Liquidated");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
            }

            //Manually stopped the algorithm
            if (_algorithmState == AlgorithmStatus.Stopped)
            {
                Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
                results.LogMessage("Algorithm Stopped");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
            }

            //Backtest deleted.
            if (_algorithmState == AlgorithmStatus.Deleted)
            {
                Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
                results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
            }

            //Algorithm finished, send regardless of commands:
            results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);

            //Take final samples:
            results.SampleRange(algorithm.GetChartUpdates());
            results.SampleEquity(_frontier, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
            results.SamplePerformance(_frontier, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
        }
Ejemplo n.º 20
0
        /********************************************************
         * CLASS METHODS
         *********************************************************/
        /// <summary>
        /// Launch the algorithm manager to run this strategy
        /// </summary>
        /// <param name="job">Algorithm job</param>
        /// <param name="algorithm">Algorithm instance</param>
        /// <param name="feed">Datafeed object</param>
        /// <param name="transactions">Transaction manager object</param>
        /// <param name="results">Result handler object</param>
        /// <param name="setup">Setup handler object</param>
        /// <param name="realtime">Realtime processing object</param>
        /// <remarks>Modify with caution</remarks>
        public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
        {
            //Initialize:
            _dataPointCount = 0;
            var startingPortfolioValue = setup.StartingPortfolioValue;
            var backtestMode           = (job.Type == PacketType.BacktestNode);
            var methodInvokers         = new Dictionary <Type, MethodInvoker>();
            var marginCallFrequency    = TimeSpan.FromMinutes(5);
            var nextMarginCallTime     = DateTime.MinValue;

            //Initialize Properties:
            _algorithmId    = job.AlgorithmId;
            _algorithmState = AlgorithmStatus.Running;
            _previousTime   = setup.StartingDate.Date;

            //Create the method accessors to push generic types into algorithm: Find all OnData events:

            // Algorithm 1.0 data accessors
            var hasOnTradeBar = AddMethodInvoker <Dictionary <string, TradeBar> >(algorithm, methodInvokers, "OnTradeBar");
            var hasOnTick     = AddMethodInvoker <Dictionary <string, List <Tick> > >(algorithm, methodInvokers, "OnTick");

            // Algorithm 2.0 data accessors
            var hasOnDataTradeBars = AddMethodInvoker <TradeBars>(algorithm, methodInvokers);
            var hasOnDataTicks     = AddMethodInvoker <Ticks>(algorithm, methodInvokers);

            // determine what mode we're in
            var backwardsCompatibilityMode = !hasOnDataTradeBars && !hasOnDataTicks;

            // dividend and split events
            var hasOnDataDividends = AddMethodInvoker <Dividends>(algorithm, methodInvokers);
            var hasOnDataSplits    = AddMethodInvoker <Splits>(algorithm, methodInvokers);

            //Go through the subscription types and create invokers to trigger the event handlers for each custom type:
            foreach (var config in feed.Subscriptions)
            {
                //If type is a tradebar, combine tradebars and ticks into unified array:
                if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
                {
                    //Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
                    var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });

                    //If we already have this Type-handler then don't add it to invokers again.
                    if (methodInvokers.ContainsKey(config.Type))
                    {
                        continue;
                    }

                    //If we couldnt find the event handler, let the user know we can't fire that event.
                    if (genericMethod == null)
                    {
                        algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) {  }");
                        _algorithmState        = AlgorithmStatus.RuntimeError;
                        return;
                    }
                    methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
                }
            }

            //Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
            Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
            foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
            {
                // reset our timer on each loop
                _currentTimeStepStopwatch = Stopwatch.StartNew();

                //Check this backtest is still running:
                if (_algorithmState != AlgorithmStatus.Running)
                {
                    break;
                }

                //Execute with TimeLimit Monitor:
                if (Isolator.IsCancellationRequested)
                {
                    return;
                }

                var time = DataStream.AlorithmTime;

                //If we're in backtest mode we need to capture the daily performance. We do this here directly
                //before updating the algorithm state with the new data from this time step, otherwise we'll
                //produce incorrect samples (they'll take into account this time step's new price values)
                if (backtestMode)
                {
                    //Refresh the realtime event monitor:
                    //in backtest mode use the algorithms clock as realtime.
                    realtime.SetTime(time);

                    //On day-change sample equity and daily performance for statistics calculations
                    if (_previousTime.Date != time.Date)
                    {
                        //Sample the portfolio value over time for chart.
                        results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));

                        //Check for divide by zero
                        if (startingPortfolioValue == 0m)
                        {
                            results.SamplePerformance(_previousTime.Date, 0);
                        }
                        else
                        {
                            results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
                        }
                        startingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
                    }
                }

                //Update algorithm state after capturing performance from previous day

                //On each time step push the real time prices to the cashbook so we can have updated conversion rates
                algorithm.Portfolio.CashBook.Update(newData);

                //Update the securities properties: first before calling user code to avoid issues with data
                algorithm.Securities.Update(time, newData);

                // perform margin calls, in live mode we can also use realtime to emit these
                if (time >= nextMarginCallTime || (Engine.LiveMode && nextMarginCallTime > DateTime.Now))
                {
                    // determine if there are possible margin call orders to be executed
                    bool issueMarginCallWarning;
                    var  marginCallOrders = algorithm.Portfolio.ScanForMarginCall(out issueMarginCallWarning);
                    if (marginCallOrders.Count != 0)
                    {
                        try
                        {
                            // tell the algorithm we're about to issue the margin call
                            algorithm.OnMarginCall(marginCallOrders);
                        }
                        catch (Exception err)
                        {
                            algorithm.RunTimeError = err;
                            _algorithmState        = AlgorithmStatus.RuntimeError;
                            Log.Debug("AlgorithmManager.Run(): RuntimeError: OnMarginCall: " + err.Message + " STACK >>> " + err.StackTrace);
                            return;
                        }

                        // execute the margin call orders
                        var executedOrders = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders);
                        foreach (var order in executedOrders)
                        {
                            algorithm.Error(string.Format("{0} - Executed MarginCallOrder: {1} - Quantity: {2} @ {3}", algorithm.Time, order.Symbol, order.Quantity, order.Price));
                        }
                    }
                    // we didn't perform a margin call, but got the warning flag back, so issue the warning to the algorithm
                    else if (issueMarginCallWarning)
                    {
                        try
                        {
                            algorithm.OnMarginCallWarning();
                        }
                        catch (Exception err)
                        {
                            algorithm.RunTimeError = err;
                            _algorithmState        = AlgorithmStatus.RuntimeError;
                            Log.Debug("AlgorithmManager.Run(): RuntimeError: OnMarginCallWarning: " + err.Message + " STACK >>> " + err.StackTrace);
                        }
                    }

                    nextMarginCallTime = time + marginCallFrequency;
                }


                //Check if the user's signalled Quit: loop over data until day changes.
                if (algorithm.GetQuit())
                {
                    _algorithmState = AlgorithmStatus.Quit;
                    break;
                }
                if (algorithm.RunTimeError != null)
                {
                    _algorithmState = AlgorithmStatus.RuntimeError;
                    break;
                }

                //Pass in the new time first:
                algorithm.SetDateTime(time);

                //Trigger the data events: Invoke the types we have data for:
                var oldBars      = new Dictionary <string, TradeBar>();
                var oldTicks     = new Dictionary <string, List <Tick> >();
                var newBars      = new TradeBars(time);
                var newTicks     = new Ticks(time);
                var newDividends = new Dividends(time);
                var newSplits    = new Splits(time);

                //Invoke all non-tradebars, non-ticks methods and build up the TradeBars and Ticks dictionaries
                // --> i == Subscription Configuration Index, so we don't need to compare types.
                foreach (var i in newData.Keys)
                {
                    //Data point and config of this point:
                    var dataPoints = newData[i];
                    var config     = feed.Subscriptions[i];

                    //Keep track of how many data points we've processed
                    _dataPointCount += dataPoints.Count;

                    //We don't want to pump data that we added just for currency conversions
                    if (config.IsInternalFeed)
                    {
                        continue;
                    }

                    //Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
                    //  Aggregate Dividends and Splits -- invoke portfolio application methods
                    foreach (var dataPoint in dataPoints)
                    {
                        var dividend = dataPoint as Dividend;
                        if (dividend != null)
                        {
                            Log.Trace("AlgorithmManager.Run(): Applying Dividend for " + dividend.Symbol);
                            // if this is a dividend apply to portfolio
                            algorithm.Portfolio.ApplyDividend(dividend);
                            if (hasOnDataDividends)
                            {
                                // and add to our data dictionary to pump into OnData(Dividends data)
                                newDividends.Add(dividend);
                            }
                            continue;
                        }

                        var split = dataPoint as Split;
                        if (split != null)
                        {
                            Log.Trace("AlgorithmManager.Run(): Applying Split for " + split.Symbol);

                            // if this is a split apply to portfolio
                            algorithm.Portfolio.ApplySplit(split);
                            if (hasOnDataSplits)
                            {
                                // and add to our data dictionary to pump into OnData(Splits data)
                                newSplits.Add(split);
                            }
                            continue;
                        }

                        //Update registered consolidators for this symbol index
                        try
                        {
                            for (var j = 0; j < config.Consolidators.Count; j++)
                            {
                                config.Consolidators[j].Update(dataPoint);
                            }
                        }
                        catch (Exception err)
                        {
                            algorithm.RunTimeError = err;
                            _algorithmState        = AlgorithmStatus.RuntimeError;
                            Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err.Message);
                            return;
                        }

                        // TRADEBAR -- add to our dictionary
                        var bar = dataPoint as TradeBar;
                        if (bar != null)
                        {
                            try
                            {
                                if (backwardsCompatibilityMode)
                                {
                                    oldBars[bar.Symbol] = bar;
                                }
                                else
                                {
                                    newBars[bar.Symbol] = bar;
                                }
                            }
                            catch (Exception err)
                            {
                                Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> "
                                          + bar.Value.ToString("C"));
                                Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString()
                                          + ") Count:(" + newBars.Count + ") " + err.Message);
                            }
                            continue;
                        }
                        // TICK -- add to our dictionary
                        var tick = dataPoint as Tick;
                        if (tick != null)
                        {
                            if (backwardsCompatibilityMode)
                            {
                                List <Tick> ticks;
                                if (!oldTicks.TryGetValue(tick.Symbol, out ticks))
                                {
                                    ticks = new List <Tick>(3);
                                    oldTicks.Add(tick.Symbol, ticks);
                                }
                                ticks.Add(tick);
                            }
                            else
                            {
                                List <Tick> ticks;
                                if (!newTicks.TryGetValue(tick.Symbol, out ticks))
                                {
                                    ticks = new List <Tick>(3);
                                    newTicks.Add(tick.Symbol, ticks);
                                }
                                ticks.Add(tick);
                            }
                            continue;
                        }

                        // if it was nothing else then it must be custom data

                        // CUSTOM DATA -- invoke on data method
                        //Send data into the generic algorithm event handlers
                        try
                        {
                            methodInvokers[config.Type](algorithm, dataPoint);
                        }
                        catch (Exception err)
                        {
                            algorithm.RunTimeError = err;
                            _algorithmState        = AlgorithmStatus.RuntimeError;
                            Log.Debug("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
                            return;
                        }
                    }
                }

                try
                {
                    // fire off the dividend and split events before pricing events
                    if (hasOnDataDividends && newDividends.Count != 0)
                    {
                        methodInvokers[typeof(Dividends)](algorithm, newDividends);
                    }
                    if (hasOnDataSplits && newSplits.Count != 0)
                    {
                        methodInvokers[typeof(Splits)](algorithm, newSplits);
                    }
                }
                catch (Exception err)
                {
                    algorithm.RunTimeError = err;
                    _algorithmState        = AlgorithmStatus.RuntimeError;
                    Log.Debug("AlgorithmManager.Run(): RuntimeError: Dividends/Splits: " + err.Message + " STACK >>> " + err.StackTrace);
                    return;
                }

                //After we've fired all other events in this second, fire the pricing events:
                if (backwardsCompatibilityMode)
                {
                    //Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
                    try
                    {
                        if (hasOnTradeBar && oldBars.Count > 0)
                        {
                            methodInvokers[typeof(Dictionary <string, TradeBar>)](algorithm, oldBars);
                        }
                        if (hasOnTick && oldTicks.Count > 0)
                        {
                            methodInvokers[typeof(Dictionary <string, List <Tick> >)](algorithm, oldTicks);
                        }
                    }
                    catch (Exception err)
                    {
                        algorithm.RunTimeError = err;
                        _algorithmState        = AlgorithmStatus.RuntimeError;
                        Log.Debug("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
                        return;
                    }
                }
                else
                {
                    //Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
                    try
                    {
                        if (hasOnDataTradeBars && newBars.Count > 0)
                        {
                            methodInvokers[typeof(TradeBars)](algorithm, newBars);
                        }
                        if (hasOnDataTicks && newTicks.Count > 0)
                        {
                            methodInvokers[typeof(Ticks)](algorithm, newTicks);
                        }
                    }
                    catch (Exception err)
                    {
                        algorithm.RunTimeError = err;
                        _algorithmState        = AlgorithmStatus.RuntimeError;
                        Log.Debug("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
                        return;
                    }
                }

                //If its the historical/paper trading models, wait until market orders have been "filled"
                // Manually trigger the event handler to prevent thread switch.
                transactions.ProcessSynchronousEvents();

                //Save the previous time for the sample calculations
                _previousTime = time;

                // Process any required events of the results handler such as sampling assets, equity, or stock prices.
                results.ProcessSynchronousEvents();
            } // End of ForEach DataStream

            //Stream over:: Send the final packet and fire final events:
            Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
            try
            {
                algorithm.OnEndOfAlgorithm();
            }
            catch (Exception err)
            {
                _algorithmState        = AlgorithmStatus.RuntimeError;
                algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
                Log.Debug("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
                return;
            }

            // Process any required events of the results handler such as sampling assets, equity, or stock prices.
            results.ProcessSynchronousEvents(forceProcess: true);

            //Liquidate Holdings for Calculations:
            if (_algorithmState == AlgorithmStatus.Liquidated || !Engine.LiveMode)
            {
                // without this we can't liquidate equities since the exchange is 'technically' closed
                var hackedFrontier = algorithm.Time == DateTime.MinValue ? DateTime.MinValue : algorithm.Time.AddMilliseconds(-1);
                algorithm.SetDateTime(hackedFrontier);
                foreach (var security in algorithm.Securities)
                {
                    security.Value.SetMarketPrice(hackedFrontier, null);
                }

                Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
                algorithm.Liquidate();
                results.LogMessage("Algorithm Liquidated");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
            }

            //Manually stopped the algorithm
            if (_algorithmState == AlgorithmStatus.Stopped)
            {
                Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
                results.LogMessage("Algorithm Stopped");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
            }

            //Backtest deleted.
            if (_algorithmState == AlgorithmStatus.Deleted)
            {
                Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
                results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
                results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
            }

            //Algorithm finished, send regardless of commands:
            results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);

            //Take final samples:
            results.SampleRange(algorithm.GetChartUpdates());
            results.SampleEquity(DataStream.AlorithmTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
            results.SamplePerformance(DataStream.AlorithmTime, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
        } // End of Run();