public override void RevertChanges()
{
Filter.RevertChanges();
Phaser.RevertChanges();
Flanger.RevertChanges();
Synth.RevertChanges();
LoFi.RevertChanges();
RingModulator.RevertChanges();
GuitarToBass.RevertChanges();
SlowGear.RevertChanges();
Transpose.RevertChanges();
PitchBend.RevertChanges();
Robot.RevertChanges();
VocalDist.RevertChanges();
Dynamics.RevertChanges();
Eq.RevertChanges();
Isolator.RevertChanges();
Octave.RevertChanges();
Pan.RevertChanges();
Tremolo.RevertChanges();
Slicer.RevertChanges();
Delay.RevertChanges();
PanningDelay.RevertChanges();
TapeEcho.RevertChanges();
GranularDelay.RevertChanges();
Roll.RevertChanges();
Chorus.RevertChanges();
Reverb.RevertChanges();
}
/// <summary>
/// Initialize the debugger
/// </summary>
/// <param name="algorithmNodePacket">The algorithm node packet</param>
/// <param name="workerThread">The worker thread instance to use</param>
public static bool InitializeDebugging(AlgorithmNodePacket algorithmNodePacket, WorkerThread workerThread)
{
var isolator = new Isolator();
return(isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5),
() => DebuggerHelper.Initialize(algorithmNodePacket.Language),
algorithmNodePacket.RamAllocation,
sleepIntervalMillis: 100,
workerThread: workerThread));
}
public int SaveIsolator(Isolator isolator, string user)
{
if (isolator.IsolatorId > 0)
{
isolator.SetUpdateDetails(user);
repository.SaveExisting(isolator);
return(isolator.IsolatorId);
}
isolator.SetCreateDetails(user);
return(repository.SaveNew(isolator).IsolatorId);
}
private void UpdateDisplay()
{
Debug.Assert(!InvokeRequired);
if (!_browserIsReadyToNavigate)
{
return;
}
if (_url != null)
{
_browser.Visible = true;
Isolator.Navigate(_browser, _url);
}
}
private object MapIsolatorRepsonse(Isolator iso, DateTime requestDate)
{
var isoTotalAvailabilityMins = iso.StaffShiftAllocations.Where(a => a.AllocatedDate.Date.Equals(requestDate.Date) && !a.IsArchived).Sum(s => s.IsolatorShift.TotalShiftDurationInMins);
var prepOrders = iso.PreperationOrders.Where(a => a.PreperationDateTime.Date.Equals(requestDate.Date) && !a.IsArchived).ToList();
var percent = isoTotalAvailabilityMins > 0 ? (prepOrders.Sum(p => p.IntegrationOrder.RequiredPreperationTimeInMins) * 100 / isoTotalAvailabilityMins) : 0;
return(new
{
iso.IsolatorId,
IsolatorName = iso.Abbriviation,
SessionDose = iso.TotalNumberOfDosesPerSession,
ScheduledOrders = prepOrders.Count(),
ScheduledPercent = Math.Round(percent),
IsolatorUnavailabe = isoTotalAvailabilityMins <= 0
});
}
public static GridBoxRow BindGridData(Isolator source)
{
var row = new GridBoxRow
{
IdentityValue = source.IsolatorId,
BoxHeading = source.Abbriviation
};
row.BoxBodyDictionary.Add("Total no. of doses per session", source.TotalNumberOfDosesPerSession.ToString());
row.AddActionIcon("fa fa-edit text-success", "Click to view/edit");
row.AddActionIcon("fa fa-trash text-danger", "Click to delete");
row.AddActionIcon("fa fa-calendar-times-o text-warning", "Click to manage isolator offline dates");
row.AddActionIcon("fa fa-list text-info", "Click to manage isolator start/stop procedures");
row.AddActionIcon("fa fa-users text-primary", "Click to manage staff allocation", "/isolator/isolatorstaffallocationindex?isoid=" + source.IsolatorId + "&nm=" + source.Abbriviation + "");
return(row);
}
public void TimeOutWorkCorrectly(Language language, bool useWorker)
{
var worker = useWorker ? new TestWorkerThread() : null;
using (worker)
{
var isolator = new Isolator();
bool result = false;
try
{
if (language == Language.CSharp)
{
result = isolator.ExecuteWithTimeLimit(
TimeSpan.FromMilliseconds(100),
() => Thread.Sleep(10000), // 10s sleep
5000,
workerThread: worker
);
}
else
{
result = isolator.ExecuteWithTimeLimit(
TimeSpan.FromMilliseconds(100),
() =>
{
using (Py.GIL())
{
// 10s sleep
PythonEngine.RunSimpleString("import time; time.sleep(10)");
}
},
5000,
workerThread: worker
);
}
Assert.Fail($"Was expecting {nameof(TimeoutException)}");
}
catch (TimeoutException)
{
Assert.IsFalse(result);
}
}
}
public void WorksCorrectlyUsingWorker()
{
using (var worker = new TestWorkerThread())
{
var isolator = new Isolator();
var executed = false;
var result = isolator.ExecuteWithTimeLimit(
TimeSpan.FromMilliseconds(100),
() =>
{
executed = true;
},
5000,
workerThread: worker
);
Assert.IsTrue(result);
Assert.IsTrue(executed);
}
}
/// <summary>
/// Creates a new instance of the class in the library, safely.
/// </summary>
/// <param name="assemblyPath">Location of the DLL</param>
/// <param name="algorithmInstance">Output algorithm instance</param>
/// <param name="errorMessage">Output error message on failure</param>
/// <returns>bool success</returns>
public bool TryCreateAlgorithmInstanceWithIsolator(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
IAlgorithm instance = null;
string error = string.Empty;
bool success = false;
bool complete = Isolator.ExecuteWithTimeLimit(_loaderTimeLimit, () =>
{
success = TryCreateAlgorithmInstance(assemblyPath, out instance, out error);
});
algorithmInstance = instance;
errorMessage = error;
// if the isolator stopped us early add that to our error message
if (!complete)
{
errorMessage = "Failed to create algorithm instance within 10 seconds. Try re-building algorithm. " + error;
}
return(complete && success && algorithmInstance != null);
}
/// <summary>
/// Creates a new instance of the class in the library, safely.
/// </summary>
/// <param name="assemblyPath">Location of the DLL</param>
/// <param name="ramLimit">Limit of the RAM for this process</param>
/// <param name="algorithmInstance">Output algorithm instance</param>
/// <param name="errorMessage">Output error message on failure</param>
/// <returns>bool success</returns>
public bool TryCreateAlgorithmInstanceWithIsolator(string assemblyPath, int ramLimit, out IAlgorithm algorithmInstance, out string errorMessage)
{
IAlgorithm instance = null;
var error = string.Empty;
var success = false;
var isolator = new Isolator();
var complete = isolator.ExecuteWithTimeLimit(_loaderTimeLimit, () =>
{
success = TryCreateAlgorithmInstance(assemblyPath, out instance, out error);
}, ramLimit, sleepIntervalMillis: 50, workerThread: _workerThread);
algorithmInstance = instance;
errorMessage = error;
// if the isolator stopped us early add that to our error message
if (!complete)
{
errorMessage = "Failed to create algorithm instance within 10 seconds. Try re-building algorithm. " + error;
}
return(complete && success && algorithmInstance != null);
}
private async void StartFlowInternal()
{
Isolator isolator = new Isolator();
var remoteExcel = isolator.GetIsolatedInstance <IRemoteExeclController, RemoteExeclController>();
await Task.Run(() =>
{
var remoteActual = remoteExcel.Item2;
remoteActual.RunExcelInit(this, new DataUpdater());
});
isolator.UnloadIsolationContext(remoteExcel.Item1);
// today's arrivals
updateArrivalsGrid();
// yesterday's sails
updateSailsGrid();
// destination port
updateDestinationGrid();
}
public override void ApplyChanges <T>(T model)
{
if (model is SettingsMemoryFxModel memoryFx)
{
Filter.ApplyChanges(memoryFx);
Phaser.ApplyChanges(memoryFx);
Flanger.ApplyChanges(memoryFx);
Synth.ApplyChanges(memoryFx);
LoFi.ApplyChanges(memoryFx);
RingModulator.ApplyChanges(memoryFx);
GuitarToBass.ApplyChanges(memoryFx);
SlowGear.ApplyChanges(memoryFx);
Transpose.ApplyChanges(memoryFx);
PitchBend.ApplyChanges(memoryFx);
Robot.ApplyChanges(memoryFx);
VocalDist.ApplyChanges(memoryFx);
Dynamics.ApplyChanges(memoryFx);
Eq.ApplyChanges(memoryFx);
Isolator.ApplyChanges(memoryFx);
Octave.ApplyChanges(memoryFx);
Pan.ApplyChanges(memoryFx);
Tremolo.ApplyChanges(memoryFx);
Slicer.ApplyChanges(memoryFx);
Delay.ApplyChanges(memoryFx);
PanningDelay.ApplyChanges(memoryFx);
TapeEcho.ApplyChanges(memoryFx);
GranularDelay.ApplyChanges(memoryFx);
Roll.ApplyChanges(memoryFx);
Chorus.ApplyChanges(memoryFx);
Reverb.ApplyChanges(memoryFx);
}
else
{
throw new ArgumentException("Model must be of type SettingsMemoryFxModel.");
}
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
public void Run(AlgorithmNodePacket job, string assemblyPath)
{
var algorithm = default(IAlgorithm);
var algorithmManager = new AlgorithmManager(_liveMode);
//Start monitoring the backtest active status:
var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results, _systemHandlers.Notify, job);
var statusPingThread = new Thread(statusPing.Run);
statusPingThread.Start();
try
{
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
//-> Initialize messaging system
_systemHandlers.Notify.SetAuthentication(job);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
_algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions);
threadResults = new Thread(_algorithmHandlers.Results.Run, 0) {Name = "Result Thread"};
threadResults.Start();
IBrokerage brokerage = null;
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language);
// Initialize the brokerage
brokerage = _algorithmHandlers.Setup.CreateBrokerage(job, algorithm);
// Initialize the data feed before we initialize so he can intercept added securities/universes via events
_algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider);
// initialize command queue system
_algorithmHandlers.CommandQueue.Initialize(job, algorithm);
// set the history provider before setting up the algorithm
_algorithmHandlers.HistoryProvider.Initialize(job, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider, progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
_algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History,
string.Format("Processing history {0}%...", progress));
}
});
algorithm.HistoryProvider = _algorithmHandlers.HistoryProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = new DefaultBrokerageMessageHandler(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime);
// set this again now that we've actually added securities
_algorithmHandlers.Results.SetAlgorithm(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors);
Log.Error("Engine.Run(): " + errorMessage);
_algorithmHandlers.Results.RuntimeError(errorMessage);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
Log.Error(err);
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
_algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Load the associated handlers for transaction and realtime events:
_algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results);
_algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
// wire up the brokerage message handler
brokerage.Message += (sender, message) =>
{
algorithm.BrokerageMessageHandler.Handle(message);
// fire brokerage message events
algorithm.OnBrokerageMessage(message);
switch (message.Type)
{
case BrokerageMessageType.Disconnect:
algorithm.OnBrokerageDisconnect();
break;
case BrokerageMessageType.Reconnect:
algorithm.OnBrokerageReconnect();
break;
}
};
//Send status to user the algorithm is now executing.
_algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) {Name = "DataFeed Thread"};
threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) {Name = "Transaction Thread"};
threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) {Name = "RealTime Thread"};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
_algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Globals.Version));
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, _algorithmHandlers.CommandQueue, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.RamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
throw algorithm.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error(err, "Breaking out of parent try catch:");
if (_algorithmHandlers.DataFeed != null) _algorithmHandlers.DataFeed.Exit();
if (_algorithmHandlers.Results != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
_algorithmHandlers.Results.LogMessage(message);
_algorithmHandlers.Results.RuntimeError(message, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
try
{
var trades = algorithm.TradeBuilder.ClosedTrades;
var charts = new Dictionary<string, Chart>(_algorithmHandlers.Results.Charts);
var orders = new Dictionary<int, Order>(_algorithmHandlers.Transactions.Orders);
var holdings = new Dictionary<string, Holding>();
var banner = new Dictionary<string, string>();
var statisticsResults = new StatisticsResults();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
const string benchmarkKey = "Benchmark";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss = new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord);
var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count();
var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values;
statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark,
_algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions);
//Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors
var netReturn = _algorithmHandlers.Setup.StartingPortfolioValue > 0 ?
(algorithm.Portfolio.TotalPortfolioValue - _algorithmHandlers.Setup.StartingPortfolioValue) / _algorithmHandlers.Setup.StartingPortfolioValue
: 0;
//Add other fixed parameters.
banner.Add("Unrealized:", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2"));
banner.Add("Fees:", "-$" + algorithm.Portfolio.TotalFees.ToString("N2"));
banner.Add("Net Profit:", "$" + algorithm.Portfolio.TotalProfit.ToString("N2"));
banner.Add("Return:", netReturn.ToString("P"));
banner.Add("Equity:", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2"));
banner.Add("Holdings:", "$" + algorithm.Portfolio.TotalHoldingsValue.ToString("N2"));
banner.Add("Volume:", "$" + algorithm.Portfolio.TotalSaleVolume.ToString("N2"));
}
}
catch (Exception err)
{
Log.Error(err, "Error generating statistics packet");
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + _algorithmHandlers.HistoryProvider.DataPointCount;
_algorithmHandlers.Results.DebugMessage(
string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints/(double) 1000)/totalSeconds).ToString("F0"),
dataPoints.ToString("N0")));
_algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statisticsResults, banner);
}
catch (Exception err)
{
Log.Error(err, "Error sending analysis results");
}
//Before we return, send terminate commands to close up the threads
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.RealTime.Exit();
}
//Close result handler:
_algorithmHandlers.Results.Exit();
statusPing.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((_algorithmHandlers.Results.IsActive
|| (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive)
|| (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive)
|| (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive))
&& ts.ElapsedMilliseconds < 30*1000)
{
Thread.Sleep(100);
Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort();
if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort();
if (threadResults != null && threadResults.IsAlive) threadResults.Abort();
if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort();
if (brokerage != null)
{
Log.Trace("Engine.Run(): Disconnecting from brokerage...");
brokerage.Disconnect();
}
if (_algorithmHandlers.Setup != null)
{
Log.Trace("Engine.Run(): Disposing of setup handler...");
_algorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error(err, "Error running algorithm");
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
_algorithmHandlers.Results.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="manager">The algorithm manager instance</param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
/// <param name="workerThread">The worker thread instance</param>
public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath, WorkerThread workerThread)
{
var marketHoursDatabaseTask = Task.Run(() => StaticInitializations());
var algorithm = default(IAlgorithm);
var algorithmManager = manager;
try
{
//Reset thread holders.
var initializeComplete = false;
//-> Initialize messaging system
SystemHandlers.Notify.SetAuthentication(job);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions);
IBrokerage brokerage = null;
DataManager dataManager = null;
var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer();
try
{
// we get the mhdb before creating the algorithm instance,
// since the algorithm constructor will use it
var marketHoursDatabase = marketHoursDatabaseTask.Result;
AlgorithmHandlers.Setup.WorkerThread = workerThread;
// Save algorithm to cache, load algorithm instance:
algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath);
// Set algorithm in ILeanManager
SystemHandlers.LeanManager.SetAlgorithm(algorithm);
// initialize the alphas handler with the algorithm instance
AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api);
// initialize the object store
AlgorithmHandlers.ObjectStore.Initialize(algorithm.Name, job.UserId, job.ProjectId, job.UserToken, job.Controls);
// notify the user of any errors w/ object store persistence
AlgorithmHandlers.ObjectStore.ErrorRaised += (sender, args) => algorithm.Debug($"ObjectStore Persistence Error: {args.Error.Message}");
// Initialize the brokerage
IBrokerageFactory factory;
brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory);
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDatabase,
algorithm,
registeredTypesProvider,
new SecurityCacheProvider(algorithm.Portfolio));
algorithm.Securities.SetSecurityService(securityService);
dataManager = new DataManager(AlgorithmHandlers.DataFeed,
new UniverseSelection(
algorithm,
securityService),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
_liveMode,
registeredTypesProvider);
AlgorithmHandlers.Results.SetDataManager(dataManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
synchronizer.Initialize(algorithm, dataManager);
// Initialize the data feed before we initialize so he can intercept added securities/universes via events
AlgorithmHandlers.DataFeed.Initialize(
algorithm,
job,
AlgorithmHandlers.Results,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
AlgorithmHandlers.DataProvider,
dataManager,
(IDataFeedTimeProvider)synchronizer);
// set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider)
algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions);
algorithm.SetOrderEventProvider(AlgorithmHandlers.Transactions);
// set the history provider before setting up the algorithm
var historyProvider = GetHistoryProvider(job.HistoryProvider);
if (historyProvider is BrokerageHistoryProvider)
{
(historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage);
}
var historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode);
historyProvider.Initialize(
new HistoryProviderInitializeParameters(
job,
SystemHandlers.Api,
AlgorithmHandlers.DataProvider,
historyDataCacheProvider,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History,
Invariant($"Processing history {progress}%..."));
}
},
// disable parallel history requests for live trading
parallelHistoryRequestsEnabled: !_liveMode
)
);
historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); };
historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); };
historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); };
historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); };
algorithm.HistoryProvider = historyProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore));
// set this again now that we've actually added securities
AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue);
// alpha handler needs start/end dates to determine sample step sizes
AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = string.Join(",", algorithm.ErrorMessages);
errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e =>
{
var message = e.Message;
if (e.InnerException != null)
{
var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value);
message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err);
}
return(message);
}));
Log.Error("Engine.Run(): " + errorMessage);
AlgorithmHandlers.Results.RuntimeError(errorMessage);
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
Log.Error(err);
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err;
AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString());
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName);
Log.Trace(" ObjectStore: " + AlgorithmHandlers.ObjectStore.GetType().FullName);
if (algorithm?.HistoryProvider != null)
{
Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName);
}
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + brokerage?.GetType().FullName);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
// notify the LEAN manager that the algorithm is initialized and starting
SystemHandlers.LeanManager.OnAlgorithmStart();
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Load the associated handlers for transaction and realtime events:
AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results);
AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api, algorithmManager.TimeLimit);
// wire up the brokerage message handler
brokerage.Message += (sender, message) =>
{
algorithm.BrokerageMessageHandler.Handle(message);
// fire brokerage message events
algorithm.OnBrokerageMessage(message);
switch (message.Type)
{
case BrokerageMessageType.Disconnect:
algorithm.OnBrokerageDisconnect();
break;
case BrokerageMessageType.Reconnect:
algorithm.OnBrokerageReconnect();
break;
}
};
//Send status to user the algorithm is now executing.
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running);
// Result manager scanning message queue: (started earlier)
AlgorithmHandlers.Results.DebugMessage(
$"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}");
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLimit.IsWithinLimit, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.Controls.RamAllocation, workerThread: workerThread);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F"));
throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
HandleAlgorithmError(job, algorithm.RunTimeError);
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
algorithm.RunTimeError = err;
algorithm.SetStatus(AlgorithmStatus.RuntimeError);
HandleAlgorithmError(job, err);
}
// notify the LEAN manager that the algorithm has finished
SystemHandlers.LeanManager.OnAlgorithmEnd();
try
{
var csvTransactionsFileName = Config.Get("transaction-log");
if (!string.IsNullOrEmpty(csvTransactionsFileName))
{
SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName);
}
if (!_liveMode)
{
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount;
var kps = dataPoints / (double)1000 / totalSeconds;
AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points.");
}
}
catch (Exception err)
{
Log.Error(err, "Error sending analysis results");
}
//Before we return, send terminate commands to close up the threads
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.RealTime.Exit();
dataManager?.RemoveAllSubscriptions();
workerThread?.Dispose();
}
// Close data feed, alphas. Could be running even if algorithm initialization failed
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.Alphas.Exit();
//Close result handler:
AlgorithmHandlers.Results.Exit();
//Wait for the threads to complete:
var millisecondInterval = 10;
var millisecondTotalWait = 0;
while ((AlgorithmHandlers.Results.IsActive ||
(AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) ||
(AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) ||
(AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) ||
(AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) &&
millisecondTotalWait < 30 * 1000)
{
Thread.Sleep(millisecondInterval);
if (millisecondTotalWait % (millisecondInterval * 10) == 0)
{
Log.Trace("Waiting for threads to exit...");
}
millisecondTotalWait += millisecondInterval;
}
if (brokerage != null)
{
Log.Trace("Engine.Run(): Disconnecting from brokerage...");
brokerage.Disconnect();
brokerage.Dispose();
}
if (AlgorithmHandlers.Setup != null)
{
Log.Trace("Engine.Run(): Disposing of setup handler...");
AlgorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error(err, "Error running algorithm");
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
AlgorithmHandlers.Results.Exit();
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
_algorithm = algorithm;
brokerage = default(IBrokerage);
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return false;
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return false;
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler<BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message));
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
//Execute the initialize code:
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () =>
{
try
{
//Set the live trading level asset/ram allocation limits.
//Protects algorithm from linux killing the job by excess memory:
switch (job.ServerType)
{
case ServerType.Server1024:
algorithm.SetAssetLimits(100, 20, 10);
break;
case ServerType.Server2048:
algorithm.SetAssetLimits(400, 50, 30);
break;
default: //512
algorithm.SetAssetLimits(50, 25, 15);
break;
}
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return false;
}
try
{
// find the correct brokerage factory based on the specified brokerage in the live job packet
_factory = Composer.Instance.Single<IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
}
catch (Exception err)
{
Log.Error("BrokerageSetupHandler.Setup(): Error resolving brokerage factory for " + liveJob.Brokerage + ". " + err.Message);
AddInitializationError("Unable to locate factory for brokerage: " + liveJob.Brokerage);
}
// let the world know what we're doing since logging in can take a minute
resultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.LoggingIn, "Logging into brokerage...");
// initialize the correct brokerage using the resolved factory
brokerage = _factory.CreateBrokerage(liveJob, algorithm);
if (brokerage == null)
{
AddInitializationError("Failed to create instance of brokerage: " + liveJob.Brokerage);
return false;
}
brokerage.Message += brokerageOnMessage;
// set the transaction models base on the brokerage properties
SetupHandler.UpdateTransactionModels(algorithm, algorithm.BrokerageModel);
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error connecting to brokerage: " + err.Message);
return false;
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return false;
}
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Quantity);
algorithm.SetCash(cash.Symbol, cash.Quantity, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message);
return false;
}
try
{
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol + " - " + order.Quantity);
order.Id = algorithm.Transactions.GetIncrementOrderId();
transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message);
return false;
}
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var minResolution = new Lazy<Resolution>(() => algorithm.Securities.Min(x => x.Value.Resolution));
foreach (var holding in holdings)
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
if (!algorithm.Portfolio.ContainsKey(holding.Symbol))
{
Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol);
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, holding.Symbol, minResolution.Value, null, true, 1.0m, false);
}
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int) holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message);
return false;
}
// call this after we've initialized everything from the brokerage since we may have added some holdings/currencies
algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager, SecurityExchangeHoursProvider.FromDataFolder());
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return Errors.Count == 0;
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="manager"></param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath)
{
var marketHoursDatabaseTask = Task.Run(() => StaticInitializations());
var algorithm = default(IAlgorithm);
var algorithmManager = manager;
try
{
//Reset thread holders.
var initializeComplete = false;
Thread threadResults = null;
Thread threadRealTime = null;
Thread threadAlphas = null;
WorkerThread workerThread = null;
//-> Initialize messaging system
SystemHandlers.Notify.SetAuthentication(job);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions);
threadResults = new Thread(AlgorithmHandlers.Results.Run, 0)
{
IsBackground = true, Name = "Result Thread"
};
threadResults.Start();
IBrokerage brokerage = null;
DataManager dataManager = null;
var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer();
try
{
// we get the mhdb before creating the algorithm instance,
// since the algorithm constructor will use it
var marketHoursDatabase = marketHoursDatabaseTask.Result;
// start worker thread
workerThread = new WorkerThread();
AlgorithmHandlers.Setup.WorkerThread = workerThread;
// Save algorithm to cache, load algorithm instance:
algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath);
// Set algorithm in ILeanManager
SystemHandlers.LeanManager.SetAlgorithm(algorithm);
// initialize the alphas handler with the algorithm instance
AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api);
// Initialize the brokerage
IBrokerageFactory factory;
brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory);
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDatabase,
(ISecurityInitializerProvider)algorithm);
algorithm.Securities.SetSecurityService(securityService);
dataManager = new DataManager(AlgorithmHandlers.DataFeed,
new UniverseSelection(
algorithm,
securityService),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
_liveMode);
AlgorithmHandlers.Results.SetDataManager(dataManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
synchronizer.Initialize(algorithm, dataManager);
// Initialize the data feed before we initialize so he can intercept added securities/universes via events
AlgorithmHandlers.DataFeed.Initialize(
algorithm,
job,
AlgorithmHandlers.Results,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
AlgorithmHandlers.DataProvider,
dataManager,
(IDataFeedTimeProvider)synchronizer);
// set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider)
algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions);
algorithm.SetOrderEventProvider(AlgorithmHandlers.Transactions);
// set the history provider before setting up the algorithm
var historyProvider = GetHistoryProvider(job.HistoryProvider);
if (historyProvider is BrokerageHistoryProvider)
{
(historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage);
}
var historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode);
historyProvider.Initialize(
new HistoryProviderInitializeParameters(
job,
SystemHandlers.Api,
AlgorithmHandlers.DataProvider,
historyDataCacheProvider,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History,
$"Processing history {progress}%...");
}
}
)
);
historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); };
historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); };
historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); };
historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); };
algorithm.HistoryProvider = historyProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime));
// set this again now that we've actually added securities
AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue);
// alpha handler needs start/end dates to determine sample step sizes
AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = string.Join(",", algorithm.ErrorMessages);
errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e =>
{
var message = e.Message;
if (e.InnerException != null)
{
var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value);
message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err);
}
return(message);
}));
Log.Error("Engine.Run(): " + errorMessage);
AlgorithmHandlers.Results.RuntimeError(errorMessage);
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
Log.Error(err);
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err;
AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString());
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName);
if (algorithm?.HistoryProvider != null)
{
Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName);
}
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + brokerage?.GetType().FullName);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
// notify the LEAN manager that the algorithm is initialized and starting
SystemHandlers.LeanManager.OnAlgorithmStart();
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Load the associated handlers for transaction and realtime events:
AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results);
AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api);
// wire up the brokerage message handler
brokerage.Message += (sender, message) =>
{
algorithm.BrokerageMessageHandler.Handle(message);
// fire brokerage message events
algorithm.OnBrokerageMessage(message);
switch (message.Type)
{
case BrokerageMessageType.Disconnect:
algorithm.OnBrokerageDisconnect();
break;
case BrokerageMessageType.Reconnect:
algorithm.OnBrokerageReconnect();
break;
}
};
//Send status to user the algorithm is now executing.
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadRealTime = new Thread(AlgorithmHandlers.RealTime.Run)
{
IsBackground = true, Name = "RealTime Thread"
};
threadAlphas = new Thread(() => AlgorithmHandlers.Alphas.Run())
{
IsBackground = true, Name = "Alpha Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadRealTime.Start(); // RealTime scan time for time based events:
threadAlphas.Start(); // Alpha thread for processing algorithm alpha insights
// Result manager scanning message queue: (started earlier)
AlgorithmHandlers.Results.DebugMessage(
$"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}");
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.Controls.RamAllocation, workerThread: workerThread);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToString("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
HandleAlgorithmError(job, algorithm.RunTimeError);
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
HandleAlgorithmError(job, err);
}
// notify the LEAN manager that the algorithm has finished
SystemHandlers.LeanManager.OnAlgorithmEnd();
try
{
var trades = algorithm.TradeBuilder.ClosedTrades;
var charts = new Dictionary <string, Chart>(AlgorithmHandlers.Results.Charts);
var orders = new Dictionary <int, Order>(AlgorithmHandlers.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var banner = new Dictionary <string, string>();
var statisticsResults = new StatisticsResults();
var csvTransactionsFileName = Config.Get("transaction-log");
if (!string.IsNullOrEmpty(csvTransactionsFileName))
{
SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName);
}
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
const string benchmarkKey = "Benchmark";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey) &&
charts.ContainsKey(benchmarkKey) &&
charts[benchmarkKey].Series.ContainsKey(benchmarkKey)
)
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count();
var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values;
statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark,
AlgorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions);
//Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors
var netReturn = AlgorithmHandlers.Setup.StartingPortfolioValue > 0 ?
(algorithm.Portfolio.TotalPortfolioValue - AlgorithmHandlers.Setup.StartingPortfolioValue) / AlgorithmHandlers.Setup.StartingPortfolioValue
: 0;
//Add other fixed parameters.
banner.Add("Unrealized", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2"));
banner.Add("Fees", "-$" + algorithm.Portfolio.TotalFees.ToString("N2"));
banner.Add("Net Profit", "$" + algorithm.Portfolio.TotalProfit.ToString("N2"));
banner.Add("Return", netReturn.ToString("P"));
banner.Add("Equity", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2"));
}
}
catch (Exception err)
{
Log.Error(err, "Error generating statistics packet");
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount;
if (!_liveMode)
{
var kps = dataPoints / (double)1000 / totalSeconds;
AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points.");
}
AlgorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, algorithm.Portfolio.CashBook, statisticsResults, banner);
}
catch (Exception err)
{
Log.Error(err, "Error sending analysis results");
}
//Before we return, send terminate commands to close up the threads
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.RealTime.Exit();
AlgorithmHandlers.Alphas.Exit();
dataManager?.RemoveAllSubscriptions();
workerThread?.Dispose();
}
//Close result handler:
AlgorithmHandlers.Results.Exit();
//Wait for the threads to complete:
var millisecondInterval = 10;
var millisecondTotalWait = 0;
while ((AlgorithmHandlers.Results.IsActive ||
(AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) ||
(AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) ||
(AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) ||
(AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) &&
millisecondTotalWait < 30 * 1000)
{
Thread.Sleep(millisecondInterval);
if (millisecondTotalWait % (millisecondInterval * 10) == 0)
{
Log.Trace("Waiting for threads to exit...");
}
millisecondTotalWait += millisecondInterval;
}
//Terminate threads still in active state.
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
if (threadAlphas != null && threadAlphas.IsAlive)
{
threadAlphas.Abort();
}
if (brokerage != null)
{
Log.Trace("Engine.Run(): Disconnecting from brokerage...");
brokerage.Disconnect();
brokerage.Dispose();
}
if (AlgorithmHandlers.Setup != null)
{
Log.Trace("Engine.Run(): Disposing of setup handler...");
AlgorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error(err, "Error running algorithm");
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
AlgorithmHandlers.Results.Exit();
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket job)
{
_algorithm = algorithm;
brokerage = default(IBrokerage);
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return(false);
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return(false);
}
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
//Execute the initialize code:
var initializeComplete = Isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () =>
{
try
{
//Set the live trading level asset/ram allocation limits.
//Protects algorithm from linux killing the job by excess memory:
switch (job.ServerType)
{
case ServerType.Server1024:
algorithm.SetAssetLimits(100, 20, 10);
break;
case ServerType.Server2048:
algorithm.SetAssetLimits(400, 50, 30);
break;
default: //512
algorithm.SetAssetLimits(50, 25, 15);
break;
}
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return(false);
}
try
{
// find the correct brokerage factory based on the specified brokerage in the live job packet
_factory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
}
catch (Exception err)
{
Log.Error("BrokerageSetupHandler.Setup(): Error resolving brokerage factory for " + liveJob.Brokerage + ". " + err.Message);
AddInitializationError("Unable to locate factory for brokerage: " + liveJob.Brokerage);
}
// let the world know what we're doing since logging in can take a minute
Engine.ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.LoggingIn, "Logging into brokerage...");
// initialize the correct brokerage using the resolved factory
brokerage = _factory.CreateBrokerage(liveJob, algorithm);
// set the transaction models base on the brokerage properties
SetupHandler.UpdateTransactionModels(algorithm, algorithm.BrokerageModel);
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error connecting to brokerage.");
return(false);
}
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Quantity);
algorithm.SetCash(cash.Symbol, cash.Quantity, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage.");
return(false);
}
try
{
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol + " - " + order.Quantity);
order.Id = algorithm.Transactions.GetIncrementOrderId();
algorithm.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage.");
return(false);
}
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Min(x => x.Value.Resolution));
foreach (var holding in holdings)
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
if (!algorithm.Portfolio.ContainsKey(holding.Symbol))
{
Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol);
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, holding.Symbol, minResolution.Value, true, 1.0m, false);
}
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(DateTime.Now, new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage.");
return(false);
}
// call this after we've initialized everything from the brokerage since we may have added some holdings/currencies
algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager);
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
return(Errors.Count == 0);
}
/// <summary>
/// Setup the algorithm cash, dates and portfolio as desired.
/// </summary>
/// <param name="parameters">The parameters object to use</param>
/// <returns>Boolean true on successfully setting up the console.</returns>
public bool Setup(SetupHandlerParameters parameters)
{
var algorithm = parameters.Algorithm;
var baseJob = parameters.AlgorithmNodePacket;
var initializeComplete = false;
try
{
//Set common variables for console programs:
if (baseJob.Type == PacketType.BacktestNode)
{
var backtestJob = baseJob as BacktestNodePacket;
if (backtestJob == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name);
}
algorithm.SetMaximumOrders(int.MaxValue);
// set our parameters
algorithm.SetParameters(baseJob.Parameters);
algorithm.SetLiveMode(false);
algorithm.SetAvailableDataTypes(GetConfiguredDataFeeds());
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler);
// set the option chain provider
algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
// set the future chain provider
algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
var isolator = new Isolator();
isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5),
() =>
{
//Setup Base Algorithm:
algorithm.Initialize();
}, baseJob.Controls.RamAllocation,
sleepIntervalMillis: 50,
workerThread: WorkerThread);
//Finalize Initialization
algorithm.PostInitialize();
//Set the time frontier of the algorithm
algorithm.SetDateTime(algorithm.StartDate.ConvertToUtc(algorithm.TimeZone));
//Construct the backtest job packet:
backtestJob.PeriodStart = algorithm.StartDate;
backtestJob.PeriodFinish = algorithm.EndDate;
//Backtest Specific Parameters:
StartingDate = backtestJob.PeriodStart;
BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection);
StartingPortfolioValue = algorithm.Portfolio.Cash;
}
else
{
throw new Exception("The ConsoleSetupHandler is for backtests only. Use the BrokerageSetupHandler.");
}
}
catch (Exception err)
{
Log.Error(err);
Errors.Add(new AlgorithmSetupException("During the algorithm initialization, the following exception has occurred: ", err));
}
if (Errors.Count == 0)
{
initializeComplete = true;
}
return(initializeComplete);
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
public void Run(AlgorithmNodePacket job, string assemblyPath)
{
var algorithm = default(IAlgorithm);
var algorithmManager = new AlgorithmManager(_liveMode);
//Start monitoring the backtest active status:
var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results, _systemHandlers.Notify, job);
var statusPingThread = new Thread(statusPing.Run);
statusPingThread.Start();
try
{
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
//-> Initialize messaging system
_systemHandlers.Notify.SetChannel(job.Channel);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
_algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions);
threadResults = new Thread(_algorithmHandlers.Results.Run, 0)
{
Name = "Result Thread"
};
threadResults.Start();
IBrokerage brokerage = null;
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language);
// set the history provider before setting up the algorithm
_algorithmHandlers.HistoryProvider.Initialize(job, progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
_algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.History,
string.Format("Processing history {0}%...", progress));
}
});
algorithm.HistoryProvider = _algorithmHandlers.HistoryProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = new DefaultBrokerageMessageHandler(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, out brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime);
// set this again now that we've actually added securities
_algorithmHandlers.Results.SetAlgorithm(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors);
_algorithmHandlers.Results.RuntimeError(errorMessage);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
_algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Wire up the universe selection event handler before kicking off the data feed
var universeSelection = new UniverseSelection(_algorithmHandlers.DataFeed, algorithm, _liveMode);
_algorithmHandlers.DataFeed.UniverseSelection += (sender, args) => universeSelection.ApplyUniverseSelection(args);
//Load the associated handlers for data, transaction and realtime events:
_algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results);
_algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results);
_algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
// wire up the brokerage message handler
brokerage.Message += (sender, message) => algorithm.BrokerageMessageHandler.Handle(message);
//Send status to user the algorithm is now executing.
_algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(_algorithmHandlers.DataFeed.Run)
{
Name = "DataFeed Thread"
};
threadTransactions = new Thread(_algorithmHandlers.Transactions.Run)
{
Name = "Transaction Thread"
};
threadRealTime = new Thread(_algorithmHandlers.RealTime.Run)
{
Name = "RealTime Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
_algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version));
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.RamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
throw algorithm.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (_algorithmHandlers.DataFeed != null)
{
_algorithmHandlers.DataFeed.Exit();
}
if (_algorithmHandlers.Results != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
_algorithmHandlers.Results.LogMessage(message);
_algorithmHandlers.Results.RuntimeError(message, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var trades = algorithm.TradeBuilder.ClosedTrades;
var charts = new Dictionary <string, Chart>(_algorithmHandlers.Results.Charts);
var orders = new Dictionary <int, Order>(_algorithmHandlers.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var banner = new Dictionary <string, string>();
var statisticsResults = new StatisticsResults();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
const string benchmarkKey = "Benchmark";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count();
var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values;
statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark,
_algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions);
}
}
catch (Exception err)
{
Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + _algorithmHandlers.HistoryProvider.DataPointCount;
_algorithmHandlers.Results.DebugMessage(
string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints / (double)1000) / totalSeconds).ToString("F0"),
dataPoints.ToString("N0")));
_algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statisticsResults, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.RealTime.Exit();
}
//Close result handler:
_algorithmHandlers.Results.Exit();
statusPing.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((_algorithmHandlers.Results.IsActive ||
(_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) ||
(_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) ||
(_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) &&
ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100);
Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadFeed != null && threadFeed.IsAlive)
{
threadFeed.Abort();
}
if (threadTransactions != null && threadTransactions.IsAlive)
{
threadTransactions.Abort();
}
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
if (statusPingThread != null && statusPingThread.IsAlive)
{
statusPingThread.Abort();
}
if (brokerage != null)
{
brokerage.Disconnect();
}
if (_algorithmHandlers.Setup != null)
{
_algorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
_algorithmHandlers.Results.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket job)
{
_algorithm = algorithm;
brokerage = default(IBrokerage);
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return(false);
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return(false);
}
try
{
//Execute the initialize code:
var initializeComplete = Isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () =>
{
try
{
//Set the live trading level asset/ram allocation limits.
//Protects algorithm from linux killing the job by excess memory:
switch (job.ServerType)
{
case ServerType.Server1024:
algorithm.SetAssetLimits(100, 20, 10);
break;
case ServerType.Server2048:
algorithm.SetAssetLimits(400, 50, 30);
break;
default: //512
algorithm.SetAssetLimits(50, 10, 5);
break;
}
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Failed to initialize algorithm.");
return(false);
}
// find the correct brokerage factory based on the specified brokerage in the live job packet
var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
// initialize the correct brokerage using the resolved factory
brokerage = brokerageFactory.CreateBrokerage(liveJob, algorithm);
brokerage.Connect();
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var item in cashBalance)
{
algorithm.SetCash(item.Key, item.Value, 0);
}
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
order.Id = algorithm.Transactions.GetIncrementOrderId();
algorithm.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Min(x => x.Value.Resolution));
foreach (var holding in holdings)
{
if (!algorithm.Portfolio.ContainsKey(holding.Symbol))
{
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, holding.Symbol, minResolution.Value, true, 1.0m, false);
}
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity);
algorithm.Securities[holding.Symbol].Update(DateTime.Now, new TradeBar
{
Time = DateTime.Now,
Open = holding.AveragePrice,
High = holding.AveragePrice,
Low = holding.AveragePrice,
Close = holding.AveragePrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
// call this after we've initialized everything from the brokerage since we may have added some holdings/currencies
algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager);
//Set the starting capital for the strategy to calculate performance:
StartingCapital = algorithm.Portfolio.Cash;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
return(Errors.Count == 0);
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="parameters">The parameters object to use</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(SetupHandlerParameters parameters)
{
var algorithm = parameters.Algorithm;
var brokerage = parameters.Brokerage;
// verify we were given the correct job packet type
var liveJob = parameters.AlgorithmNodePacket as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return(false);
}
algorithm.Name = liveJob.GetAlgorithmName();
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return(false);
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError($"Brokerage Error Code: {args.Code} - {args.Message}");
}
};
try
{
// let the world know what we're doing since logging in can take a minute
parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage...");
brokerage.Message += brokerageOnMessage;
Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage...");
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError(
$"Error connecting to brokerage: {err.Message}. " +
"This may be caused by incorrect login credentials or an unsupported account type.", err);
return(false);
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return(false);
}
var message = $"{brokerage.Name} account base currency: {brokerage.AccountBaseCurrency ?? algorithm.AccountCurrency}";
var accountCurrency = brokerage.AccountBaseCurrency;
if (liveJob.BrokerageData.ContainsKey(MaxAllocationLimitConfig))
{
accountCurrency = Currencies.USD;
message += ". Allocation limited, will use 'USD' account currency";
}
Log.Trace($"BrokerageSetupHandler.Setup(): {message}");
algorithm.Debug(message);
if (accountCurrency != null && accountCurrency != algorithm.AccountCurrency)
{
algorithm.SetAccountCurrency(accountCurrency);
}
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Execute the initialize code:
var controls = liveJob.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () =>
{
try
{
//Set the default brokerage model before initialize
algorithm.SetBrokerageModel(_factory.GetBrokerageModel(algorithm.Transactions));
//Margin calls are disabled by default in live mode
algorithm.Portfolio.MarginCallModel = MarginCallModel.Null;
//Set our parameters
algorithm.SetParameters(liveJob.Parameters);
algorithm.SetAvailableDataTypes(BaseSetupHandler.GetConfiguredDataFeeds());
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler);
var optionChainProvider = Composer.Instance.GetPart <IOptionChainProvider>();
if (optionChainProvider == null)
{
optionChainProvider = new CachingOptionChainProvider(new LiveOptionChainProvider());
}
// set the option chain provider
algorithm.SetOptionChainProvider(optionChainProvider);
var futureChainProvider = Composer.Instance.GetPart <IFutureChainProvider>();
if (futureChainProvider == null)
{
futureChainProvider = new CachingFutureChainProvider(new LiveFutureChainProvider());
}
// set the future chain provider
algorithm.SetFutureChainProvider(futureChainProvider);
// set the object store
algorithm.SetObjectStore(parameters.ObjectStore);
// If we're going to receive market data from IB,
// set the default subscription limit to 100,
// algorithms can override this setting in the Initialize method
if (brokerage is InteractiveBrokersBrokerage &&
liveJob.DataQueueHandler.EndsWith("InteractiveBrokersBrokerage"))
{
algorithm.Settings.DataSubscriptionLimit = 100;
}
//Initialise the algorithm, get the required data:
algorithm.Initialize();
if (liveJob.Brokerage != "PaperBrokerage")
{
//Zero the CashBook - we'll populate directly from brokerage
foreach (var kvp in algorithm.Portfolio.CashBook)
{
kvp.Value.SetAmount(0);
}
}
}
catch (Exception err)
{
AddInitializationError(err.ToString(), err);
}
}, controls.RamAllocation,
sleepIntervalMillis: 100); // entire system is waiting on this, so be as fast as possible
if (Errors.Count != 0)
{
// if we already got an error just exit right away
return(false);
}
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return(false);
}
if (!LoadCashBalance(brokerage, algorithm))
{
return(false);
}
if (!LoadExistingHoldingsAndOrders(brokerage, algorithm, parameters))
{
return(false);
}
//Finalize Initialization
algorithm.PostInitialize();
BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection);
if (algorithm.Portfolio.TotalPortfolioValue == 0)
{
algorithm.Debug("Warning: No cash balances or holdings were found in the brokerage account.");
}
string maxCashLimitStr;
if (liveJob.BrokerageData.TryGetValue(MaxAllocationLimitConfig, out maxCashLimitStr))
{
var maxCashLimit = decimal.Parse(maxCashLimitStr, NumberStyles.Any, CultureInfo.InvariantCulture);
// If allocation exceeded by more than $10,000; block deployment
if (algorithm.Portfolio.TotalPortfolioValue > (maxCashLimit + 10000m))
{
var exceptionMessage = $"TotalPortfolioValue '{algorithm.Portfolio.TotalPortfolioValue}' exceeds allocation limit '{maxCashLimit}'";
algorithm.Debug(exceptionMessage);
throw new ArgumentException(exceptionMessage);
}
}
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
// we set the free portfolio value based on the initial total value and the free percentage value
algorithm.Settings.FreePortfolioValue =
algorithm.Portfolio.TotalPortfolioValue * algorithm.Settings.FreePortfolioValuePercentage;
}
catch (Exception err)
{
AddInitializationError(err.ToString(), err);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return(Errors.Count == 0);
}
/// <summary>
/// Setup the algorithm cash, dates and data subscriptions as desired.
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">Brokerage instance</param>
/// <param name="baseJob">Algorithm job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <returns>Boolean true on successfully initializing the algorithm</returns>
public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler)
{
var job = baseJob as BacktestNodePacket;
if (job == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name);
}
Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource));
brokerage = null;
if (algorithm == null)
{
Errors.Add("Could not create instance of algorithm");
return false;
}
//Make sure the algorithm start date ok.
if (job.PeriodStart == default(DateTime))
{
Errors.Add("Algorithm start date was never set");
return false;
}
//Execute the initialize code:
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () =>
{
try
{
//Algorithm is backtesting, not live:
algorithm.SetLiveMode(false);
//Set the backtest level asset ram allocation limits
algorithm.SetAssetLimits(500, 100, 30);
//Set the algorithm time before we even initialize:
algorithm.SetDateTime(job.PeriodStart);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
//Add currency data feeds that weren't explicity added in Initialize
algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager);
}
catch (Exception err)
{
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
});
//Before continuing, detect if this is ready:
if (!initializeComplete) return false;
// this needs to be done after algorithm initialization
brokerage = new BacktestingBrokerage(algorithm);
SetupHandler.UpdateTransactionModels(algorithm, algorithm.BrokerageModel);
algorithm.Transactions.SetOrderProcessor(transactionHandler);
//Calculate the max runtime for the strategy
_maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count);
//Get starting capital:
_startingCaptial = algorithm.Portfolio.Cash;
//Max Orders: 10k per backtest:
if (job.UserPlan == UserPlan.Free)
{
_maxOrders = 10000;
}
else
{
_maxOrders = int.MaxValue;
_maxRuntime += _maxRuntime;
}
//Set back to the algorithm,
algorithm.SetMaximumOrders(_maxOrders);
//Starting date of the algorithm:
_startingDate = job.PeriodStart;
//Put into log for debugging:
Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId);
Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C"));
if (Errors.Count > 0)
{
initializeComplete = false;
}
return initializeComplete;
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
_algorithm = algorithm;
brokerage = default(IBrokerage);
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return(false);
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return(false);
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message));
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
resultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Initializing, "Initializing algorithm...");
//Execute the initialize code:
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () =>
{
try
{
//Set the live trading level asset/ram allocation limits.
//Protects algorithm from linux killing the job by excess memory:
switch (job.ServerType)
{
case ServerType.Server1024:
algorithm.SetAssetLimits(100, 20, 10);
break;
case ServerType.Server2048:
algorithm.SetAssetLimits(400, 50, 30);
break;
default: //512
algorithm.SetAssetLimits(50, 25, 15);
break;
}
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return(false);
}
try
{
// find the correct brokerage factory based on the specified brokerage in the live job packet
_factory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
}
catch (Exception err)
{
Log.Error("BrokerageSetupHandler.Setup(): Error resolving brokerage factory for " + liveJob.Brokerage + ". " + err.Message);
AddInitializationError("Unable to locate factory for brokerage: " + liveJob.Brokerage);
}
// let the world know what we're doing since logging in can take a minute
resultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.LoggingIn, "Logging into brokerage...");
// initialize the correct brokerage using the resolved factory
brokerage = _factory.CreateBrokerage(liveJob, algorithm);
if (brokerage == null)
{
AddInitializationError("Failed to create instance of brokerage: " + liveJob.Brokerage);
return(false);
}
brokerage.Message += brokerageOnMessage;
// set the transaction and settlement models based on the brokerage properties
SetupHandler.UpdateModels(algorithm, algorithm.BrokerageModel);
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage...");
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError(string.Format("Error connecting to brokerage: {0}. " +
"This may be caused by incorrect login credentials or an unsupported account type.", err.Message));
return(false);
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage...");
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Quantity);
algorithm.SetCash(cash.Symbol, cash.Quantity, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage...");
try
{
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol + " - " + order.Quantity);
order.Id = algorithm.Transactions.GetIncrementOrderId();
transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage...");
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var supportedSecurityTypes = new HashSet <SecurityType> {
SecurityType.Equity, SecurityType.Forex
};
var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Min(x => x.Value.Resolution));
foreach (var holding in holdings)
{
var symbol = new Symbol(holding.Symbol);
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
// verify existing holding security type
if (!supportedSecurityTypes.Contains(holding.Type))
{
Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.ToUpper());
AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " +
"QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes));
// keep aggregating these errors
continue;
}
if (!algorithm.Portfolio.ContainsKey(symbol))
{
Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol);
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, symbol, minResolution.Value, null, true, 1.0m, false);
}
algorithm.Portfolio[symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity);
algorithm.Securities[symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Ensuring currency data feeds present...");
// call this after we've initialized everything from the brokerage since we may have added some holdings/currencies
algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager, SecurityExchangeHoursProvider.FromDataFolder());
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return(Errors.Count == 0);
}
/// <summary>
/// Setup the algorithm cash, dates and data subscriptions as desired.
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">Brokerage instance</param>
/// <param name="baseJob">Algorithm job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>Boolean true on successfully initializing the algorithm</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
var job = baseJob as BacktestNodePacket;
if (job == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name);
}
Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource));
if (algorithm == null)
{
Errors.Add("Could not create instance of algorithm");
return false;
}
//Make sure the algorithm start date ok.
if (job.PeriodStart == default(DateTime))
{
Errors.Add("Algorithm start date was never set");
return false;
}
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () =>
{
try
{
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is backtesting, not live:
algorithm.SetLiveMode(false);
//Set the algorithm time before we even initialize:
algorithm.SetDateTime(job.PeriodStart.ConvertToUtc(algorithm.TimeZone));
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
}
catch (Exception err)
{
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
});
//Before continuing, detect if this is ready:
if (!initializeComplete) return false;
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
//Calculate the max runtime for the strategy
_maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count);
//Get starting capital:
_startingCaptial = algorithm.Portfolio.Cash;
//Max Orders: 10k per backtest:
if (job.UserPlan == UserPlan.Free)
{
_maxOrders = 10000;
}
else
{
_maxOrders = int.MaxValue;
_maxRuntime += _maxRuntime;
}
//Set back to the algorithm,
algorithm.SetMaximumOrders(_maxOrders);
//Starting date of the algorithm:
_startingDate = job.PeriodStart;
//Put into log for debugging:
Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId);
Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C"));
if (Errors.Count > 0)
{
initializeComplete = false;
}
return initializeComplete;
}
/// <summary>
/// Creates a new instance of the class in the library, safely.
/// </summary>
/// <param name="assemblyPath">Location of the DLL</param>
/// <param name="algorithmInstance">Output algorithm instance</param>
/// <param name="errorMessage">Output error message on failure</param>
/// <returns>bool success</returns>
public bool TryCreateAlgorithmInstanceWithIsolator(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
IAlgorithm instance = null;
var error = string.Empty;
var success = false;
var isolator = new Isolator();
var complete = isolator.ExecuteWithTimeLimit(_loaderTimeLimit, () =>
{
success = TryCreateAlgorithmInstance(assemblyPath, out instance, out error);
});
algorithmInstance = instance;
errorMessage = error;
// if the isolator stopped us early add that to our error message
if (!complete)
{
errorMessage = "Failed to create algorithm instance within 10 seconds. Try re-building algorithm. " + error;
}
return complete && success && algorithmInstance != null;
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Primary Analysis Thread:
/// </summary>
public static void Main(string[] args)
{
//Initialize:
AlgorithmNodePacket job = null;
var timer = Stopwatch.StartNew();
var algorithm = default(IAlgorithm);
_version = DateTime.ParseExact(Config.Get("version", DateTime.Now.ToString(DateFormat.UI)), DateFormat.UI, CultureInfo.InvariantCulture);
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + _version);
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
var catalog = new AggregateCatalog();
catalog.Catalogs.Add(new DirectoryCatalog(@"../../Extensions"));
var container = new CompositionContainer(catalog);
try
{
// grab the right export based on configuration
Notify = container.GetExportedValueByTypeName <IMessagingHandler>(Config.Get("messaging-handler"));
Queue = container.GetExportedValueByTypeName <IQueueHandler>(Config.Get("queue-handler"));
Api = container.GetExportedValueByTypeName <IApi>(Config.Get("api-handler"));
}
catch (CompositionException compositionException)
{ Log.Error("Engine.Main(): Failed to load library: " + compositionException); }
//Setup packeting, queue and controls system: These don't do much locally.
Api.Initialize();
Notify.Initialize();
Queue.Initialize(_liveMode);
//Start monitoring the backtest active status:
var statusPingThread = new Thread(StateCheck.Ping.Run);
statusPingThread.Start();
do
{
try
{
//Reset algo manager internal variables preparing for a new algorithm.
AlgorithmManager.ResetManager();
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
//-> Pull job from QuantConnect job queue, or, pull local build:
var algorithmPath = "";
job = Queue.NextJob(out algorithmPath); // Blocking.
//-> Initialize messaging system
Notify.SetChannel(job.Channel);
//-> Reset the backtest stopwatch; we're now running the algorithm.
timer.Restart();
//-> Create SetupHandler to configure internal algorithm state:
SetupHandler = GetSetupHandler(job.SetupEndpoint);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
ResultHandler = GetResultHandler(job);
threadResults = new Thread(ResultHandler.Run, 0)
{
Name = "Result Thread"
};
threadResults.Start();
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", SetupHandler.Errors);
throw new Exception(errorMessage);
}
}
catch (Exception err)
{
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
ResultHandler.RuntimeError(runtimeMessage, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLiveMode(LiveMode);
algorithm.SetLocked();
//Load the associated handlers for data, transaction and realtime events:
ResultHandler.SetAlgorithm(algorithm);
DataFeed = GetDataFeedHandler(algorithm, job);
TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job);
RealTimeHandler = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job);
//Set the error handlers for the brokerage asynchronous errors.
SetupHandler.SetupErrorHandler(ResultHandler, _brokerage);
//Send status to user the algorithm is now executing.
ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(DataFeed.Run, 0)
{
Name = "DataFeed Thread"
};
threadTransactions = new Thread(TransactionHandler.Run, 0)
{
Name = "Transaction Thread"
};
threadRealTime = new Thread(RealTimeHandler.Run, 0)
{
Name = "RealTime Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
try
{
// Execute the Algorithm Code:
var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error("Engine.Run(): Error in Algo Manager: " + err.Message + " ST >> " + err.StackTrace);
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, MaximumRamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (AlgorithmManager.RunTimeError != null)
{
throw AlgorithmManager.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (DataFeed != null)
{
DataFeed.Exit();
}
if (ResultHandler != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
ResultHandler.LogMessage(message);
ResultHandler.RuntimeError(message, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var charts = new Dictionary <string, Chart>(ResultHandler.Charts);
var orders = new Dictionary <int, Order>(algorithm.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var statistics = new Dictionary <string, string>();
var banner = new Dictionary <string, string>();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss =
new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
statistics = Statistics.Statistics.Generate(equity, profitLoss, performance,
SetupHandler.StartingCapital, 252);
}
}
catch (Exception err)
{
Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
}
//Diagnostics Completed, Send Result Packet:
ResultHandler.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") completed analysis in " + timer.Elapsed.TotalSeconds.ToString("F2") + " seconds");
ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
timer.Stop(); //Algorithm finished running.
TransactionHandler.Exit();
DataFeed.Exit();
RealTimeHandler.Exit();
}
//Close result handler:
ResultHandler.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100); Log.Trace("Waiting for threads to exit...");
}
if (threadFeed != null && threadFeed.IsAlive)
{
threadFeed.Abort();
}
if (threadTransactions != null && threadTransactions.IsAlive)
{
threadTransactions.Abort();
}
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//Delete the message from the job queue:
Queue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (LiveMode && AlgorithmManager.State != AlgorithmStatus.Running && AlgorithmManager.State != AlgorithmStatus.RuntimeError)
{
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmManager.State);
}
//Attempt to clean up ram usage:
GC.Collect();
}
//If we're running locally will execute just once.
} while (!IsLocal);
// Send the exit signal and then kill the thread
StateCheck.Ping.Exit();
// Make the console window pause so we can read log output before exiting and killing the application completely
Console.ReadKey();
//Finally if ping thread still not complete, kill.
if (statusPingThread != null && statusPingThread.IsAlive)
{
statusPingThread.Abort();
}
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
_algorithm = algorithm;
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return(false);
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return(false);
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message));
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
resultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Execute the initialize code:
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () =>
{
try
{
//Set the default brokerage model before initialize
algorithm.SetBrokerageModel(_factory.BrokerageModel);
//Set our default markets
algorithm.SetDefaultMarkets(_factory.DefaultMarkets.ToDictionary());
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
if (liveJob.Brokerage != "PaperBrokerage")
{
//Zero the CashBook - we'll populate directly from brokerage
foreach (var kvp in algorithm.Portfolio.CashBook)
{
kvp.Value.SetAmount(0);
}
}
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return(false);
}
// let the world know what we're doing since logging in can take a minute
resultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage...");
brokerage.Message += brokerageOnMessage;
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage...");
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError(string.Format("Error connecting to brokerage: {0}. " +
"This may be caused by incorrect login credentials or an unsupported account type.", err.Message));
return(false);
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage...");
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Amount);
algorithm.Portfolio.SetCash(cash.Symbol, cash.Amount, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage...");
try
{
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol.ToString() + " - " + order.Quantity);
order.Id = algorithm.Transactions.GetIncrementOrderId();
transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage...");
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var supportedSecurityTypes = new HashSet <SecurityType> {
SecurityType.Equity, SecurityType.Forex
};
var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min());
foreach (var holding in holdings)
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
// verify existing holding security type
if (!supportedSecurityTypes.Contains(holding.Type))
{
Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value);
AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " +
"QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes));
// keep aggregating these errors
continue;
}
if (!algorithm.Portfolio.ContainsKey(holding.Symbol))
{
Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol.ToString());
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, holding.Symbol.Value, minResolution.Value, null, true, 1.0m, false);
}
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message);
return(false);
}
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return(Errors.Count == 0);
}
/// <summary>
/// Setup the algorithm cash, dates and data subscriptions as desired.
/// </summary>
/// <param name="parameters">The parameters object to use</param>
/// <returns>Boolean true on successfully initializing the algorithm</returns>
public bool Setup(SetupHandlerParameters parameters)
{
var algorithm = parameters.Algorithm;
var job = parameters.AlgorithmNodePacket as BacktestNodePacket;
if (job == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + parameters.AlgorithmNodePacket.GetType().Name);
}
Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource));
if (algorithm == null)
{
Errors.Add(new AlgorithmSetupException("Could not create instance of algorithm"));
return(false);
}
algorithm.Name = job.GetAlgorithmName();
//Make sure the algorithm start date ok.
if (job.PeriodStart == default(DateTime))
{
Errors.Add(new AlgorithmSetupException("Algorithm start date was never set"));
return(false);
}
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () =>
{
try
{
parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is backtesting, not live:
algorithm.SetLiveMode(false);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler);
// set the option chain provider
algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
// set the future chain provider
algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
//Initialise the algorithm, get the required data:
algorithm.Initialize();
// finalize initialization
algorithm.PostInitialize();
}
catch (Exception err)
{
Log.Error(err);
Errors.Add(new AlgorithmSetupException("During the algorithm initialization, the following exception has occurred: ", err));
}
}, controls.RamAllocation,
sleepIntervalMillis: 50); // entire system is waiting on this, so be as fast as possible
//Before continuing, detect if this is ready:
if (!initializeComplete)
{
return(false);
}
// TODO: Refactor the BacktestResultHandler to use algorithm not job to set times
job.PeriodStart = algorithm.StartDate;
job.PeriodFinish = algorithm.EndDate;
//Calculate the max runtime for the strategy
_maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager, algorithm.UniverseManager, parameters.AlgorithmNodePacket.Controls);
// Python takes forever; lets give it 10x longer to finish.
if (job.Language == Language.Python)
{
_maxRuntime = _maxRuntime.Add(TimeSpan.FromSeconds(_maxRuntime.TotalSeconds * 9));
}
BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection);
StartingPortfolioValue = algorithm.Portfolio.Cash;
//Max Orders: 10k per backtest:
if (job.UserPlan == UserPlan.Free)
{
_maxOrders = 10000;
}
else
{
_maxOrders = int.MaxValue;
_maxRuntime += _maxRuntime;
}
//Set back to the algorithm,
algorithm.SetMaximumOrders(_maxOrders);
//Starting date of the algorithm:
_startingDate = job.PeriodStart;
//Put into log for debugging:
Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId);
Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + StartingPortfolioValue.ToString("C"));
if (Errors.Count > 0)
{
initializeComplete = false;
}
return(initializeComplete);
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="parameters">The parameters object to use</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(SetupHandlerParameters parameters)
{
var algorithm = parameters.Algorithm;
var brokerage = parameters.Brokerage;
// verify we were given the correct job packet type
var liveJob = parameters.AlgorithmNodePacket as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return(false);
}
algorithm.Name = liveJob.GetAlgorithmName();
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return(false);
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError($"Brokerage Error Code: {args.Code} - {args.Message}");
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Execute the initialize code:
var controls = liveJob.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () =>
{
try
{
//Set the default brokerage model before initialize
algorithm.SetBrokerageModel(_factory.BrokerageModel);
//Margin calls are disabled by default in live mode
algorithm.Portfolio.MarginCallModel = MarginCallModel.Null;
//Set our parameters
algorithm.SetParameters(liveJob.Parameters);
algorithm.SetAvailableDataTypes(GetConfiguredDataFeeds());
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler);
// set the option chain provider
algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new LiveOptionChainProvider()));
// set the future chain provider
algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new LiveFutureChainProvider()));
// If we're going to receive market data from IB,
// set the default subscription limit to 100,
// algorithms can override this setting in the Initialize method
if (brokerage is InteractiveBrokersBrokerage &&
liveJob.DataQueueHandler.EndsWith("InteractiveBrokersBrokerage"))
{
algorithm.Settings.DataSubscriptionLimit = 100;
}
//Initialise the algorithm, get the required data:
algorithm.Initialize();
if (algorithm.AccountCurrency != Currencies.USD)
{
throw new NotImplementedException(
"BrokerageSetupHandler.Setup(): calling 'QCAlgorithm.SetAccountCurrency()' " +
"is only supported in backtesting for now.");
}
if (liveJob.Brokerage != "PaperBrokerage")
{
//Zero the CashBook - we'll populate directly from brokerage
foreach (var kvp in algorithm.Portfolio.CashBook)
{
kvp.Value.SetAmount(0);
}
}
}
catch (Exception err)
{
AddInitializationError(err.ToString(), err);
}
}, controls.RamAllocation,
sleepIntervalMillis: 50); // entire system is waiting on this, so be as fast as possible
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return(false);
}
// let the world know what we're doing since logging in can take a minute
parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage...");
brokerage.Message += brokerageOnMessage;
Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage...");
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError(
$"Error connecting to brokerage: {err.Message}. " +
"This may be caused by incorrect login credentials or an unsupported account type.", err);
return(false);
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage...");
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Currency + " cash to " + cash.Amount);
algorithm.Portfolio.SetCash(cash.Currency, cash.Amount, 0);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message, err);
return(false);
}
var supportedSecurityTypes = new HashSet <SecurityType>
{
SecurityType.Equity, SecurityType.Forex, SecurityType.Cfd, SecurityType.Option, SecurityType.Future, SecurityType.Crypto
};
var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min());
Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage...");
try
{
GetOpenOrders(algorithm, parameters.ResultHandler, parameters.TransactionHandler, brokerage, supportedSecurityTypes, minResolution.Value);
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message, err);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage...");
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
// add options first to ensure raw data normalization mode is set on the equity underlyings
foreach (var holding in holdings.OrderByDescending(x => x.Type))
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
// verify existing holding security type
if (!supportedSecurityTypes.Contains(holding.Type))
{
Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value);
AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " +
"QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes));
// keep aggregating these errors
continue;
}
AddUnrequestedSecurity(algorithm, holding.Symbol, minResolution.Value);
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message, err);
return(false);
}
//Finalize Initialization
algorithm.PostInitialize();
BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection);
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.ToString(), err);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return(Errors.Count == 0);
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
_algorithm = algorithm;
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return false;
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return false;
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler<BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message));
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
resultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Execute the initialize code:
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () =>
{
try
{
//Set the default brokerage model before initialize
algorithm.SetBrokerageModel(_factory.BrokerageModel);
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
if (liveJob.Brokerage != "PaperBrokerage")
{
//Zero the CashBook - we'll populate directly from brokerage
foreach (var kvp in algorithm.Portfolio.CashBook)
{
kvp.Value.SetAmount(0);
}
}
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return false;
}
// let the world know what we're doing since logging in can take a minute
resultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage...");
brokerage.Message += brokerageOnMessage;
algorithm.Transactions.SetOrderProcessor(transactionHandler);
Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage...");
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError(string.Format("Error connecting to brokerage: {0}. " +
"This may be caused by incorrect login credentials or an unsupported account type.", err.Message));
return false;
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return false;
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage...");
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Amount);
algorithm.Portfolio.SetCash(cash.Symbol, cash.Amount, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message);
return false;
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage...");
try
{
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol.ToString() + " - " + order.Quantity);
order.Id = algorithm.Transactions.GetIncrementOrderId();
transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message);
return false;
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage...");
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var supportedSecurityTypes = new HashSet<SecurityType> { SecurityType.Equity, SecurityType.Forex, SecurityType.Cfd };
var minResolution = new Lazy<Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min());
foreach (var holding in holdings)
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
// verify existing holding security type
if (!supportedSecurityTypes.Contains(holding.Type))
{
Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value);
AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " +
"QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes));
// keep aggregating these errors
continue;
}
if (!algorithm.Portfolio.ContainsKey(holding.Symbol))
{
Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol.ToString());
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, holding.Symbol.Value, minResolution.Value, null, true, 1.0m, false);
}
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int) holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message);
return false;
}
algorithm.PostInitialize();
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return Errors.Count == 0;
}
/// <summary>
/// Setup the algorithm cash, dates and data subscriptions as desired.
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">Brokerage instance</param>
/// <param name="baseJob">Algorithm job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <returns>Boolean true on successfully initializing the algorithm</returns>
public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler)
{
var job = baseJob as BacktestNodePacket;
if (job == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name);
}
Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource));
brokerage = null;
if (algorithm == null)
{
Errors.Add("Could not create instance of algorithm");
return(false);
}
//Make sure the algorithm start date ok.
if (job.PeriodStart == default(DateTime))
{
Errors.Add("Algorithm start date was never set");
return(false);
}
//Execute the initialize code:
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () =>
{
try
{
//Algorithm is backtesting, not live:
algorithm.SetLiveMode(false);
//Set the backtest level asset ram allocation limits
algorithm.SetAssetLimits(500, 100, 30);
//Set the algorithm time before we even initialize:
algorithm.SetDateTime(job.PeriodStart);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
//Add currency data feeds that weren't explicity added in Initialize
algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager, SecurityExchangeHoursProvider.FromDataFolder());
}
catch (Exception err)
{
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
});
//Before continuing, detect if this is ready:
if (!initializeComplete)
{
return(false);
}
// this needs to be done after algorithm initialization
brokerage = new BacktestingBrokerage(algorithm);
SetupHandler.UpdateTransactionModels(algorithm, algorithm.BrokerageModel);
algorithm.Transactions.SetOrderProcessor(transactionHandler);
//Calculate the max runtime for the strategy
_maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count);
//Get starting capital:
_startingCaptial = algorithm.Portfolio.Cash;
//Max Orders: 10k per backtest:
if (job.UserPlan == UserPlan.Free)
{
_maxOrders = 10000;
}
else
{
_maxOrders = int.MaxValue;
_maxRuntime += _maxRuntime;
}
//Set back to the algorithm,
algorithm.SetMaximumOrders(_maxOrders);
//Starting date of the algorithm:
_startingDate = job.PeriodStart;
//Put into log for debugging:
Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId);
Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C"));
if (Errors.Count > 0)
{
initializeComplete = false;
}
return(initializeComplete);
}
/// <summary>
/// Setup the algorithm cash, dates and data subscriptions as desired.
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">Brokerage instance</param>
/// <param name="baseJob">Algorithm job</param>
/// <returns>Boolean true on successfully initializing the algorithm</returns>
public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket baseJob)
{
var job = baseJob as BacktestNodePacket;
if (job == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name);
}
// Must be set since its defined as an out parameters
brokerage = new BacktestingBrokerage(algorithm);
if (algorithm == null)
{
Errors.Add("Could not create instance of algorithm");
return(false);
}
//Make sure the algorithm start date ok.
if (job.PeriodStart == default(DateTime))
{
Errors.Add("Algorithm start date was never set");
return(false);
}
//Execute the initialize code:
var initializeComplete = Isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () =>
{
try
{
//0.0 Set the algorithm time before we even initialize:
algorithm.SetDateTime(job.PeriodStart);
//1.0 Initialise the algorithm, get the required data:
algorithm.Initialize();
//1.2 Set the algorithm to locked to avoid messing with cash:
algorithm.SetLocked();
}
catch (Exception err)
{
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
});
//Before continuing, detect if this is ready:
if (!initializeComplete)
{
return(false);
}
//Calculate the max runtime for the strategy
_maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count);
//Get starting capital:
_startingCaptial = algorithm.Portfolio.Cash;
//Max Orders: 100 per day:
_maxOrders = (int)(job.PeriodFinish - job.PeriodStart).TotalDays * 100;
//Starting date of the algorithm:
_startingDate = job.PeriodStart;
//Put into log for debugging:
Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId);
Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C"));
if (Errors.Count > 0)
{
initializeComplete = false;
}
return(initializeComplete);
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
public void Run(AlgorithmNodePacket job, string assemblyPath)
{
var algorithm = default(IAlgorithm);
var algorithmManager = new AlgorithmManager(_liveMode);
//Start monitoring the backtest active status:
var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results);
var statusPingThread = new Thread(statusPing.Run);
statusPingThread.Start();
try
{
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
//-> Initialize messaging system
_systemHandlers.Notify.SetChannel(job.Channel);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
_algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions);
threadResults = new Thread(_algorithmHandlers.Results.Run, 0) {Name = "Result Thread"};
threadResults.Start();
IBrokerage brokerage = null;
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, out brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors);
_algorithmHandlers.Results.RuntimeError(errorMessage);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
_algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLiveMode(_liveMode);
algorithm.SetLocked();
//Load the associated handlers for data, transaction and realtime events:
_algorithmHandlers.Results.SetAlgorithm(algorithm);
_algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results);
_algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results);
_algorithmHandlers.RealTime.Initialize(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
//Set the error handlers for the brokerage asynchronous errors.
_algorithmHandlers.Setup.SetupErrorHandler(_algorithmHandlers.Results, brokerage);
//Send status to user the algorithm is now executing.
_algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) {Name = "DataFeed Thread"};
threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) {Name = "Transaction Thread"};
threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) {Name = "RealTime Thread"};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
_algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version));
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error("Engine.Run", err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.RamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
throw algorithm.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (_algorithmHandlers.DataFeed != null) _algorithmHandlers.DataFeed.Exit();
if (_algorithmHandlers.Results != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
_algorithmHandlers.Results.LogMessage(message);
_algorithmHandlers.Results.RuntimeError(message, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var charts = new Dictionary<string, Chart>(_algorithmHandlers.Results.Charts);
var orders = new Dictionary<int, Order>(_algorithmHandlers.Transactions.Orders);
var holdings = new Dictionary<string, Holding>();
var statistics = new Dictionary<string, string>();
var banner = new Dictionary<string, string>();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
const string benchmarkKey = "Benchmark";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss =
new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord);
var numberOfTrades = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count();
var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values.ToDictionary(chartPoint => Time.UnixTimeStampToDateTime(chartPoint.x), chartPoint => chartPoint.y);
statistics = Statistics.Statistics.Generate(equity, profitLoss, performance, benchmark,
_algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, numberOfTrades, 252);
}
}
catch (Exception err)
{
Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
_algorithmHandlers.Results.DebugMessage(
string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
job.AlgorithmId, totalSeconds.ToString("F2"), ((algorithmManager.DataPoints/(double) 1000)/totalSeconds).ToString("F0"),
algorithmManager.DataPoints.ToString("N0")));
_algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.RealTime.Exit();
}
//Close result handler:
_algorithmHandlers.Results.Exit();
statusPing.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((_algorithmHandlers.Results.IsActive
|| (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive)
|| (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive)
|| (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive))
&& ts.ElapsedMilliseconds < 30*1000)
{
Thread.Sleep(100);
Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort();
if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort();
if (threadResults != null && threadResults.IsAlive) threadResults.Abort();
if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort();
if (brokerage != null)
{
brokerage.Disconnect();
}
if (_algorithmHandlers.Setup != null)
{
_algorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
_algorithmHandlers.Results.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Setup the algorithm cash, dates and data subscriptions as desired.
/// </summary>
/// <param name="parameters">The parameters object to use</param>
/// <returns>Boolean true on successfully initializing the algorithm</returns>
public bool Setup(SetupHandlerParameters parameters)
{
var algorithm = parameters.Algorithm;
var job = parameters.AlgorithmNodePacket as BacktestNodePacket;
if (job == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + parameters.AlgorithmNodePacket.GetType().Name);
}
Log.Trace($"BacktestingSetupHandler.Setup(): Setting up job: Plan: {job.UserPlan}, UID: {job.UserId.ToStringInvariant()}, " +
$"PID: {job.ProjectId.ToStringInvariant()}, Version: {job.Version}, Source: {job.RequestSource}"
);
if (algorithm == null)
{
Errors.Add(new AlgorithmSetupException("Could not create instance of algorithm"));
return(false);
}
algorithm.Name = job.GetAlgorithmName();
//Make sure the algorithm start date ok.
if (job.PeriodStart == default(DateTime))
{
Errors.Add(new AlgorithmSetupException("Algorithm start date was never set"));
return(false);
}
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () =>
{
try
{
parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is backtesting, not live:
algorithm.SetLiveMode(false);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler);
// set the option chain provider
algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
// set the future chain provider
algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
// set the object store
algorithm.SetObjectStore(parameters.ObjectStore);
// before we call initialize
BaseSetupHandler.LoadBacktestJobAccountCurrency(algorithm, job);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
// set start and end date if present in the job
if (job.PeriodStart.HasValue)
{
algorithm.SetStartDate(job.PeriodStart.Value);
}
if (job.PeriodFinish.HasValue)
{
algorithm.SetEndDate(job.PeriodFinish.Value);
}
// after we call initialize
BaseSetupHandler.LoadBacktestJobCashAmount(algorithm, job);
// finalize initialization
algorithm.PostInitialize();
}
catch (Exception err)
{
Log.Error(err);
Errors.Add(new AlgorithmSetupException("During the algorithm initialization, the following exception has occurred: ", err));
}
}, controls.RamAllocation,
sleepIntervalMillis: 10, // entire system is waiting on this, so be as fast as possible
workerThread: WorkerThread);
//Before continuing, detect if this is ready:
if (!initializeComplete)
{
return(false);
}
//Calculate the max runtime for the strategy
MaximumRuntime = GetMaximumRuntime(algorithm.StartDate, algorithm.EndDate, algorithm.SubscriptionManager, algorithm.UniverseManager, parameters.AlgorithmNodePacket.Controls);
// Python takes forever; lets give it 10x longer to finish.
if (job.Language == Language.Python)
{
MaximumRuntime = MaximumRuntime.Add(TimeSpan.FromSeconds(MaximumRuntime.TotalSeconds * 9));
}
BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection);
StartingPortfolioValue = algorithm.Portfolio.Cash;
// we set the free portfolio value based on the initial total value and the free percentage value
algorithm.Settings.FreePortfolioValue =
algorithm.Portfolio.TotalPortfolioValue * algorithm.Settings.FreePortfolioValuePercentage;
//Max Orders: 10k per backtest:
if (job.UserPlan == UserPlan.Free)
{
MaxOrders = 10000;
}
else
{
MaxOrders = int.MaxValue;
MaximumRuntime += MaximumRuntime;
}
MaxOrders = job.Controls.BacktestingMaxOrders;
//Set back to the algorithm,
algorithm.SetMaximumOrders(MaxOrders);
//Starting date of the algorithm:
StartingDate = algorithm.StartDate;
//Put into log for debugging:
Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId);
Log.Trace($"Dates: Start: {algorithm.StartDate.ToStringInvariant("d")} " +
$"End: {algorithm.EndDate.ToStringInvariant("d")} " +
$"Cash: {StartingPortfolioValue.ToStringInvariant("C")}");
if (Errors.Count > 0)
{
initializeComplete = false;
}
return(initializeComplete);
}
/// <summary>
/// Setup the algorithm cash, dates and data subscriptions as desired.
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">Brokerage instance</param>
/// <param name="baseJob">Algorithm job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>Boolean true on successfully initializing the algorithm</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
var job = baseJob as BacktestNodePacket;
if (job == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name);
}
Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource));
if (algorithm == null)
{
Errors.Add("Could not create instance of algorithm");
return(false);
}
//Make sure the algorithm start date ok.
if (job.PeriodStart == default(DateTime))
{
Errors.Add("Algorithm start date was never set");
return(false);
}
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () =>
{
try
{
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is backtesting, not live:
algorithm.SetLiveMode(false);
//Set the algorithm time before we even initialize:
algorithm.SetDateTime(job.PeriodStart.ConvertToUtc(algorithm.TimeZone));
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
}
catch (Exception err)
{
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
});
//Before continuing, detect if this is ready:
if (!initializeComplete)
{
return(false);
}
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
//Calculate the max runtime for the strategy
_maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count);
//Get starting capital:
_startingCaptial = algorithm.Portfolio.Cash;
//Max Orders: 10k per backtest:
if (job.UserPlan == UserPlan.Free)
{
_maxOrders = 10000;
}
else
{
_maxOrders = int.MaxValue;
_maxRuntime += _maxRuntime;
}
//Set back to the algorithm,
algorithm.SetMaximumOrders(_maxOrders);
//Starting date of the algorithm:
_startingDate = job.PeriodStart;
//Put into log for debugging:
Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId);
Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C"));
if (Errors.Count > 0)
{
initializeComplete = false;
}
return(initializeComplete);
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="manager"></param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath)
{
var algorithm = default(IAlgorithm);
var algorithmManager = manager;
try
{
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
//-> Initialize messaging system
_systemHandlers.Notify.SetAuthentication(job);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
_algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions);
threadResults = new Thread(_algorithmHandlers.Results.Run, 0)
{
IsBackground = true, Name = "Result Thread"
};
threadResults.Start();
IBrokerage brokerage = null;
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath);
// Set algorithm in ILeanManager
_systemHandlers.LeanManager.SetAlgorithm(algorithm);
// Initialize the brokerage
IBrokerageFactory factory;
brokerage = _algorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory);
// Initialize the data feed before we initialize so he can intercept added securities/universes via events
_algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider, _algorithmHandlers.DataProvider);
// set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider)
algorithm.Transactions.SetOrderProcessor(_algorithmHandlers.Transactions);
// set the history provider before setting up the algorithm
var historyProvider = GetHistoryProvider(job.HistoryProvider);
if (historyProvider is BrokerageHistoryProvider)
{
(historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage);
}
var historyDataCacheProvider = new ZipDataCacheProvider(_algorithmHandlers.DataProvider);
historyProvider.Initialize(job, _algorithmHandlers.DataProvider, historyDataCacheProvider, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider, progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
_algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History,
string.Format("Processing history {0}%...", progress));
}
});
algorithm.HistoryProvider = historyProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, _systemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime);
// set this again now that we've actually added securities
_algorithmHandlers.Results.SetAlgorithm(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors);
Log.Error("Engine.Run(): " + errorMessage);
_algorithmHandlers.Results.RuntimeError(errorMessage);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
Log.Error(err);
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
_algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + _algorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + _algorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + _algorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + _algorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + _algorithmHandlers.Transactions.GetType().FullName);
if (algorithm != null && algorithm.HistoryProvider != null)
{
Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName);
}
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + brokerage.GetType().FullName);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Load the associated handlers for transaction and realtime events:
_algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results);
_algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
// wire up the brokerage message handler
brokerage.Message += (sender, message) =>
{
algorithm.BrokerageMessageHandler.Handle(message);
// fire brokerage message events
algorithm.OnBrokerageMessage(message);
switch (message.Type)
{
case BrokerageMessageType.Disconnect:
algorithm.OnBrokerageDisconnect();
break;
case BrokerageMessageType.Reconnect:
algorithm.OnBrokerageReconnect();
break;
}
};
//Send status to user the algorithm is now executing.
_algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(_algorithmHandlers.DataFeed.Run)
{
IsBackground = true, Name = "DataFeed Thread"
};
threadTransactions = new Thread(_algorithmHandlers.Transactions.Run)
{
IsBackground = true, Name = "Transaction Thread"
};
threadRealTime = new Thread(_algorithmHandlers.RealTime.Run)
{
IsBackground = true, Name = "RealTime Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
_algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Globals.Version));
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, _systemHandlers.LeanManager, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.Controls.RamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
HandleAlgorithmError(job, algorithm.RunTimeError);
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
HandleAlgorithmError(job, err);
}
try
{
var trades = algorithm.TradeBuilder.ClosedTrades;
var charts = new Dictionary <string, Chart>(_algorithmHandlers.Results.Charts);
var orders = new Dictionary <int, Order>(_algorithmHandlers.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var banner = new Dictionary <string, string>();
var statisticsResults = new StatisticsResults();
var csvTransactionsFileName = Config.Get("transaction-log");
if (!string.IsNullOrEmpty(csvTransactionsFileName))
{
SaveListOfTrades(_algorithmHandlers.Transactions, csvTransactionsFileName);
}
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
const string benchmarkKey = "Benchmark";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey) &&
charts.ContainsKey(benchmarkKey) &&
charts[benchmarkKey].Series.ContainsKey(benchmarkKey)
)
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count();
var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values;
statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark,
_algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions);
//Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors
var netReturn = _algorithmHandlers.Setup.StartingPortfolioValue > 0 ?
(algorithm.Portfolio.TotalPortfolioValue - _algorithmHandlers.Setup.StartingPortfolioValue) / _algorithmHandlers.Setup.StartingPortfolioValue
: 0;
//Add other fixed parameters.
banner.Add("Unrealized", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2"));
banner.Add("Fees", "-$" + algorithm.Portfolio.TotalFees.ToString("N2"));
banner.Add("Net Profit", "$" + algorithm.Portfolio.TotalProfit.ToString("N2"));
banner.Add("Return", netReturn.ToString("P"));
banner.Add("Equity", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2"));
}
}
catch (Exception err)
{
Log.Error(err, "Error generating statistics packet");
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount;
_algorithmHandlers.Results.DebugMessage(
string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints / (double)1000) / totalSeconds).ToString("F0"),
dataPoints.ToString("N0")));
_algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, algorithm.Portfolio.CashBook, statisticsResults, banner);
}
catch (Exception err)
{
Log.Error(err, "Error sending analysis results");
}
//Before we return, send terminate commands to close up the threads
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.RealTime.Exit();
}
//Close result handler:
_algorithmHandlers.Results.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((_algorithmHandlers.Results.IsActive ||
(_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) ||
(_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) ||
(_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) &&
ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100);
Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadFeed != null && threadFeed.IsAlive)
{
threadFeed.Abort();
}
if (threadTransactions != null && threadTransactions.IsAlive)
{
threadTransactions.Abort();
}
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
if (brokerage != null)
{
Log.Trace("Engine.Run(): Disconnecting from brokerage...");
brokerage.Disconnect();
brokerage.Dispose();
}
if (_algorithmHandlers.Setup != null)
{
Log.Trace("Engine.Run(): Disposing of setup handler...");
_algorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error(err, "Error running algorithm");
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
_algorithmHandlers.Results.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.RealTime.Exit();
}
}
