/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
/// <param name="job">Lean AlgorithmJob task</param>
/// <param name="orders">Collection of orders from the algorithm</param>
/// <param name="profitLoss">Collection of time-profit values for the algorithm</param>
/// <param name="holdings">Current holdings state for the algorithm</param>
/// <param name="statisticsResults">Statistics information for the algorithm (empty if not finished)</param>
/// <param name="runtime">Runtime statistics banner information</param>
public void SendFinalResult(AlgorithmNodePacket job, Dictionary<int, Order> orders, Dictionary<DateTime, decimal> profitLoss, Dictionary<string, Holding> holdings, StatisticsResults statisticsResults, Dictionary<string, string> runtime)
{
try
{
//Convert local dictionary:
var charts = new Dictionary<string, Chart>(Charts);
//Create a packet:
var result = new LiveResultPacket((LiveNodePacket)job, new LiveResult(charts, orders, profitLoss, holdings, statisticsResults.Summary, runtime));
//Save the processing time:
result.ProcessingTime = (DateTime.Now - _startTime).TotalSeconds;
//Store to S3:
StoreResult(result, false);
//Truncate packet to fit within 32kb:
result.Results = new LiveResult();
//Send the truncated packet:
_messagingHandler.Send(result);
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Send a live trading packet result.
/// </summary>
public void LiveTradingResult(LiveResultPacket packet)
{
//
}
} // End Run();
/// <summary>
/// Every so often send an update to the browser with the current state of the algorithm.
/// </summary>
public void Update()
{
//Initialize:
Dictionary<int, Order> deltaOrders;
//Error checks if the algorithm & threads have not loaded yet, or are closing down.
if (_algorithm == null || _algorithm.Transactions == null || _transactionHandler.Orders == null || !_algorithm.GetLocked())
{
Log.Error("LiveTradingResultHandler.Update(): Algorithm not yet initialized.");
return;
}
try
{
if (DateTime.Now > _nextUpdate || _exitTriggered)
{
//Extract the orders created since last update
OrderEvent orderEvent;
deltaOrders = new Dictionary<int, Order>();
var stopwatch = Stopwatch.StartNew();
while (_orderEvents.TryDequeue(out orderEvent) && stopwatch.ElapsedMilliseconds < 15)
{
var order = _algorithm.Transactions.GetOrderById(orderEvent.OrderId);
deltaOrders[orderEvent.OrderId] = order.Clone();
}
//For charting convert to UTC
foreach (var order in deltaOrders)
{
order.Value.Price = order.Value.Price.SmartRounding();
order.Value.Time = order.Value.Time.ToUniversalTime();
}
//Reset loop variables:
_lastOrderId = (from order in deltaOrders.Values select order.Id).DefaultIfEmpty(_lastOrderId).Max();
//Limit length of orders we pass back dynamically to avoid flooding.
//if (deltaOrders.Count > 50) deltaOrders.Clear();
//Create and send back the changes in chart since the algorithm started.
var deltaCharts = new Dictionary<string, Chart>();
Log.Debug("LiveTradingResultHandler.Update(): Build delta charts");
lock (_chartLock)
{
//Get the updates since the last chart
foreach (var chart in _charts)
{
// remove directory pathing characters from chart names
var safeName = chart.Value.Name.Replace('/', '-');
deltaCharts.Add(safeName, chart.Value.GetUpdates());
}
}
Log.Debug("LiveTradingResultHandler.Update(): End build delta charts");
//Profit loss changes, get the banner statistics, summary information on the performance for the headers.
var holdings = new Dictionary<string, Holding>();
var deltaStatistics = new Dictionary<string, string>();
var runtimeStatistics = new Dictionary<string, string>();
var serverStatistics = OS.GetServerStatistics();
var upTime = DateTime.UtcNow - _launchTimeUtc;
serverStatistics["Up Time"] = string.Format("{0}d {1:hh\\:mm\\:ss}", upTime.Days, upTime);
// only send holdings updates when we have changes in orders, except for first time, then we want to send all
foreach (var asset in _algorithm.Securities.Values.Where(x => !x.SubscriptionDataConfig.IsInternalFeed).OrderBy(x => x.Symbol.Value))
{
holdings.Add(asset.Symbol.Value, new Holding(asset));
}
//Add the algorithm statistics first.
Log.Debug("LiveTradingResultHandler.Update(): Build run time stats");
lock (_runtimeLock)
{
foreach (var pair in _runtimeStatistics)
{
runtimeStatistics.Add(pair.Key, pair.Value);
}
}
Log.Debug("LiveTradingResultHandler.Update(): End build run time stats");
//Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors
var netReturn = _setupHandler.StartingPortfolioValue > 0 ?
(_algorithm.Portfolio.TotalPortfolioValue - _setupHandler.StartingPortfolioValue) / _setupHandler.StartingPortfolioValue
: 0;
//Add other fixed parameters.
runtimeStatistics.Add("Unrealized:", "$" + _algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2"));
runtimeStatistics.Add("Fees:", "-$" + _algorithm.Portfolio.TotalFees.ToString("N2"));
runtimeStatistics.Add("Net Profit:", "$" + _algorithm.Portfolio.TotalProfit.ToString("N2"));
runtimeStatistics.Add("Return:", netReturn.ToString("P"));
runtimeStatistics.Add("Equity:", "$" + _algorithm.Portfolio.TotalPortfolioValue.ToString("N2"));
runtimeStatistics.Add("Holdings:", "$" + _algorithm.Portfolio.TotalHoldingsValue.ToString("N2"));
runtimeStatistics.Add("Volume:", "$" + _algorithm.Portfolio.TotalSaleVolume.ToString("N2"));
// since we're sending multiple packets, let's do it async and forget about it
// chart data can get big so let's break them up into groups
var splitPackets = SplitPackets(deltaCharts, deltaOrders, holdings, deltaStatistics, runtimeStatistics, serverStatistics);
foreach (var liveResultPacket in splitPackets)
{
_messagingHandler.Send(liveResultPacket);
}
//Send full packet to storage.
if (DateTime.Now > _nextChartsUpdate || _exitTriggered)
{
Log.Debug("LiveTradingResultHandler.Update(): Pre-store result");
_nextChartsUpdate = DateTime.Now.AddMinutes(1);
var chartComplete = new Dictionary<string, Chart>();
lock (_chartLock)
{
foreach (var chart in Charts)
{
// remove directory pathing characters from chart names
var safeName = chart.Value.Name.Replace('/', '-');
chartComplete.Add(safeName, chart.Value);
}
}
var orders = new Dictionary<int, Order>(_transactionHandler.Orders);
var complete = new LiveResultPacket(_job, new LiveResult(chartComplete, orders, _algorithm.Transactions.TransactionRecord, holdings, deltaStatistics, runtimeStatistics, serverStatistics));
StoreResult(complete);
Log.Debug("LiveTradingResultHandler.Update(): End-store result");
}
// Upload the logs every 1-2 minutes; this can be a heavy operation depending on amount of live logging and should probably be done asynchronously.
if (DateTime.Now > _nextLogStoreUpdate || _exitTriggered)
{
List<LogEntry> logs;
Log.Debug("LiveTradingResultHandler.Update(): Storing log...");
lock (_logStoreLock)
{
var utc = DateTime.UtcNow;
logs = (from log in _logStore
where log.Time >= utc.RoundDown(TimeSpan.FromHours(1))
select log).ToList();
//Override the log master to delete the old entries and prevent memory creep.
_logStore = logs;
}
StoreLog(logs);
_nextLogStoreUpdate = DateTime.Now.AddMinutes(2);
Log.Debug("LiveTradingResultHandler.Update(): Finished storing log");
}
// Every minute send usage statistics:
if (DateTime.Now > _nextStatisticsUpdate || _exitTriggered)
{
try
{
_api.SendStatistics(
_job.AlgorithmId,
_algorithm.Portfolio.TotalUnrealizedProfit,
_algorithm.Portfolio.TotalFees,
_algorithm.Portfolio.TotalProfit,
_algorithm.Portfolio.TotalHoldingsValue,
_algorithm.Portfolio.TotalPortfolioValue,
netReturn,
_algorithm.Portfolio.TotalSaleVolume,
_lastOrderId, 0);
}
catch (Exception err)
{
Log.Error(err, "Error sending statistics:");
}
_nextStatisticsUpdate = DateTime.Now.AddMinutes(1);
}
Log.Debug("LiveTradingResultHandler.Update(): Trimming charts");
lock (_chartLock)
{
foreach (var chart in Charts)
{
foreach (var series in chart.Value.Series)
{
// trim data that's older than 2 days
series.Value.Values =
(from v in series.Value.Values
where v.x > Time.DateTimeToUnixTimeStamp(DateTime.UtcNow.AddDays(-2))
select v).ToList();
}
}
}
Log.Debug("LiveTradingResultHandler.Update(): Finished trimming charts");
//Set the new update time after we've finished processing.
// The processing can takes time depending on how large the packets are.
_nextUpdate = DateTime.Now.AddSeconds(2);
} // End Update Charts:
}
catch (Exception err)
{
Log.Error(err, "LiveTradingResultHandler().Update(): ", true);
}
}
/// <summary>
/// Every so often send an update to the browser with the current state of the algorithm.
/// </summary>
public void Update()
{
//Initialize:
Dictionary<int, Order> deltaOrders;
//Error checks if the algorithm & threads have not loaded yet, or are closing down.
if (_algorithm == null || _algorithm.Transactions == null || _algorithm.Transactions.Orders == null || !_algorithm.GetLocked())
{
Log.Error("LiveTradingResultHandler.Update(): Algorithm not yet initialized.");
return;
}
try
{
if (DateTime.Now > _nextUpdate)
{
//Extract the orders created since last update
OrderEvent orderEvent;
deltaOrders = new Dictionary<int, Order>();
var stopwatch = Stopwatch.StartNew();
while (_orderEvents.TryDequeue(out orderEvent) && stopwatch.ElapsedMilliseconds < 15)
{
var order = _algorithm.Transactions.GetOrderById(orderEvent.OrderId);
deltaOrders[orderEvent.OrderId] = ObjectActivator.Clone(order) as Order;
}
//For charting convert to UTC
foreach (var order in deltaOrders)
{
order.Value.Time = order.Value.Time.ToUniversalTime();
}
//Reset loop variables:
_lastOrderId = (from order in deltaOrders.Values select order.Id).DefaultIfEmpty(_lastOrderId).Max();
//Limit length of orders we pass back dynamically to avoid flooding.
//if (deltaOrders.Count > 50) deltaOrders.Clear();
//Create and send back the changes in chart since the algorithm started.
var deltaCharts = new Dictionary<string, Chart>();
lock (_chartLock)
{
//Get the updates since the last chart
foreach (var chart in _charts)
{
deltaCharts.Add(chart.Value.Name, chart.Value.GetUpdates());
}
}
//Profit loss changes, get the banner statistics, summary information on the performance for the headers.
var holdings = new Dictionary<string, Holding>();
var deltaStatistics = new Dictionary<string, string>();
var runtimeStatistics = new Dictionary<string, string>();
var serverStatistics = OS.GetServerStatistics();
// only send holdings updates when we have changes in orders, except for first time, then we want to send all
foreach (var asset in _algorithm.Securities.Values.OrderBy(x => x.Symbol))
{
holdings.Add(asset.Symbol, new Holding(asset.Holdings));
}
//Add the algorithm statistics first.
lock (_runtimeStatistics)
{
foreach (var pair in _runtimeStatistics)
{
runtimeStatistics.Add(pair.Key, pair.Value);
}
}
//Add other fixed parameters.
runtimeStatistics.Add("Unrealized:", "$" + _algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2"));
runtimeStatistics.Add("Fees:", "-$" + _algorithm.Portfolio.TotalFees.ToString("N2"));
runtimeStatistics.Add("Net Profit:", "$" + _algorithm.Portfolio.TotalProfit.ToString("N2"));
runtimeStatistics.Add("Return:", ((_algorithm.Portfolio.TotalPortfolioValue - Engine.SetupHandler.StartingPortfolioValue) / Engine.SetupHandler.StartingPortfolioValue).ToString("P"));
runtimeStatistics.Add("Equity:", "$" + _algorithm.Portfolio.TotalPortfolioValue.ToString("N2"));
runtimeStatistics.Add("Holdings:", "$" + _algorithm.Portfolio.TotalHoldingsValue.ToString("N2"));
runtimeStatistics.Add("Volume:", "$" + _algorithm.Portfolio.TotalSaleVolume.ToString("N2"));
// since we're sending multiple packets, let's do it async and forget about it
// chart data can get big so let's break them up into groups
var splitPackets = SplitPackets(deltaCharts, deltaOrders, holdings, deltaStatistics, runtimeStatistics, serverStatistics);
foreach (var liveResultPacket in splitPackets)
{
Engine.Notify.Send(liveResultPacket);
}
//Send full packet to storage.
if (DateTime.Now > _nextChartsUpdate)
{
_nextChartsUpdate = DateTime.Now.AddMinutes(1);
lock (_chartLock)
{
var chartComplete = new Dictionary<string, Chart>(Charts);
var orders = new Dictionary<int, Order>(_algorithm.Transactions.Orders);
var complete = new LiveResultPacket(_job, new LiveResult(chartComplete, orders, _algorithm.Transactions.TransactionRecord, holdings, deltaStatistics, runtimeStatistics, serverStatistics));
StoreResult(complete);
}
}
// Upload the logs every 1-2 minutes; this can be a heavy operation depending on amount of live logging and should probably be done asynchronously.
if (DateTime.Now > _nextLogStoreUpdate)
{
Log.Trace("LiveTradingResultHandler.Update(): Storing log...");
lock (_logStoreLock)
{
var utc = DateTime.UtcNow;
var logs = (from log in _logStore
where log.Time >= utc.RoundDown(TimeSpan.FromHours(1))
select log).ToList();
//Override the log master to delete the old entries and prevent memory creep.
_logStore = logs;
StoreLog(logs);
}
_nextLogStoreUpdate = DateTime.Now.AddMinutes(2);
}
// Every 5 send usage statistics:
if (DateTime.Now > _nextStatisticsUpdate)
{
try
{
Engine.Api.SendStatistics(
_job.AlgorithmId,
_algorithm.Portfolio.TotalUnrealizedProfit,
_algorithm.Portfolio.TotalFees,
_algorithm.Portfolio.TotalProfit,
_algorithm.Portfolio.TotalHoldingsValue,
_algorithm.Portfolio.TotalPortfolioValue,
((_algorithm.Portfolio.TotalPortfolioValue - Engine.SetupHandler.StartingPortfolioValue) / Engine.SetupHandler.StartingPortfolioValue),
_algorithm.Portfolio.TotalSaleVolume,
_lastOrderId, 0);
}
catch (Exception err)
{
Log.Error("LiveTradingResultHandler.Update(): Error sending statistics: " + err.Message);
}
_nextStatisticsUpdate = DateTime.Now.AddMinutes(1);
}
lock (_chartLock)
{
foreach (var chart in Charts)
{
foreach (var series in chart.Value.Series)
{
// trim data that's older than 2 days
series.Value.Values =
(from v in series.Value.Values
where v.x > Time.DateTimeToUnixTimeStamp(DateTime.UtcNow.AddDays(-2))
select v).ToList();
}
}
}
//Set the new update time after we've finished processing.
// The processing can takes time depending on how large the packets are.
_nextUpdate = DateTime.Now.AddSeconds(2);
} // End Update Charts:
}
catch (Exception err)
{
Log.Error("LiveTradingResultHandler().Update(): " + err.Message, true);
}
}