public void MessageHandler_SendsCorrectPackets_ToCorrectRoutes()
{
//Allow proper decoding of orders.
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
{
Converters = { new OrderJsonConverter() }
};
// Create list of packets to test
var debug = new DebugPacket();
var log = new LogPacket();
var backtest = new BacktestResultPacket();
var handled = new HandledErrorPacket();
var error = new RuntimeErrorPacket();
var packetList = new List<Packet>
{
log,
debug,
backtest,
handled,
error
};
using (var pullSocket = new PullSocket(">tcp://localhost:" + _port))
{
var count = 0;
while (count < packetList.Count)
{
_messageHandler.Send(packetList[count]);
var message = pullSocket.ReceiveMultipartMessage();
var payload = message[0].ConvertToString();
var packet = JsonConvert.DeserializeObject<Packet>(payload);
Assert.IsTrue(message.FrameCount == 1);
if (PacketType.Debug == packet.Type)
Assert.IsTrue(payload == JsonConvert.SerializeObject(debug));
if (PacketType.HandledError == packet.Type)
Assert.IsTrue(payload == JsonConvert.SerializeObject(handled));
if (PacketType.BacktestResult == packet.Type)
Assert.IsTrue(payload == JsonConvert.SerializeObject(backtest));
if (PacketType.RuntimeError == packet.Type)
Assert.IsTrue(payload == JsonConvert.SerializeObject(error));
if (PacketType.Log == packet.Type)
Assert.IsTrue(payload == JsonConvert.SerializeObject(log));
count++;
}
}
}
private void HandleBacktestResultPacket(BacktestResultPacket packet)
{
if (packet.Progress == 1)
{
foreach (var pair in packet.Results.Statistics)
{
_model.LogText += "STATISTICS:: " + pair.Key + " " + pair.Value;
Log.Trace("STATISTICS:: " + pair.Key + " " + pair.Value);
}
if (StreamingApi.IsEnabled)
{
if ((LeanEngineWinForm) _view != null)
{
var url = string.Format("https://www.quantconnect.com/terminal/embedded?user={0}&token={1}&bid={2}&pid={3}",
packet.UserId, Config.Get("api-access-token"), packet.BacktestId, packet.ProjectId);
((LeanEngineWinForm)_view).Browser.Navigate(url);
}
}
}
}
/// <summary>
/// Send a backtest update to the browser taking a latest snapshot of the charting data.
/// </summary>
public void Update()
{
try
{
//Sometimes don't run the update, if not ready or we're ending.
if (Algorithm == null || Algorithm.Transactions == null || _processingFinalPacket)
{
return;
}
if (DateTime.Now <= _nextUpdate || !(_daysProcessed > (_lastDaysProcessed + 1))) return;
//Extract the orders since last update
var deltaOrders = new Dictionary<int, Order>();
try
{
deltaOrders = (from order in Algorithm.Transactions.Orders
where order.Value.Time.Date >= _lastUpdate && order.Value.Status == OrderStatus.Filled
select order).ToDictionary(t => t.Key, t => t.Value);
}
catch (Exception err)
{
Log.Error("BacktestingResultHandler().Update(): Transactions: " + err.Message);
}
//Limit length of orders we pass back dynamically to avoid flooding.
if (deltaOrders.Count > 50) deltaOrders.Clear();
//Reset loop variables:
try
{
_lastUpdate = DataStream.AlorithmTime.Date;
_lastDaysProcessed = _daysProcessed;
_nextUpdate = DateTime.Now.AddSeconds(0.5);
}
catch (Exception err)
{
Log.Error("BacktestingResultHandler.Update(): Can't update variables: " + err.Message);
}
var deltaCharts = new Dictionary<string, Chart>();
lock (_chartLock)
{
//Get the updates since the last chart
foreach (var chart in Charts.Values)
{
deltaCharts.Add(chart.Name, chart.GetUpdates());
}
}
//Profit Loss Changes:
var progress = Convert.ToDecimal(_daysProcessed / _jobDays);
if (progress > 0.999m) progress = 0.999m;
//1. Cloud Upload -> Upload the whole packet to S3 Immediately:
var completeResult = new BacktestResult(Charts, Algorithm.Transactions.Orders, Algorithm.Transactions.TransactionRecord, new Dictionary<string, string>());
var complete = new BacktestResultPacket(_job, completeResult, progress);
if (DateTime.Now > _nextS3Update)
{
_nextS3Update = DateTime.Now.AddSeconds(30);
StoreResult(complete, false);
}
//2. Backtest Update -> Send the truncated packet to the backtester:
var splitPackets = SplitPackets(deltaCharts, deltaOrders, progress);
foreach (var backtestingPacket in splitPackets)
{
Engine.Notify.Send(backtestingPacket);
}
}
catch (Exception err)
{
Log.Error("BacktestingResultHandler().Update(): " + err.Message + " >> " + err.StackTrace );
}
}
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
/// <param name="job">Lean AlgorithmJob task</param>
/// <param name="orders">Collection of orders from the algorithm</param>
/// <param name="profitLoss">Collection of time-profit values for the algorithm</param>
/// <param name="holdings">Current holdings state for the algorithm</param>
/// <param name="statistics">Statistics information for the algorithm (empty if not finished)</param>
/// <param name="banner">Runtime statistics banner information</param>
public void SendFinalResult(AlgorithmNodePacket job, Dictionary<int, Order> orders, Dictionary<DateTime, decimal> profitLoss, Dictionary<string, Holding> holdings, Dictionary<string, string> statistics, Dictionary<string, string> banner)
{
try
{
//Convert local dictionary:
var charts = new Dictionary<string, Chart>(Charts);
_processingFinalPacket = true;
//Create a result packet to send to the browser.
BacktestResultPacket result = new BacktestResultPacket((BacktestNodePacket) job,
new BacktestResult(charts, orders, profitLoss, statistics), 1m)
{
ProcessingTime = (DateTime.Now - _startTime).TotalSeconds,
DateFinished = DateTime.Now,
Progress = 1
};
//Place result into storage.
StoreResult(result);
//Truncate packet to fit within 32kb of messaging limits.
result.Results = new BacktestResult();
//Second, send the truncated packet:
Engine.Notify.BacktestResult(result, finalPacket: true);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error("Algorithm.Worker.SendResult(): " + err.Message);
}
}
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
/// <param name="job">Lean AlgorithmJob task</param>
/// <param name="orders">Collection of orders from the algorithm</param>
/// <param name="profitLoss">Collection of time-profit values for the algorithm</param>
/// <param name="holdings">Current holdings state for the algorithm</param>
/// <param name="statisticsResults">Statistics information for the algorithm (empty if not finished)</param>
/// <param name="banner">Runtime statistics banner information</param>
public void SendFinalResult(AlgorithmNodePacket job, Dictionary<int, Order> orders, Dictionary<DateTime, decimal> profitLoss, Dictionary<string, Holding> holdings, StatisticsResults statisticsResults, Dictionary<string, string> banner)
{
try
{
FinalStatistics = statisticsResults.Summary;
//Convert local dictionary:
var charts = new Dictionary<string, Chart>(Charts);
_processingFinalPacket = true;
// clear the trades collection before placing inside the backtest result
foreach (var ap in statisticsResults.RollingPerformances.Values)
{
ap.ClosedTrades.Clear();
}
//Create a result packet to send to the browser.
var result = new BacktestResultPacket((BacktestNodePacket) job,
new BacktestResult(charts, orders, profitLoss, statisticsResults.Summary, statisticsResults.RollingPerformances), 1m)
{
ProcessingTime = (DateTime.Now - _startTime).TotalSeconds,
DateFinished = DateTime.Now,
Progress = 1
};
//Place result into storage.
StoreResult(result);
//Second, send the truncated packet:
_messagingHandler.Send(result);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error(err);
}
}
} // End Run();
/// <summary>
/// Send a backtest update to the browser taking a latest snapshot of the charting data.
/// </summary>
public void Update()
{
try
{
//Sometimes don't run the update, if not ready or we're ending.
if (Algorithm == null || Algorithm.Transactions == null || _processingFinalPacket)
{
return;
}
if (DateTime.Now <= _nextUpdate || !(_daysProcessed > (_lastDaysProcessed + 1))) return;
//Extract the orders since last update
var deltaOrders = new Dictionary<int, Order>();
try
{
deltaOrders = (from order in _transactionHandler.Orders
where order.Value.Time.Date >= _lastUpdate && order.Value.Status == OrderStatus.Filled
select order).ToDictionary(t => t.Key, t => t.Value);
}
catch (Exception err)
{
Log.Error(err, "Transactions");
}
//Limit length of orders we pass back dynamically to avoid flooding.
if (deltaOrders.Count > 50) deltaOrders.Clear();
//Reset loop variables:
try
{
_lastUpdate = Algorithm.Time.Date;
_lastDaysProcessed = _daysProcessed;
_nextUpdate = DateTime.Now.AddSeconds(0.5);
}
catch (Exception err)
{
Log.Error(err, "Can't update variables");
}
var deltaCharts = new Dictionary<string, Chart>();
lock (_chartLock)
{
//Get the updates since the last chart
foreach (var chart in Charts.Values)
{
deltaCharts.Add(chart.Name, chart.GetUpdates());
}
}
//Get the runtime statistics from the user algorithm:
var runtimeStatistics = new Dictionary<string, string>();
lock (_runtimeLock)
{
foreach (var pair in _runtimeStatistics)
{
runtimeStatistics.Add(pair.Key, pair.Value);
}
}
runtimeStatistics.Add("Unrealized", "$" + _algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2"));
runtimeStatistics.Add("Fees", "-$" + _algorithm.Portfolio.TotalFees.ToString("N2"));
runtimeStatistics.Add("Net Profit", "$" + _algorithm.Portfolio.TotalProfit.ToString("N2"));
runtimeStatistics.Add("Return", ((_algorithm.Portfolio.TotalPortfolioValue - _setupHandler.StartingPortfolioValue) / _setupHandler.StartingPortfolioValue).ToString("P"));
runtimeStatistics.Add("Equity", "$" + _algorithm.Portfolio.TotalPortfolioValue.ToString("N2"));
//Profit Loss Changes:
var progress = Convert.ToDecimal(_daysProcessed / _jobDays);
if (progress > 0.999m) progress = 0.999m;
//1. Cloud Upload -> Upload the whole packet to S3 Immediately:
var completeResult = new BacktestResult(Charts, _transactionHandler.Orders, Algorithm.Transactions.TransactionRecord, new Dictionary<string, string>(), runtimeStatistics, new Dictionary<string, AlgorithmPerformance>());
var complete = new BacktestResultPacket(_job, completeResult, progress);
if (DateTime.Now > _nextS3Update)
{
_nextS3Update = DateTime.Now.AddSeconds(30);
StoreResult(complete, false);
}
//2. Backtest Update -> Send the truncated packet to the backtester:
var splitPackets = SplitPackets(deltaCharts, deltaOrders, runtimeStatistics, progress);
foreach (var backtestingPacket in splitPackets)
{
_messagingHandler.Send(backtestingPacket);
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Send a backtest result packet
/// </summary>
public void BacktestResult(BacktestResultPacket packet, bool finalPacket = false)
{
//
}
/// <summary>
/// Raise a backtest result event safely.
/// </summary>
protected virtual void OnBacktestResultEvent(BacktestResultPacket packet)
{
var handler = BacktestResultEvent;
if (handler != null)
{
handler(packet);
}
}
/// <summary>
/// Backtest result packet
/// </summary>
/// <param name="packet"></param>
private void MessagingOnBacktestResultEvent(BacktestResultPacket packet)
{
if (packet.Progress != 1) return;
//Remove previous event handler:
var url = GetUrl(_job, _liveMode, true);
//Generate JSON:
var jObj = new JObject();
var dateFormat = "yyyy-MM-dd HH:mm:ss";
dynamic final = jObj;
final.dtPeriodStart = packet.PeriodStart.ToString(dateFormat);
final.dtPeriodFinished = packet.PeriodFinish.AddDays(1).ToString(dateFormat);
dynamic resultData = new JObject();
resultData.version = "3";
resultData.results = JObject.FromObject(packet.Results);
resultData.statistics = JObject.FromObject(packet.Results.Statistics);
resultData.iTradeableDates = 1;
resultData.ranking = null;
final.oResultData = resultData;
var json = JsonConvert.SerializeObject(final);
//GECKO RESULT SET:
//_geckoBrowser.DOMContentLoaded += (sender, args) =>
//{
// var executor = new JQueryExecutor(_geckoBrowser.Window);
// executor.ExecuteJQuery("window.jnBacktest = JSON.parse('" + json + "');");
// executor.ExecuteJQuery("$.holdReady(false)");
//};
//_geckoBrowser.Navigate(url);
//MONO WEB BROWSER RESULT SET:
_monoBrowser.DocumentCompleted += (sender, args) =>
{
if (_monoBrowser.Document == null) return;
_monoBrowser.Document.InvokeScript("eval", new object[] { "window.jnBacktest = JSON.parse('" + json + "');" });
_monoBrowser.Document.InvokeScript("eval", new object[] { "$.holdReady(false)" });
};
_monoBrowser.Navigate(url);
foreach (var pair in packet.Results.Statistics)
{
_logging.Trace("STATISTICS:: " + pair.Key + " " + pair.Value);
}
}