/// <summary>
/// The main processing method steps through the messaging queue and processes the messages one by one.
/// </summary>
protected override void Run()
{
try
{
while (!(ExitTriggered && Messages.Count == 0))
{
//While there's no work to do, go back to the algorithm:
if (Messages.Count == 0)
{
ExitEvent.WaitOne(50);
}
else
{
//1. Process Simple Messages in Queue
Packet packet;
if (Messages.TryDequeue(out packet))
{
MessagingHandler.Send(packet);
}
}
//2. Update the packet scanner:
Update();
} // While !End.
}
catch (Exception err)
{
// unexpected error, we need to close down shop
Log.Error(err);
// quit the algorithm due to error
Algorithm.RunTimeError = err;
}
Log.Trace("BacktestingResultHandler.Run(): Ending Thread...");
} // End Run();
/// <summary>
/// Send an algorithm status update to the browser.
/// </summary>
/// <param name="status">Status enum value.</param>
/// <param name="message">Additional optional status message.</param>
public virtual void SendStatusUpdate(AlgorithmStatus status, string message = "")
{
var statusPacket = new AlgorithmStatusPacket(_algorithmId, _projectId, status, message)
{
OptimizationId = _job.OptimizationId
};
MessagingHandler.Send(statusPacket);
}
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
protected void SendFinalResult()
{
try
{
BacktestResultPacket result;
// could happen if algorithm failed to init
if (Algorithm != null)
{
//Convert local dictionary:
var charts = new Dictionary <string, Chart>(Charts);
var orders = new Dictionary <int, Order>(TransactionHandler.Orders);
var profitLoss = new SortedDictionary <DateTime, decimal>(Algorithm.Transactions.TransactionRecord);
var statisticsResults = GenerateStatisticsResults(charts, profitLoss, _capacityEstimate);
var runtime = GetAlgorithmRuntimeStatistics(statisticsResults.Summary, capacityEstimate: _capacityEstimate);
FinalStatistics = statisticsResults.Summary;
// clear the trades collection before placing inside the backtest result
foreach (var ap in statisticsResults.RollingPerformances.Values)
{
ap.ClosedTrades.Clear();
}
var orderEvents = TransactionHandler.OrderEvents.ToList();
//Create a result packet to send to the browser.
result = new BacktestResultPacket(_job,
new BacktestResult(new BacktestResultParameters(charts, orders, profitLoss, statisticsResults.Summary, runtime, statisticsResults.RollingPerformances, orderEvents, statisticsResults.TotalPerformance, AlphaRuntimeStatistics)),
Algorithm.EndDate, Algorithm.StartDate);
}
else
{
result = BacktestResultPacket.CreateEmpty(_job);
}
var utcNow = DateTime.UtcNow;
result.ProcessingTime = (utcNow - StartTime).TotalSeconds;
result.DateFinished = DateTime.Now;
result.Progress = 1;
//Place result into storage.
StoreResult(result);
result.Results.ServerStatistics = GetServerStatistics(utcNow);
//Second, send the truncated packet:
MessagingHandler.Send(result);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
public void SendFinalResult()
{
try
{
_processingFinalPacket = true;
//Convert local dictionary:
var charts = new Dictionary <string, Chart>(Charts);
var orders = new Dictionary <int, Order>(TransactionHandler.Orders);
var profitLoss = new SortedDictionary <DateTime, decimal>(Algorithm.Transactions.TransactionRecord);
var statisticsResults = GenerateStatisticsResults(charts, profitLoss);
var runtime = GetAlgorithmRuntimeStatistics(statisticsResults.Summary);
FinalStatistics = statisticsResults.Summary;
// clear the trades collection before placing inside the backtest result
foreach (var ap in statisticsResults.RollingPerformances.Values)
{
ap.ClosedTrades.Clear();
}
//Create a result packet to send to the browser.
var result = new BacktestResultPacket(_job,
new BacktestResult(charts, orders, profitLoss, statisticsResults.Summary, runtime, statisticsResults.RollingPerformances, statisticsResults.TotalPerformance)
{
AlphaRuntimeStatistics = AlphaRuntimeStatistics
})
{
ProcessingTime = (DateTime.UtcNow - StartTime).TotalSeconds,
DateFinished = DateTime.Now,
Progress = 1
};
//Place result into storage.
StoreResult(result);
//Second, send the truncated packet:
MessagingHandler.Send(result);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// The main processing method steps through the messaging queue and processes the messages one by one.
/// </summary>
public void Run()
{
//Setup minimum result arrays:
//SampleEquity(job.periodStart, job.startingCapital);
//SamplePerformance(job.periodStart, 0);
try
{
while (!(_exitTriggered && Messages.Count == 0))
{
//While there's no work to do, go back to the algorithm:
if (Messages.Count == 0)
{
Thread.Sleep(50);
}
else
{
//1. Process Simple Messages in Queue
Packet packet;
if (Messages.TryDequeue(out packet))
{
MessagingHandler.Send(packet);
}
}
//2. Update the packet scanner:
Update();
} // While !End.
}
catch (Exception err)
{
// unexpected error, we need to close down shop
Log.Error(err);
// quit the algorithm due to error
Algorithm.RunTimeError = err;
}
Log.Trace("BacktestingResultHandler.Run(): Ending Thread...");
IsActive = false;
// reset standard out/error
Console.SetOut(StandardOut);
Console.SetError(StandardError);
} // End Run();
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
/// <param name="job">Lean AlgorithmJob task</param>
/// <param name="orders">Collection of orders from the algorithm</param>
/// <param name="profitLoss">Collection of time-profit values for the algorithm</param>
/// <param name="holdings">Current holdings state for the algorithm</param>
/// <param name="cashbook">Cashbook for the holdingss</param>
/// <param name="statisticsResults">Statistics information for the algorithm (empty if not finished)</param>
/// <param name="banner">Runtime statistics banner information</param>
public void SendFinalResult(AlgorithmNodePacket job, Dictionary <int, Order> orders, Dictionary <DateTime, decimal> profitLoss, Dictionary <string, Holding> holdings, CashBook cashbook, StatisticsResults statisticsResults, Dictionary <string, string> banner)
{
try
{
FinalStatistics = statisticsResults.Summary;
//Convert local dictionary:
var charts = new Dictionary <string, Chart>(Charts);
_processingFinalPacket = true;
// clear the trades collection before placing inside the backtest result
foreach (var ap in statisticsResults.RollingPerformances.Values)
{
ap.ClosedTrades.Clear();
}
//Create a result packet to send to the browser.
var result = new BacktestResultPacket((BacktestNodePacket)job,
new BacktestResult(charts, orders, profitLoss, statisticsResults.Summary, banner, statisticsResults.RollingPerformances, statisticsResults.TotalPerformance)
{
AlphaRuntimeStatistics = AlphaRuntimeStatistics
})
{
ProcessingTime = (DateTime.UtcNow - StartTime).TotalSeconds,
DateFinished = DateTime.Now,
Progress = 1
};
//Place result into storage.
StoreResult(result);
//Second, send the truncated packet:
MessagingHandler.Send(result);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error(err);
}
}
} // End Run();
/// <summary>
/// Send a backtest update to the browser taking a latest snapshot of the charting data.
/// </summary>
public void Update()
{
try
{
//Sometimes don't run the update, if not ready or we're ending.
if (Algorithm?.Transactions == null || _processingFinalPacket)
{
return;
}
if (DateTime.UtcNow <= _nextUpdate || _daysProcessed < _daysProcessedFrontier)
{
return;
}
//Extract the orders since last update
var deltaOrders = new Dictionary <int, Order>();
try
{
deltaOrders = (from order in TransactionHandler.Orders
where order.Value.Time.Date >= _lastUpdate && order.Value.Status == OrderStatus.Filled
select order).ToDictionary(t => t.Key, t => t.Value);
}
catch (Exception err)
{
Log.Error(err, "Transactions");
}
//Limit length of orders we pass back dynamically to avoid flooding.
if (deltaOrders.Count > 50)
{
deltaOrders.Clear();
}
//Reset loop variables:
try
{
_lastUpdate = Algorithm.UtcTime.Date;
_daysProcessedFrontier = _daysProcessed + 1;
_nextUpdate = DateTime.UtcNow.AddSeconds(2);
}
catch (Exception err)
{
Log.Error(err, "Can't update variables");
}
var deltaCharts = new Dictionary <string, Chart>();
var performanceCharts = new Dictionary <string, Chart>();
lock (ChartLock)
{
//Get the updates since the last chart
foreach (var kvp in Charts)
{
var chart = kvp.Value;
deltaCharts.Add(chart.Name, chart.GetUpdates());
if (AlgorithmPerformanceCharts.Contains(kvp.Key))
{
performanceCharts[kvp.Key] = chart.Clone();
}
}
}
//Get the runtime statistics from the user algorithm:
var runtimeStatistics = new Dictionary <string, string>();
lock (RuntimeStatistics)
{
foreach (var pair in RuntimeStatistics)
{
runtimeStatistics.Add(pair.Key, pair.Value);
}
}
var summary = GenerateStatisticsResults(performanceCharts).Summary;
GetAlgorithmRuntimeStatistics(summary, runtimeStatistics);
//Profit Loss Changes:
var progress = Convert.ToDecimal(_daysProcessed / _jobDays);
if (progress > 0.999m)
{
progress = 0.999m;
}
//1. Cloud Upload -> Upload the whole packet to S3 Immediately:
if (DateTime.UtcNow > _nextS3Update)
{
// For intermediate backtesting results, we truncate the order list to include only the last 100 orders
// The final packet will contain the full list of orders.
const int maxOrders = 100;
var orderCount = TransactionHandler.Orders.Count;
var completeResult = new BacktestResult(
Charts,
orderCount > maxOrders ? TransactionHandler.Orders.Skip(orderCount - maxOrders).ToDictionary() : TransactionHandler.Orders.ToDictionary(),
Algorithm.Transactions.TransactionRecord,
new Dictionary <string, string>(),
runtimeStatistics,
new Dictionary <string, AlgorithmPerformance>());
StoreResult(new BacktestResultPacket(_job, completeResult, progress));
_nextS3Update = DateTime.UtcNow.AddSeconds(30);
}
//2. Backtest Update -> Send the truncated packet to the backtester:
var splitPackets = SplitPackets(deltaCharts, deltaOrders, runtimeStatistics, progress);
foreach (var backtestingPacket in splitPackets)
{
MessagingHandler.Send(backtestingPacket);
}
}
catch (Exception err)
{
Log.Error(err);
}
}
} // End Run();
/// <summary>
/// Send a backtest update to the browser taking a latest snapshot of the charting data.
/// </summary>
private void Update()
{
try
{
//Sometimes don't run the update, if not ready or we're ending.
if (Algorithm?.Transactions == null || ExitTriggered)
{
return;
}
var utcNow = DateTime.UtcNow;
if (utcNow <= _nextUpdate || _daysProcessed < _daysProcessedFrontier)
{
return;
}
var deltaOrders = GetDeltaOrders(LastDeltaOrderPosition, shouldStop: orderCount => orderCount >= 50);
// Deliberately skip to the end of order event collection to prevent overloading backtesting UX
LastDeltaOrderPosition = TransactionHandler.OrderEvents.Count();
//Reset loop variables:
try
{
_daysProcessedFrontier = _daysProcessed + 1;
_nextUpdate = utcNow.AddSeconds(3);
}
catch (Exception err)
{
Log.Error(err, "Can't update variables");
}
var deltaCharts = new Dictionary <string, Chart>();
var serverStatistics = GetServerStatistics(utcNow);
var performanceCharts = new Dictionary <string, Chart>();
lock (ChartLock)
{
//Get the updates since the last chart
foreach (var kvp in Charts)
{
var chart = kvp.Value;
var updates = chart.GetUpdates();
if (!updates.IsEmpty())
{
deltaCharts.Add(chart.Name, updates);
}
if (AlgorithmPerformanceCharts.Contains(kvp.Key))
{
performanceCharts[kvp.Key] = chart.Clone();
}
}
}
//Get the runtime statistics from the user algorithm:
var runtimeStatistics = new Dictionary <string, string>();
lock (RuntimeStatistics)
{
foreach (var pair in RuntimeStatistics)
{
runtimeStatistics.Add(pair.Key, pair.Value);
}
}
var summary = GenerateStatisticsResults(performanceCharts).Summary;
GetAlgorithmRuntimeStatistics(summary, runtimeStatistics);
//Profit Loss Changes:
var progress = Convert.ToDecimal(_daysProcessed / _jobDays);
if (progress > 0.999m)
{
progress = 0.999m;
}
//1. Cloud Upload -> Upload the whole packet to S3 Immediately:
if (utcNow > _nextS3Update)
{
// For intermediate backtesting results, we truncate the order list to include only the last 100 orders
// The final packet will contain the full list of orders.
const int maxOrders = 100;
var orderCount = TransactionHandler.Orders.Count;
var completeResult = new BacktestResult(new BacktestResultParameters(
Charts,
orderCount > maxOrders ? TransactionHandler.Orders.Skip(orderCount - maxOrders).ToDictionary() : TransactionHandler.Orders.ToDictionary(),
Algorithm.Transactions.TransactionRecord,
new Dictionary <string, string>(),
runtimeStatistics,
new Dictionary <string, AlgorithmPerformance>(),
// we store the last 100 order events, the final packet will contain the full list
TransactionHandler.OrderEvents.Reverse().Take(100).ToList()));
StoreResult(new BacktestResultPacket(_job, completeResult, Algorithm.EndDate, Algorithm.StartDate, progress));
_nextS3Update = DateTime.UtcNow.AddSeconds(30);
}
//2. Backtest Update -> Send the truncated packet to the backtester:
var splitPackets = SplitPackets(deltaCharts, deltaOrders, runtimeStatistics, progress, serverStatistics);
foreach (var backtestingPacket in splitPackets)
{
MessagingHandler.Send(backtestingPacket);
}
// let's re update this value after we finish just in case, so we don't re enter in the next loop
_nextUpdate = DateTime.UtcNow.AddSeconds(3);
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
protected void SendFinalResult()
{
try
{
//Convert local dictionary:
var charts = new Dictionary <string, Chart>(Charts);
var orders = new Dictionary <int, Order>(TransactionHandler.Orders);
var profitLoss = new SortedDictionary <DateTime, decimal>(Algorithm.Transactions.TransactionRecord);
var statisticsResults = GenerateStatisticsResults(charts, profitLoss);
var runtime = GetAlgorithmRuntimeStatistics(statisticsResults.Summary);
FinalStatistics = statisticsResults.Summary;
// clear the trades collection before placing inside the backtest result
foreach (var ap in statisticsResults.RollingPerformances.Values)
{
ap.ClosedTrades.Clear();
}
var orderEvents = TransactionHandler.OrderEvents.ToList();
//Create a result packet to send to the browser.
var result = new BacktestResultPacket(_job,
new BacktestResult(new BacktestResultParameters(charts, orders, profitLoss, statisticsResults.Summary, runtime, statisticsResults.RollingPerformances, orderEvents, statisticsResults.TotalPerformance, AlphaRuntimeStatistics)),
Algorithm.EndDate, Algorithm.StartDate)
{
ProcessingTime = (DateTime.UtcNow - StartTime).TotalSeconds,
DateFinished = DateTime.Now,
Progress = 1
};
// Create new connection factory
var factory = new ConnectionFactory()
{
HostName = "localhost"
};
using (var connection = factory.CreateConnection())
using (var channel = connection.CreateModel())
{
// Set up queue for RabbitMQ
channel.QueueDeclare(queue: "rabbitBroker",
durable: false,
exclusive: false,
autoDelete: false,
arguments: null);
// Create test message for RabbitMQProducer
string producerMessage = JsonConvert.SerializeObject(FinalStatistics, Formatting.Indented);
var producerBody = Encoding.UTF8.GetBytes(producerMessage);
channel.BasicPublish(exchange: "",
routingKey: "rabbitBroker",
basicProperties: null,
body: producerBody);
}
//Place result into storage.
StoreResult(result);
//Second, send the truncated packet:
MessagingHandler.Send(result);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error(err);
}
}