/// <summary>
/// The main processing method steps through the messaging queue and processes the messages one by one.
/// </summary>
protected override void Run()
{
try
{
while (!(ExitTriggered && Messages.Count == 0))
{
//While there's no work to do, go back to the algorithm:
if (Messages.Count == 0)
{
ExitEvent.WaitOne(50);
}
else
{
//1. Process Simple Messages in Queue
Packet packet;
if (Messages.TryDequeue(out packet))
{
MessagingHandler.Send(packet);
}
}
//2. Update the packet scanner:
Update();
} // While !End.
}
catch (Exception err)
{
// unexpected error, we need to close down shop
Log.Error(err);
// quit the algorithm due to error
Algorithm.RunTimeError = err;
}
Log.Trace("BacktestingResultHandler.Run(): Ending Thread...");
} // End Run();
/// <summary>
/// Send an algorithm status update to the browser.
/// </summary>
/// <param name="status">Status enum value.</param>
/// <param name="message">Additional optional status message.</param>
public virtual void SendStatusUpdate(AlgorithmStatus status, string message = "")
{
var statusPacket = new AlgorithmStatusPacket(_algorithmId, _projectId, status, message)
{
OptimizationId = _job.OptimizationId
};
MessagingHandler.Send(statusPacket);
}
/// <summary>
/// Sends a message to the group.
/// </summary>
/// <returns>Whether the message has been received.</returns>
public bool SendMessage(MessagingHandler <T> handler)
{
var hasReceivers = receivers.Count > 0; //Receivers can change while looping through it.
foreach (var receiver in GetReceivers())
{
handler(receiver);
}
return(hasReceivers);
}
private static bool SendMessage <T>(this T receiver, MessagingHandler <T> handler)
{
if (receiver != null)
{
handler(receiver);
return(true);
}
return(false);
}
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
protected void SendFinalResult()
{
try
{
BacktestResultPacket result;
// could happen if algorithm failed to init
if (Algorithm != null)
{
//Convert local dictionary:
var charts = new Dictionary <string, Chart>(Charts);
var orders = new Dictionary <int, Order>(TransactionHandler.Orders);
var profitLoss = new SortedDictionary <DateTime, decimal>(Algorithm.Transactions.TransactionRecord);
var statisticsResults = GenerateStatisticsResults(charts, profitLoss, _capacityEstimate);
var runtime = GetAlgorithmRuntimeStatistics(statisticsResults.Summary, capacityEstimate: _capacityEstimate);
FinalStatistics = statisticsResults.Summary;
// clear the trades collection before placing inside the backtest result
foreach (var ap in statisticsResults.RollingPerformances.Values)
{
ap.ClosedTrades.Clear();
}
var orderEvents = TransactionHandler.OrderEvents.ToList();
//Create a result packet to send to the browser.
result = new BacktestResultPacket(_job,
new BacktestResult(new BacktestResultParameters(charts, orders, profitLoss, statisticsResults.Summary, runtime, statisticsResults.RollingPerformances, orderEvents, statisticsResults.TotalPerformance, AlphaRuntimeStatistics)),
Algorithm.EndDate, Algorithm.StartDate);
}
else
{
result = BacktestResultPacket.CreateEmpty(_job);
}
var utcNow = DateTime.UtcNow;
result.ProcessingTime = (utcNow - StartTime).TotalSeconds;
result.DateFinished = DateTime.Now;
result.Progress = 1;
//Place result into storage.
StoreResult(result);
result.Results.ServerStatistics = GetServerStatistics(utcNow);
//Second, send the truncated packet:
MessagingHandler.Send(result);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error(err);
}
}
private void HandleClientComm(object client)
{
TcpClient tcpClient = (TcpClient)client;
Logger.Log("Client connected to server");
byte[] readBuffer = new byte[1024];
int bytesRead = 0;
StringBuilder completeMessage = new StringBuilder();
String returnMessage = String.Empty;
using (NetworkStream clientStream = tcpClient.GetStream())
{
try
{
do
{
bytesRead = clientStream.Read(readBuffer, 0, readBuffer.Length);
completeMessage.Append(Encoding.ASCII.GetString(readBuffer, 0, bytesRead));
} while (clientStream.DataAvailable);
MessagingHandler myMessageHandler = new MessagingHandler();
returnMessage = myMessageHandler.runMessageHandling(completeMessage.ToString());
Console.WriteLine(returnMessage);
//Extract message from completeMessage
}
catch
{
Logger.Log("Client had socket error");
}
if (tcpClient.Connected) //If the tcpClient is still open
{
Logger.Log("Writing response to client");
byte[] writeBuffer;
writeBuffer = Encoding.ASCII.GetBytes(returnMessage);
clientStream.Write(writeBuffer, 0, writeBuffer.Length);
clientStream.Flush();
}
else
{
//Log unable to send return message, no socket.
}
}
if (tcpClient.Connected)
{
Logger.Log("Closing connection to client");
tcpClient.Close();
}
}
static void Main(string[] args)
{
handlerA = new MessagingHandler(m => handlerB.Receive(m));
handlerB = new MessagingHandler(m =>
{
handlerA.Receive(m);
});
handlerA.Register(new A());
handlerB.Timeout = 2000;
SendMessage();
Console.Read();
}
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
public void SendFinalResult()
{
try
{
_processingFinalPacket = true;
//Convert local dictionary:
var charts = new Dictionary <string, Chart>(Charts);
var orders = new Dictionary <int, Order>(TransactionHandler.Orders);
var profitLoss = new SortedDictionary <DateTime, decimal>(Algorithm.Transactions.TransactionRecord);
var statisticsResults = GenerateStatisticsResults(charts, profitLoss);
var runtime = GetAlgorithmRuntimeStatistics(statisticsResults.Summary);
FinalStatistics = statisticsResults.Summary;
// clear the trades collection before placing inside the backtest result
foreach (var ap in statisticsResults.RollingPerformances.Values)
{
ap.ClosedTrades.Clear();
}
//Create a result packet to send to the browser.
var result = new BacktestResultPacket(_job,
new BacktestResult(charts, orders, profitLoss, statisticsResults.Summary, runtime, statisticsResults.RollingPerformances, statisticsResults.TotalPerformance)
{
AlphaRuntimeStatistics = AlphaRuntimeStatistics
})
{
ProcessingTime = (DateTime.UtcNow - StartTime).TotalSeconds,
DateFinished = DateTime.Now,
Progress = 1
};
//Place result into storage.
StoreResult(result);
//Second, send the truncated packet:
MessagingHandler.Send(result);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// The main processing method steps through the messaging queue and processes the messages one by one.
/// </summary>
public void Run()
{
//Setup minimum result arrays:
//SampleEquity(job.periodStart, job.startingCapital);
//SamplePerformance(job.periodStart, 0);
try
{
while (!(_exitTriggered && Messages.Count == 0))
{
//While there's no work to do, go back to the algorithm:
if (Messages.Count == 0)
{
Thread.Sleep(50);
}
else
{
//1. Process Simple Messages in Queue
Packet packet;
if (Messages.TryDequeue(out packet))
{
MessagingHandler.Send(packet);
}
}
//2. Update the packet scanner:
Update();
} // While !End.
}
catch (Exception err)
{
// unexpected error, we need to close down shop
Log.Error(err);
// quit the algorithm due to error
Algorithm.RunTimeError = err;
}
Log.Trace("BacktestingResultHandler.Run(): Ending Thread...");
IsActive = false;
// reset standard out/error
Console.SetOut(StandardOut);
Console.SetError(StandardError);
} // End Run();
private IList <ChatPayloadDTO> ProcessRequestMessaging(string userId, Message message)
{
IList <ChatPayloadDTO> result = new List <ChatPayloadDTO>();
message.User = GetUserById(userId);
message = new MessagingHandler(_userService, _roomService)
.Handle(userId, message);
// Composing message with target user informations
if (message.Event != EventEnum.Error &&
!String.IsNullOrWhiteSpace(message.TargetUserId))
{
message.TargetUser = GetUserById(message.TargetUserId);
}
result.Add(message.MessageToPayload(userId));
return(result);
}
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
/// <param name="job">Lean AlgorithmJob task</param>
/// <param name="orders">Collection of orders from the algorithm</param>
/// <param name="profitLoss">Collection of time-profit values for the algorithm</param>
/// <param name="holdings">Current holdings state for the algorithm</param>
/// <param name="cashbook">Cashbook for the holdingss</param>
/// <param name="statisticsResults">Statistics information for the algorithm (empty if not finished)</param>
/// <param name="banner">Runtime statistics banner information</param>
public void SendFinalResult(AlgorithmNodePacket job, Dictionary <int, Order> orders, Dictionary <DateTime, decimal> profitLoss, Dictionary <string, Holding> holdings, CashBook cashbook, StatisticsResults statisticsResults, Dictionary <string, string> banner)
{
try
{
FinalStatistics = statisticsResults.Summary;
//Convert local dictionary:
var charts = new Dictionary <string, Chart>(Charts);
_processingFinalPacket = true;
// clear the trades collection before placing inside the backtest result
foreach (var ap in statisticsResults.RollingPerformances.Values)
{
ap.ClosedTrades.Clear();
}
//Create a result packet to send to the browser.
var result = new BacktestResultPacket((BacktestNodePacket)job,
new BacktestResult(charts, orders, profitLoss, statisticsResults.Summary, banner, statisticsResults.RollingPerformances, statisticsResults.TotalPerformance)
{
AlphaRuntimeStatistics = AlphaRuntimeStatistics
})
{
ProcessingTime = (DateTime.UtcNow - StartTime).TotalSeconds,
DateFinished = DateTime.Now,
Progress = 1
};
//Place result into storage.
StoreResult(result);
//Second, send the truncated packet:
MessagingHandler.Send(result);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error(err);
}
}
// Protected Functions.
public override VirtualMachine CreateVM()
{
Transform thisTra = transform;
GameObject msgHandObj = new GameObject("_MessagingHandler");
MessagingHandler msgHnd = msgHandObj.AddComponent <MessagingHandler>();
msgHandObj.transform.parent = thisTra;
GameObject vmObj = new GameObject("_VM");
vmObj.transform.parent = thisTra;
VM vm = vmObj.AddComponent <VM>();
vm.m_MessagingHandler = msgHnd;
vm.scriptEngineBehaviour = this;
// Dont Show in Hierarchy these objects.
msgHandObj.hideFlags = HideFlags.HideInHierarchy;
vmObj.hideFlags = HideFlags.HideInHierarchy;
return(vm);
}
/// <summary>
/// Sends message to every receiver found on the GameObject root.
/// </summary>
/// <param name="target">Target.</param>
/// <param name="handler">Callback. Use this to send the message.</param>
/// <returns>Whether the message has been received.</returns>
public static bool SendMessageToAllOnRoot <T>(this Component target, MessagingHandler <T> handler)
{
return(target.Root().SendMessageToAll(handler));
}
} // End Run();
/// <summary>
/// Send a backtest update to the browser taking a latest snapshot of the charting data.
/// </summary>
public void Update()
{
try
{
//Sometimes don't run the update, if not ready or we're ending.
if (Algorithm?.Transactions == null || _processingFinalPacket)
{
return;
}
if (DateTime.UtcNow <= _nextUpdate || _daysProcessed < _daysProcessedFrontier)
{
return;
}
//Extract the orders since last update
var deltaOrders = new Dictionary <int, Order>();
try
{
deltaOrders = (from order in TransactionHandler.Orders
where order.Value.Time.Date >= _lastUpdate && order.Value.Status == OrderStatus.Filled
select order).ToDictionary(t => t.Key, t => t.Value);
}
catch (Exception err)
{
Log.Error(err, "Transactions");
}
//Limit length of orders we pass back dynamically to avoid flooding.
if (deltaOrders.Count > 50)
{
deltaOrders.Clear();
}
//Reset loop variables:
try
{
_lastUpdate = Algorithm.UtcTime.Date;
_daysProcessedFrontier = _daysProcessed + 1;
_nextUpdate = DateTime.UtcNow.AddSeconds(2);
}
catch (Exception err)
{
Log.Error(err, "Can't update variables");
}
var deltaCharts = new Dictionary <string, Chart>();
var performanceCharts = new Dictionary <string, Chart>();
lock (ChartLock)
{
//Get the updates since the last chart
foreach (var kvp in Charts)
{
var chart = kvp.Value;
deltaCharts.Add(chart.Name, chart.GetUpdates());
if (AlgorithmPerformanceCharts.Contains(kvp.Key))
{
performanceCharts[kvp.Key] = chart.Clone();
}
}
}
//Get the runtime statistics from the user algorithm:
var runtimeStatistics = new Dictionary <string, string>();
lock (RuntimeStatistics)
{
foreach (var pair in RuntimeStatistics)
{
runtimeStatistics.Add(pair.Key, pair.Value);
}
}
var summary = GenerateStatisticsResults(performanceCharts).Summary;
GetAlgorithmRuntimeStatistics(summary, runtimeStatistics);
//Profit Loss Changes:
var progress = Convert.ToDecimal(_daysProcessed / _jobDays);
if (progress > 0.999m)
{
progress = 0.999m;
}
//1. Cloud Upload -> Upload the whole packet to S3 Immediately:
if (DateTime.UtcNow > _nextS3Update)
{
// For intermediate backtesting results, we truncate the order list to include only the last 100 orders
// The final packet will contain the full list of orders.
const int maxOrders = 100;
var orderCount = TransactionHandler.Orders.Count;
var completeResult = new BacktestResult(
Charts,
orderCount > maxOrders ? TransactionHandler.Orders.Skip(orderCount - maxOrders).ToDictionary() : TransactionHandler.Orders.ToDictionary(),
Algorithm.Transactions.TransactionRecord,
new Dictionary <string, string>(),
runtimeStatistics,
new Dictionary <string, AlgorithmPerformance>());
StoreResult(new BacktestResultPacket(_job, completeResult, progress));
_nextS3Update = DateTime.UtcNow.AddSeconds(30);
}
//2. Backtest Update -> Send the truncated packet to the backtester:
var splitPackets = SplitPackets(deltaCharts, deltaOrders, runtimeStatistics, progress);
foreach (var backtestingPacket in splitPackets)
{
MessagingHandler.Send(backtestingPacket);
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Sends message to every receiver found on the GameObject root or his children.
/// </summary>
/// <param name="target">Target.</param>
/// <param name="handler">Callback. Use this to send the message.</param>
/// <returns>Whether the message has been received.</returns>
public static bool SendMessageToAllRootChildren <T>(this Component target, MessagingHandler <T> handler)
{
return(target.Root().SendMessageToAllChildren(handler));
}
} // End Run();
/// <summary>
/// Send a backtest update to the browser taking a latest snapshot of the charting data.
/// </summary>
private void Update()
{
try
{
//Sometimes don't run the update, if not ready or we're ending.
if (Algorithm?.Transactions == null || ExitTriggered)
{
return;
}
var utcNow = DateTime.UtcNow;
if (utcNow <= _nextUpdate || _daysProcessed < _daysProcessedFrontier)
{
return;
}
var deltaOrders = GetDeltaOrders(LastDeltaOrderPosition, shouldStop: orderCount => orderCount >= 50);
// Deliberately skip to the end of order event collection to prevent overloading backtesting UX
LastDeltaOrderPosition = TransactionHandler.OrderEvents.Count();
//Reset loop variables:
try
{
_daysProcessedFrontier = _daysProcessed + 1;
_nextUpdate = utcNow.AddSeconds(3);
}
catch (Exception err)
{
Log.Error(err, "Can't update variables");
}
var deltaCharts = new Dictionary <string, Chart>();
var serverStatistics = GetServerStatistics(utcNow);
var performanceCharts = new Dictionary <string, Chart>();
lock (ChartLock)
{
//Get the updates since the last chart
foreach (var kvp in Charts)
{
var chart = kvp.Value;
var updates = chart.GetUpdates();
if (!updates.IsEmpty())
{
deltaCharts.Add(chart.Name, updates);
}
if (AlgorithmPerformanceCharts.Contains(kvp.Key))
{
performanceCharts[kvp.Key] = chart.Clone();
}
}
}
//Get the runtime statistics from the user algorithm:
var runtimeStatistics = new Dictionary <string, string>();
lock (RuntimeStatistics)
{
foreach (var pair in RuntimeStatistics)
{
runtimeStatistics.Add(pair.Key, pair.Value);
}
}
var summary = GenerateStatisticsResults(performanceCharts).Summary;
GetAlgorithmRuntimeStatistics(summary, runtimeStatistics);
//Profit Loss Changes:
var progress = Convert.ToDecimal(_daysProcessed / _jobDays);
if (progress > 0.999m)
{
progress = 0.999m;
}
//1. Cloud Upload -> Upload the whole packet to S3 Immediately:
if (utcNow > _nextS3Update)
{
// For intermediate backtesting results, we truncate the order list to include only the last 100 orders
// The final packet will contain the full list of orders.
const int maxOrders = 100;
var orderCount = TransactionHandler.Orders.Count;
var completeResult = new BacktestResult(new BacktestResultParameters(
Charts,
orderCount > maxOrders ? TransactionHandler.Orders.Skip(orderCount - maxOrders).ToDictionary() : TransactionHandler.Orders.ToDictionary(),
Algorithm.Transactions.TransactionRecord,
new Dictionary <string, string>(),
runtimeStatistics,
new Dictionary <string, AlgorithmPerformance>(),
// we store the last 100 order events, the final packet will contain the full list
TransactionHandler.OrderEvents.Reverse().Take(100).ToList()));
StoreResult(new BacktestResultPacket(_job, completeResult, Algorithm.EndDate, Algorithm.StartDate, progress));
_nextS3Update = DateTime.UtcNow.AddSeconds(30);
}
//2. Backtest Update -> Send the truncated packet to the backtester:
var splitPackets = SplitPackets(deltaCharts, deltaOrders, runtimeStatistics, progress, serverStatistics);
foreach (var backtestingPacket in splitPackets)
{
MessagingHandler.Send(backtestingPacket);
}
// let's re update this value after we finish just in case, so we don't re enter in the next loop
_nextUpdate = DateTime.UtcNow.AddSeconds(3);
}
catch (Exception err)
{
Log.Error(err);
}
}
private static bool SendMessageToAll <T>(this IReadOnlyCollection <T> receivers, MessagingHandler <T> handler)
{
foreach (var receiver in receivers)
{
handler(receiver);
}
return(receivers.Count > 0);
}
/// <summary>
/// Sends message to the first receiver found on the GameObject.
/// </summary>
/// <param name="target">Target.</param>
/// <param name="handler">Callback. Use this to send the message.</param>
/// <returns>Whether the message has been received.</returns>
public static bool SendMessage <T>(this Component target, MessagingHandler <T> handler)
{
return(target.GetComponent <T>().SendMessage(handler));
}
/// <summary>
/// Sends message to every receiver found on the GameObject.
/// </summary>
/// <param name="target">Target.</param>
/// <param name="handler">Callback. Use this to send the message.</param>
/// <returns>Whether the message has been received.</returns>
public static bool SendMessageToAll <T>(this GameObject target, MessagingHandler <T> handler)
{
return(target.GetComponents <T>().SendMessageToAll(handler));
}
/// <summary>
/// Sends message to the first receiver found on the GameObject or one of his children.
/// </summary>
/// <param name="target">Target.</param>
/// <param name="handler">Callback. Use this to send the message.</param>
/// <returns>Whether the message has been received.</returns>
public static bool SendMessageToChild <T>(this GameObject target, MessagingHandler <T> handler)
{
return(target.GetComponentInChildren <T>().SendMessage(handler));
}
/// <summary>
/// Sends message to every receiver found on the GameObject or one of his children.
/// </summary>
/// <param name="target">Target.</param>
/// <param name="handler">Callback. Use this to send the message.</param>
/// <returns>Whether the message has been received.</returns>
public static bool SendMessageToAllChildren <T>(this Component target, MessagingHandler <T> handler)
{
return(target.GetComponentsInChildren <T>().SendMessageToAll(handler));
}
/// <summary>
/// Send a final analysis result back to the IDE.
/// </summary>
protected void SendFinalResult()
{
try
{
//Convert local dictionary:
var charts = new Dictionary <string, Chart>(Charts);
var orders = new Dictionary <int, Order>(TransactionHandler.Orders);
var profitLoss = new SortedDictionary <DateTime, decimal>(Algorithm.Transactions.TransactionRecord);
var statisticsResults = GenerateStatisticsResults(charts, profitLoss);
var runtime = GetAlgorithmRuntimeStatistics(statisticsResults.Summary);
FinalStatistics = statisticsResults.Summary;
// clear the trades collection before placing inside the backtest result
foreach (var ap in statisticsResults.RollingPerformances.Values)
{
ap.ClosedTrades.Clear();
}
var orderEvents = TransactionHandler.OrderEvents.ToList();
//Create a result packet to send to the browser.
var result = new BacktestResultPacket(_job,
new BacktestResult(new BacktestResultParameters(charts, orders, profitLoss, statisticsResults.Summary, runtime, statisticsResults.RollingPerformances, orderEvents, statisticsResults.TotalPerformance, AlphaRuntimeStatistics)),
Algorithm.EndDate, Algorithm.StartDate)
{
ProcessingTime = (DateTime.UtcNow - StartTime).TotalSeconds,
DateFinished = DateTime.Now,
Progress = 1
};
// Create new connection factory
var factory = new ConnectionFactory()
{
HostName = "localhost"
};
using (var connection = factory.CreateConnection())
using (var channel = connection.CreateModel())
{
// Set up queue for RabbitMQ
channel.QueueDeclare(queue: "rabbitBroker",
durable: false,
exclusive: false,
autoDelete: false,
arguments: null);
// Create test message for RabbitMQProducer
string producerMessage = JsonConvert.SerializeObject(FinalStatistics, Formatting.Indented);
var producerBody = Encoding.UTF8.GetBytes(producerMessage);
channel.BasicPublish(exchange: "",
routingKey: "rabbitBroker",
basicProperties: null,
body: producerBody);
}
//Place result into storage.
StoreResult(result);
//Second, send the truncated packet:
MessagingHandler.Send(result);
Log.Trace("BacktestingResultHandler.SendAnalysisResult(): Processed final packet");
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Sends message to the first receiver found on the GameObject root or his children.
/// </summary>
/// <param name="target">Target.</param>
/// <param name="handler">Callback. Use this to send the message.</param>
/// <returns>Whether the message has been received.</returns>
public static bool SendMessageToRootChild <T>(this GameObject target, MessagingHandler <T> handler)
{
return(target.Root().SendMessageToChild(handler));
}