/// <summary>
/// Send a message to the QuantConnect Chart Streaming API.
/// </summary>
/// <param name="userId">User Id</param>
/// <param name="apiToken">API token for authentication</param>
/// <param name="packet">Packet to transmit</param>
public static void Transmit(int userId, string apiToken, Packet packet)
{
try
{
using (var client = new WebClient())
{
var tx = JsonConvert.SerializeObject(packet);
if (tx.Length > 10000)
{
Log.Trace("StreamingApi.Transmit(): Packet too long: " + packet.GetType());
}
var response = client.UploadValues("http://streaming.quantconnect.com", new NameValueCollection
{
{"uid", userId.ToString()},
{"token", apiToken},
{"tx", tx}
});
Log.Trace("StreamingApi.Transmit(): Sending Packet ({0})", packet.GetType());
//Deserialize the response from the streaming API and throw in error case.
var result = JsonConvert.DeserializeObject<Response>(System.Text.Encoding.UTF8.GetString(response));
if (result.Type == "error")
{
throw new Exception(result.Message);
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Send all types of packets
/// </summary>
public void Send(Packet packet)
{
Transmit(packet);
if (StreamingApi.IsEnabled)
{
StreamingApi.Transmit(_job.UserId, _job.Channel, packet);
}
}
/// <summary>
/// Send a message to the _server using ZeroMQ
/// </summary>
/// <param name="packet">Packet to transmit</param>
public void Transmit(Packet packet)
{
var payload = JsonConvert.SerializeObject(packet);
var message = new NetMQMessage();
message.Append(payload);
_server.SendMultipartMessage(message);
}
/// <summary>
/// Send a generic base packet without processing
/// </summary>
public void Send(Packet packet)
{
switch (packet.Type)
{
case PacketType.Debug:
var debug = (DebugPacket) packet;
Log.Trace("Debug: " + debug.Message);
break;
case PacketType.Log:
var log = (LogPacket) packet;
Log.Trace("Log: " + log.Message);
break;
case PacketType.RuntimeError:
var runtime = (RuntimeErrorPacket) packet;
var rstack = (!string.IsNullOrEmpty(runtime.StackTrace) ? (Environment.NewLine + " " + runtime.StackTrace) : string.Empty);
Log.Error(runtime.Message + rstack);
break;
case PacketType.HandledError:
var handled = (HandledErrorPacket) packet;
var hstack = (!string.IsNullOrEmpty(handled.StackTrace) ? (Environment.NewLine + " " + handled.StackTrace) : string.Empty);
Log.Error(handled.Message + hstack);
break;
case PacketType.BacktestResult:
var result = (BacktestResultPacket) packet;
if (result.Progress == 1)
{
// uncomment these code traces to help write regression tests
//Console.WriteLine("new Dictionary<string, string>");
//Console.WriteLine("\t\t\t{");
foreach (var pair in result.Results.Statistics)
{
Log.Trace("STATISTICS:: " + pair.Key + " " + pair.Value);
//Console.WriteLine("\t\t\t\t{{\"{0}\",\"{1}\"}},", pair.Key, pair.Value);
}
//Console.WriteLine("\t\t\t};");
//foreach (var pair in statisticsResults.RollingPerformances)
//{
// Log.Trace("ROLLINGSTATS:: " + pair.Key + " SharpeRatio: " + Math.Round(pair.Value.PortfolioStatistics.SharpeRatio, 3));
//}
}
break;
}
if (StreamingApi.IsEnabled)
{
StreamingApi.Transmit(_job.UserId, _job.Channel, packet);
}
}
/// <summary>
/// Send a message to the QuantConnect Chart Streaming API.
/// </summary>
/// <param name="userId">User Id</param>
/// <param name="apiToken">API token for authentication</param>
/// <param name="packet">Packet to transmit</param>
public static void Transmit(int userId, string apiToken, Packet packet)
{
try
{
var tx = JsonConvert.SerializeObject(packet);
if (tx.Length > 10000)
{
Log.Trace("StreamingApi.Transmit(): Packet too long: " + packet.GetType());
return;
}
if (userId == 0)
{
Log.Error("StreamingApi.Transmit(): UserId is not set. Check your config.json file 'job-user-id' property.");
return;
}
if (apiToken == "")
{
Log.Error("StreamingApi.Transmit(): API Access token not set. Check your config.json file 'api-access-token' property.");
return;
}
var request = new RestRequest();
request.AddParameter("uid", userId);
request.AddParameter("token", apiToken);
request.AddParameter("tx", tx);
Client.ExecuteAsyncPost(request, (response, handle) =>
{
try
{
var result = JsonConvert.DeserializeObject<Response>(response.Content);
if (result.Type == "error")
{
Log.Error(new Exception(result.Message), "PacketType: " + packet.Type);
}
}
catch
{
Log.Error("StreamingApi.Client.ExecuteAsyncPost(): Error deserializing JSON content.");
}
}, "POST");
}
catch (Exception err)
{
Log.Error(err, "PacketType: " + packet.Type);
}
}
/// <summary>
/// Send any message with a base type of Packet.
/// </summary>
/// <param name="packet"></param>
public void Send(Packet packet)
{
switch (packet.Type)
{
case PacketType.Debug:
var debug = (DebugPacket)packet;
DebugEvent(debug);
break;
case PacketType.Log:
var log = (LogPacket)packet;
LogEvent(log);
break;
case PacketType.RuntimeError:
var runtime = (RuntimeErrorPacket)packet;
RuntimeErrorEvent(runtime);
break;
case PacketType.HandledError:
var handled = (HandledErrorPacket)packet;
HandledErrorEvent(handled);
break;
case PacketType.BacktestResult:
var result = (BacktestResultPacket)packet;
BacktestResultEvent(result);
break;
}
if (StreamingApi.IsEnabled)
{
StreamingApi.Transmit(_job.UserId, _job.Channel, packet);
}
}
/// <summary>
/// Save the snapshot of the total results to storage.
/// </summary>
/// <param name="packet">Packet to store.</param>
/// <param name="async">Store the packet asyncronously to speed up the thread.</param>
/// <remarks>Async creates crashes in Mono 3.10 if the thread disappears before the upload is complete so it is disabled for now.</remarks>
public void StoreResult(Packet packet, bool async = false)
{
//Initialize:
var serialized = "";
var key = "";
try
{
lock (_chartLock)
{
//1. Make sure this is the right type of packet:
if (packet.Type != PacketType.BacktestResult) return;
//2. Port to packet format:
var result = packet as BacktestResultPacket;
if (result != null)
{
//3. Get Storage Location:
key = "backtests/" + _job.UserId + "/" + _job.ProjectId + "/" + _job.BacktestId + ".json";
//4. Serialize to JSON:
serialized = JsonConvert.SerializeObject(result.Results);
}
else
{
Log.Error("BacktestingResultHandler.StoreResult(): Result Null.");
}
//Upload Results Portion
Engine.Api.Store(serialized, key, StoragePermissions.Authenticated, async);
}
}
catch (Exception err)
{
Log.Error("BacktestingResultHandler.StoreResult(): " + err.Message);
}
}
public void StoreResult(Packet packet, bool async = false)
{
}
/// <summary>
/// Store result on desktop.
/// </summary>
/// <param name="packet">Packet of data to store.</param>
/// <param name="async">Store the packet asyncronously to speed up the thread.</param>
/// <remarks>Async creates crashes in Mono 3.10 if the thread disappears before the upload is complete so it is disabled for now.</remarks>
public void StoreResult(Packet packet, bool async = false)
{
// Do nothing.
}
/// <summary>
/// Save the snapshot of the total results to storage.
/// </summary>
/// <param name="packet">Packet to store.</param>
/// <param name="async">Store the packet asyncronously to speed up the thread.</param>
/// <remarks>
/// Async creates crashes in Mono 3.10 if the thread disappears before the upload is complete so it is disabled for now.
/// For live trading we're making assumption its a long running task and safe to async save large files.
/// </remarks>
public void StoreResult(Packet packet, bool async = true)
{
// this will hold all the serialized data and the keys to be stored
var data_keys = Enumerable.Range(0, 0).Select(x => new
{
Key = (string)null,
Serialized = (string)null
}).ToList();
try
{
Log.Debug("LiveTradingResultHandler.StoreResult(): Begin store result sampling");
lock (_chartLock)
{
// Make sure this is the right type of packet:
if (packet.Type != PacketType.LiveResult) return;
// Port to packet format:
var live = packet as LiveResultPacket;
if (live != null)
{
// we need to down sample
var start = DateTime.UtcNow.Date;
var stop = start.AddDays(1);
// truncate to just today, we don't need more than this for anyone
Truncate(live.Results, start, stop);
var highResolutionCharts = new Dictionary<string, Chart>(live.Results.Charts);
// 10 minute resolution data, save today
var tenminuteSampler = new SeriesSampler(TimeSpan.FromMinutes(10));
var tenminuteCharts = tenminuteSampler.SampleCharts(live.Results.Charts, start, stop);
live.Results.Charts = tenminuteCharts;
data_keys.Add(new
{
Key = CreateKey("10minute"),
Serialized = JsonConvert.SerializeObject(live.Results)
});
// minute resoluton data, save today
var minuteSampler = new SeriesSampler(TimeSpan.FromMinutes(1));
var minuteCharts = minuteSampler.SampleCharts(live.Results.Charts, start, stop);
// swap out our charts with the sampeld data
live.Results.Charts = minuteCharts;
data_keys.Add(new
{
Key = CreateKey("minute"),
Serialized = JsonConvert.SerializeObject(live.Results)
});
// high resolution data, we only want to save an hour
live.Results.Charts = highResolutionCharts;
start = DateTime.UtcNow.RoundDown(TimeSpan.FromHours(1));
stop = DateTime.UtcNow.RoundUp(TimeSpan.FromHours(1));
Truncate(live.Results, start, stop);
foreach (var name in live.Results.Charts.Keys)
{
var newPacket = new LiveResult();
newPacket.Orders = new Dictionary<int, Order>(live.Results.Orders);
newPacket.Holdings = new Dictionary<string, Holding>(live.Results.Holdings);
newPacket.Charts = new Dictionary<string, Chart>();
newPacket.Charts.Add(name, live.Results.Charts[name]);
data_keys.Add(new
{
Key = CreateKey("second_" + Uri.EscapeUriString(name), "yyyy-MM-dd-HH"),
Serialized = JsonConvert.SerializeObject(newPacket)
});
}
}
else
{
Log.Error("LiveResultHandler.StoreResult(): Result Null.");
}
}
Log.Debug("LiveTradingResultHandler.StoreResult(): End store result sampling");
// Upload Results Portion
foreach (var dataKey in data_keys)
{
_api.Store(dataKey.Serialized, dataKey.Key, StoragePermissions.Authenticated, async);
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Send a generic base packet without processing
/// </summary>
public void Send(Packet packet)
{
//
}
/// <summary>
/// Send any message with a base type of Packet.
/// </summary>
public void Send(Packet packet)
{
//Until we're loaded queue it up
if (!_loaded)
{
_queue.Enqueue(packet);
return;
}
//Catch up if this is the first time
while (_queue.Count > 0)
{
ProcessPacket(_queue.Dequeue());
}
//Finally process this new packet
ProcessPacket(packet);
}
/// <summary>
/// Packet processing implementation
/// </summary>
private void ProcessPacket(Packet packet)
{
//Packets we handled in the UX.
switch (packet.Type)
{
case PacketType.Debug:
var debug = (DebugPacket)packet;
OnDebugEvent(debug);
break;
case PacketType.Log:
var log = (LogPacket)packet;
OnLogEvent(log);
break;
case PacketType.RuntimeError:
var runtime = (RuntimeErrorPacket)packet;
OnRuntimeErrorEvent(runtime);
break;
case PacketType.HandledError:
var handled = (HandledErrorPacket)packet;
OnHandledErrorEvent(handled);
break;
case PacketType.BacktestResult:
var result = (BacktestResultPacket)packet;
OnBacktestResultEvent(result);
break;
}
if (StreamingApi.IsEnabled)
{
StreamingApi.Transmit(_job.UserId, _job.Channel, packet);
}
}