private ClusterNodeStatusReport BuildReport(BlittableJsonReaderObject rawReport, TcpConnectionHeaderMessage.SupportedFeatures supportedFeatures)
{
using (rawReport)
{
if (supportedFeatures.Heartbeats.IncludeServerInfo)
{
var maintenanceReport = JsonDeserializationServer.MaintenanceReport(rawReport);
return(new ClusterNodeStatusReport(
maintenanceReport.ServerReport,
maintenanceReport.DatabasesReport,
GetStatus(),
null,
DateTime.UtcNow,
_lastSuccessfulReceivedReport));
ClusterNodeStatusReport.ReportStatus GetStatus()
{
if (maintenanceReport.ServerReport.OutOfCpuCredits == true)
{
return(ClusterNodeStatusReport.ReportStatus.OutOfCredits);
}
if (maintenanceReport.ServerReport.EarlyOutOfMemory == true)
{
return(ClusterNodeStatusReport.ReportStatus.EarlyOutOfMemory);
}
if (maintenanceReport.ServerReport.HighDirtyMemory == true)
{
return(ClusterNodeStatusReport.ReportStatus.HighDirtyMemory);
}
return(ClusterNodeStatusReport.ReportStatus.Ok);
}
}
var report = new Dictionary <string, DatabaseStatusReport>();
foreach (var property in rawReport.GetPropertyNames())
{
var value = (BlittableJsonReaderObject)rawReport[property];
report.Add(property, JsonDeserializationServer.DatabaseStatusReport(value));
}
return(new ClusterNodeStatusReport(
new ServerReport(),
report,
ClusterNodeStatusReport.ReportStatus.Ok,
null,
DateTime.UtcNow,
_lastSuccessfulReceivedReport));
}
}
private ClusterNodeStatusReport BuildReport(BlittableJsonReaderObject rawReport)
{
using (rawReport)
{
var report = new Dictionary <string, DatabaseStatusReport>();
foreach (var property in rawReport.GetPropertyNames())
{
var value = (BlittableJsonReaderObject)rawReport[property];
report.Add(property, JsonDeserializationServer.DatabaseStatusReport(value));
}
return(new ClusterNodeStatusReport(
report,
ClusterNodeStatusReport.ReportStatus.Ok,
null,
DateTime.UtcNow,
_lastSuccessfulReceivedReport));
}
}
private async Task ListenToMaintenanceWorker()
{
bool needToWait = false;
var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan;
var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
TcpConnectionInfo tcpConnection = null;
try
{
tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor",
_parent._server.RavenServer.ClusterCertificateHolder?.Certificate);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"ClusterMaintenanceSupervisor() => Failed to add to cluster node key = {ClusterTag}", e);
}
}
while (_token.IsCancellationRequested == false)
{
var internalTaskCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(_token);
try
{
if (needToWait)
{
needToWait = false; // avoid tight loop if there was timeout / error
await TimeoutManager.WaitFor(onErrorDelayTime, _token);
tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor",
_parent._server.RavenServer.ClusterCertificateHolder.Certificate);
}
if (tcpConnection == null)
{
needToWait = true;
continue;
}
var(tcpClient, connection) = await ConnectToClientNodeAsync(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
using (tcpClient)
using (_cts.Token.Register(tcpClient.Dispose))
using (connection)
{
while (_token.IsCancellationRequested == false)
{
using (_contextPool.AllocateOperationContext(out JsonOperationContext context))
{
var readResponseTask = context.ReadForMemoryAsync(connection, _readStatusUpdateDebugString, internalTaskCancellationToken.Token);
var timeout = TimeoutManager.WaitFor(receiveFromWorkerTimeout, _token);
if (await Task.WhenAny(readResponseTask.AsTask(), timeout) == timeout)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Timeout occurred while collecting info from {ClusterTag}");
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Timeout,
null,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
internalTaskCancellationToken.Cancel();
break;
}
using (var statusUpdateJson = await readResponseTask)
{
var report = new Dictionary <string, DatabaseStatusReport>();
foreach (var property in statusUpdateJson.GetPropertyNames())
{
var value = (BlittableJsonReaderObject)statusUpdateJson[property];
report.Add(property, JsonDeserializationServer.DatabaseStatusReport(value));
}
ReceivedReport = new ClusterNodeStatusReport(
report,
ClusterNodeStatusReport.ReportStatus.Ok,
null,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
_lastSuccessfulReceivedReport = ReceivedReport;
}
}
}
}
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
}
finally
{
internalTaskCancellationToken.Dispose();
}
}
}