private void ListenToMaintenanceWorker()
{
var needToWait = false;
var firstIteration = true;
var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan;
var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
var tcpTimeout = _parent.Config.TcpConnectionTimeout.AsTimeSpan;
if (tcpTimeout < receiveFromWorkerTimeout)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Warning: TCP timeout is lower than the receive from worker timeout ({tcpTimeout} < {receiveFromWorkerTimeout}), " +
"this could affect the cluster observer's decisions.");
}
}
TcpConnectionInfo tcpConnection = null;
while (_token.IsCancellationRequested == false)
{
try
{
if (needToWait)
{
needToWait = false; // avoid tight loop if there was timeout / error
if (firstIteration == false)
{
_token.WaitHandle.WaitOne(onErrorDelayTime);
}
firstIteration = false;
using (var timeout = new CancellationTokenSource(tcpTimeout))
using (var combined = CancellationTokenSource.CreateLinkedTokenSource(_token, timeout.Token))
{
tcpConnection = ReplicationUtils.GetTcpInfo(Url, null, "Supervisor", _parent._server.Server.Certificate.Certificate, combined.Token);
}
}
if (tcpConnection == null)
{
needToWait = true;
continue;
}
var connection = ConnectToClientNode(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
var tcpClient = connection.TcpClient;
var stream = connection.Stream;
using (tcpClient)
using (_cts.Token.Register(tcpClient.Dispose))
using (_contextPool.AllocateOperationContext(out JsonOperationContext context))
using (var timeoutEvent = new TimeoutEvent(receiveFromWorkerTimeout, $"Timeout event for: {_name}", singleShot: false))
{
timeoutEvent.Start(OnTimeout);
while (_token.IsCancellationRequested == false)
{
BlittableJsonReaderObject rawReport;
try
{
// even if there is a timeout event, we will keep waiting on the same connection until the TCP timeout occurs.
rawReport = context.ReadForMemory(stream, _readStatusUpdateDebugString);
timeoutEvent.Defer(_parent._leaderClusterTag);
}
catch (Exception e)
{
if (_token.IsCancellationRequested)
{
return;
}
if (_log.IsInfoEnabled)
{
_log.Info("Exception occurred while reading the report from the connection", e);
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
break;
}
var report = BuildReport(rawReport);
timeoutEvent.Defer(_parent._leaderClusterTag);
ReceivedReport = _lastSuccessfulReceivedReport = report;
}
}
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
}
}
}
private void ListenToMaintenanceWorker()
{
var firstIteration = true;
var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan;
var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
var tcpTimeout = _parent.Config.TcpConnectionTimeout.AsTimeSpan;
if (tcpTimeout < receiveFromWorkerTimeout)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Warning: TCP timeout is lower than the receive from worker timeout ({tcpTimeout} < {receiveFromWorkerTimeout}), " +
"this could affect the cluster observer's decisions.");
}
}
while (_token.IsCancellationRequested == false)
{
try
{
if (firstIteration == false)
{
// avoid tight loop if there was timeout / error
_token.WaitHandle.WaitOne(onErrorDelayTime);
if (_token.IsCancellationRequested)
{
return;
}
}
firstIteration = false;
TcpConnectionInfo tcpConnection = null;
using (var timeout = new CancellationTokenSource(tcpTimeout))
using (var combined = CancellationTokenSource.CreateLinkedTokenSource(_token, timeout.Token))
{
tcpConnection = ReplicationUtils.GetTcpInfo(Url, null, "Supervisor", _parent._server.Server.Certificate.Certificate, combined.Token);
if (tcpConnection == null)
{
continue;
}
}
var connection = ConnectToClientNode(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
var tcpClient = connection.TcpClient;
var stream = connection.Stream;
using (tcpClient)
using (_cts.Token.Register(tcpClient.Dispose))
using (_contextPool.AllocateOperationContext(out JsonOperationContext contextForParsing))
using (_contextPool.AllocateOperationContext(out JsonOperationContext contextForBuffer))
using (contextForBuffer.GetMemoryBuffer(out var readBuffer))
using (var timeoutEvent = new TimeoutEvent(receiveFromWorkerTimeout, $"Timeout event for: {_name}", singleShot: false))
{
timeoutEvent.Start(OnTimeout);
var unchangedReports = new List <DatabaseStatusReport>();
while (_token.IsCancellationRequested == false)
{
contextForParsing.Reset();
contextForParsing.Renew();
BlittableJsonReaderObject rawReport;
try
{
// even if there is a timeout event, we will keep waiting on the same connection until the TCP timeout occurs.
rawReport = contextForParsing.Sync.ParseToMemory(stream, _readStatusUpdateDebugString, BlittableJsonDocumentBuilder.UsageMode.None, readBuffer);
timeoutEvent.Defer(_parent._leaderClusterTag);
}
catch (Exception e)
{
if (_token.IsCancellationRequested)
{
return;
}
if (_log.IsInfoEnabled)
{
_log.Info("Exception occurred while reading the report from the connection", e);
}
ReceivedReport = new ClusterNodeStatusReport(new ServerReport(), new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
break;
}
_parent.ForTestingPurposes?.BeforeReportBuildAction(this);
var nodeReport = BuildReport(rawReport, connection.SupportedFeatures);
timeoutEvent.Defer(_parent._leaderClusterTag);
UpdateNodeReportIfNeeded(nodeReport, unchangedReports);
unchangedReports.Clear();
ReceivedReport = _lastSuccessfulReceivedReport = nodeReport;
_parent.ForTestingPurposes?.AfterSettingReportAction(this);
}
}
}
catch (Exception e)
{
if (_token.IsCancellationRequested)
{
return;
}
if (_log.IsInfoEnabled)
{
_log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
}
ReceivedReport = new ClusterNodeStatusReport(new ServerReport(), new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
}
}
}