private void ConnectSocket(TcpConnectionInfo connection, TcpClient tcpClient)
{
var uri = new Uri(connection.Url);
var host = uri.Host;
var port = uri.Port;
try
{
tcpClient.ConnectAsync(host, port).Wait(CancellationToken);
}
catch (SocketException e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Failed to connect to remote replication destination {connection.Url}. Socket Error Code = {e.SocketErrorCode}", e);
}
throw;
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Failed to connect to remote replication destination {connection.Url}", e);
}
throw;
}
}
public void RawDataReceived(TcpConnectionInfo connection, byte[] data)
{
if (_channelDataReceivedAction != null)
{
ChannelData <byte[]> channelData = new ChannelData <byte[]>(data, connection.RemoteEndPoint, DateTime.Now);
_channelDataReceivedAction(channelData);
}
}
public static async Task ConnectToClientNodeAsync(RavenServer server, TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout, Logger log, string database,
NodeConnectionTestResult result, CancellationToken token = default)
{
List <string> negLogs = new();
using (server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext ctx))
{
using var tcpSocketResult = await TcpUtils.ConnectSecuredTcpSocket(tcpConnectionInfo, server.Certificate.Certificate, server.CipherSuitesPolicy,
TcpConnectionHeaderMessage.OperationTypes.TestConnection,
NegotiateWithRemote, ctx, timeout, negLogs, token);
}
result.Log = negLogs;
async Task <TcpConnectionHeaderMessage.SupportedFeatures> NegotiateWithRemote(string url, TcpConnectionInfo info, Stream stream, JsonOperationContext context, List <string> logs = null)
{
await using (var writer = new AsyncBlittableJsonTextWriter(context, stream))
{
await WriteOperationHeaderToRemote(writer, TcpConnectionHeaderMessage.OperationTypes.TestConnection, database, info.ServerId);
using (var responseJson = await context.ReadForMemoryAsync(stream, $"TestConnectionHandler/{url}/Read-Handshake-Response"))
{
var headerResponse = JsonDeserializationServer.TcpConnectionHeaderResponse(responseJson);
switch (headerResponse.Status)
{
case TcpConnectionStatus.Ok:
result.Success = true;
result.Error = null;
logs?.Add($"Successfully negotiated with {url}.");
break;
case TcpConnectionStatus.AuthorizationFailed:
result.Success = false;
result.Error = $"Connection to {url} failed because of authorization failure: {headerResponse.Message}";
logs?.Add(result.Error);
throw new AuthorizationException(result.Error);
case TcpConnectionStatus.TcpVersionMismatch:
result.Success = false;
result.Error = $"Connection to {url} failed because of mismatching tcp version: {headerResponse.Message}";
logs?.Add(result.Error);
await WriteOperationHeaderToRemote(writer, TcpConnectionHeaderMessage.OperationTypes.Drop, database, info.ServerId);
throw new AuthorizationException(result.Error);
case TcpConnectionStatus.InvalidNetworkTopology:
result.Success = false;
result.Error = $"Connection to {url} failed because of {nameof(TcpConnectionStatus.InvalidNetworkTopology)} error: {headerResponse.Message}";
logs?.Add(result.Error);
throw new InvalidNetworkTopologyException(result.Error);
}
}
}
return(null);
}
}
private static int GetChromeListeningPortWithInterOp(int pid)
{
int port = 0;
if (pid > 0)
{
port = TcpConnectionInfo.GetChromeListeningPortWithInterOp(pid);
}
return(port);
}
public OutgoingReplicationHandler(ReplicationLoader parent, DocumentDatabase database, ReplicationNode node, bool external, TcpConnectionInfo connectionInfo)
{
_parent = parent;
_database = database;
Destination = node;
_external = external;
_log = LoggingSource.Instance.GetLogger <OutgoingReplicationHandler>(_database.Name);
_connectionInfo = connectionInfo;
_database.Changes.OnDocumentChange += OnDocumentChange;
_cts = CancellationTokenSource.CreateLinkedTokenSource(_database.DatabaseShutdown);
}
public static async Task <TcpClient> ConnectSocketAsync(TcpConnectionInfo connection, TimeSpan timeout, Logger log)
{
try
{
return(await ConnectAsync(connection.Url, timeout).ConfigureAwait(false));
}
catch (AggregateException ae) when(ae.InnerException is SocketException)
{
if (log.IsInfoEnabled)
{
log.Info(
$"Failed to connect to remote replication destination {connection.Url}. Socket Error Code = {((SocketException)ae.InnerException).SocketErrorCode}",
ae.InnerException);
}
throw;
}
catch (AggregateException ae) when(ae.InnerException is OperationCanceledException)
{
if (log.IsInfoEnabled)
{
log.Info(
$@"Tried to connect to remote replication destination {connection.Url}, but the operation was aborted.
This is not necessarily an issue, it might be that replication destination document has changed at
the same time we tried to connect. We will try to reconnect later.",
ae.InnerException);
}
throw;
}
catch (OperationCanceledException e)
{
if (log.IsInfoEnabled)
{
log.Info(
$@"Tried to connect to remote replication destination {connection.Url}, but the operation was aborted.
This is not necessarily an issue, it might be that replication destination document has changed at
the same time we tried to connect. We will try to reconnect later.",
e);
}
throw;
}
catch (Exception e)
{
if (log.IsInfoEnabled)
{
log.Info($"Failed to connect to remote replication destination {connection.Url}", e);
}
throw;
}
}
public void ClientDisconnected(TcpConnectionInfo connection, Exception ex)
{
if (ex != null)
{
OnDisconnected(connection.RemoteEndPoint, ex.Message);
_logWriter.Log(string.Format("Remote tcp client disconnected. IP:{0}, Port:{1}, Reason:{2}", connection.RemoteEndPoint.Address.ToString(), connection.RemoteEndPoint.Port, ex.Message));
_logWriter.Log("Tcp client disconnected", ex);
}
else
{
OnDisconnected(connection.RemoteEndPoint, string.Empty);
_logWriter.Log(string.Format("Remote tcp client disconnected. IP:{0}, Port:{1}, Reason:{2}", connection.RemoteEndPoint.Address.ToString(), connection.RemoteEndPoint.Port, string.Empty));
}
}
public static async Task ConnectToClientNodeAsync(RavenServer server, TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout, Logger log, string database, NodeConnectionTestResult result)
{
TcpClient tcpClient;
string url;
(tcpClient, url) = await TcpUtils.ConnectSocketAsync(tcpConnectionInfo, timeout, log);
var connection = await TcpUtils.WrapStreamWithSslAsync(tcpClient, tcpConnectionInfo, server.Certificate.Certificate, server.CipherSuitesPolicy, timeout);
using (tcpClient)
{
using (server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext ctx))
await using (var writer = new AsyncBlittableJsonTextWriter(ctx, connection))
{
await WriteOperationHeaderToRemote(writer, TcpConnectionHeaderMessage.OperationTypes.TestConnection, database);
using (var responseJson = await ctx.ReadForMemoryAsync(connection, $"TestConnectionHandler/{url}/Read-Handshake-Response"))
{
var headerResponse = JsonDeserializationServer.TcpConnectionHeaderResponse(responseJson);
switch (headerResponse.Status)
{
case TcpConnectionStatus.Ok:
result.Success = true;
break;
case TcpConnectionStatus.AuthorizationFailed:
result.Success = false;
result.Error = $"Connection to {url} failed because of authorization failure: {headerResponse.Message}";
break;
case TcpConnectionStatus.TcpVersionMismatch:
result.Success = false;
result.Error = $"Connection to {url} failed because of mismatching tcp version: {headerResponse.Message}";
await WriteOperationHeaderToRemote(writer, TcpConnectionHeaderMessage.OperationTypes.Drop, database);
break;
}
}
}
}
}
private async Task <DynamicJsonValue> ConnectToClientNodeAsync(TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout, Logger log)
{
var(tcpClient, connection) = await ConnectAndGetNetworkStreamAsync(tcpConnectionInfo, timeout, log);
using (tcpClient)
{
var result = new DynamicJsonValue();
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext ctx))
using (var writer = new BlittableJsonTextWriter(ctx, connection))
{
WriteOperationHeaderToRemote(writer);
using (var responseJson = await ctx.ReadForMemoryAsync(connection, $"TestConnectionHandler/{tcpConnectionInfo.Url}/Read-Handshake-Response"))
{
var headerResponse = JsonDeserializationServer.TcpConnectionHeaderResponse(responseJson);
switch (headerResponse.Status)
{
case TcpConnectionStatus.Ok:
result["Success"] = true;
break;
case TcpConnectionStatus.AuthorizationFailed:
result["Success"] = false;
result["Error"] = $"Connection to {tcpConnectionInfo.Url} failed because of authorization failure: {headerResponse.Message}";
break;
case TcpConnectionStatus.TcpVersionMismatch:
WriteOperationHeaderToRemote(writer, drop: true);
result["Success"] = false;
result["Error"] = $"Connection to {tcpConnectionInfo.Url} failed because of mismatching tcp version {headerResponse.Message}";
break;
}
}
}
return(result);
}
}
/// <summary>
/// Запуск клиента
/// </summary>
public void StartClient(string sIpAddr, int port, bool bCreateGame, string token)
{
mode = CLIENT;
if (Protocol == "TCP")
{
client = new TcpClient();
TcpConnectionInfo connectInfo = new TcpConnectionInfo();
gameClientSession = new TcpClientSession(client, connectInfo, context);
client.SetSession(gameClientSession);
(gameClientSession as TcpClientSession).OnReceiveFirstData += OnReceiveFirstData;
if (client.Connect(sIpAddr, port))
{
(gameClientSession as TcpClientSession).SendQueryInfo();
}
return;
}
if (Protocol == "HTTP")
{
client = new HttpClient();
HttpConnectionInfo connectInfo = new HttpConnectionInfo();
gameClientSession = new HttpClientSession(client, connectInfo, context);
client.SetSession(gameClientSession);
(gameClientSession as HttpClientSession).OnReceiveFirstData += OnReceiveFirstData;
if (client.Connect(sIpAddr, port))
{
if (bCreateGame)
{
(gameClientSession as HttpClientSession).SendCreateGame();
}
else
{
(gameClientSession as HttpClientSession).SendJoinToGame(token);
}
}
}
}
public static int GetChromeListeningPortWithInterOp(int pid)
{
int port = 0;
try
{
TcpConnectionInfo.MIB_TCPROW_OWNER_PID[] tcpConnections = TcpConnectionInfo.GetAllTcpConnections();
for (int i = 0; i < tcpConnections.Length; i++)
{
TcpConnectionInfo.MIB_TCPROW_OWNER_PID mIB_TCPROW_OWNER_PID = tcpConnections[i];
if (mIB_TCPROW_OWNER_PID.localAddr == 16777343u && mIB_TCPROW_OWNER_PID.owningPid == pid)
{
port = (int)mIB_TCPROW_OWNER_PID.LocalPort;
break;
}
}
}
catch (Exception ex)
{
Log.Error("GetChromeListeningPortWithInterOp,exp=" + ex.Message);
Trace.Assert(false, ex.Message);
}
return(port);
}
public void ClientConnected(TcpConnectionInfo connection)
{
OnConnected(connection.RemoteEndPoint);
_logWriter.Log(string.Format("Remote tcp client connected. IP:{0}, Port:{1}", connection.RemoteEndPoint.Address.ToString(), connection.RemoteEndPoint.Port));
}
private async Task <(TcpClient TcpClient, Stream Connection)> ConnectToClientNodeAsync(TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout)
{
var tcpClient = await TcpUtils.ConnectSocketAsync(tcpConnectionInfo, timeout, _log);
var connection = await TcpUtils.WrapStreamWithSslAsync(tcpClient, tcpConnectionInfo, _parent._server.RavenServer.ClusterCertificateHolder.Certificate);
using (_contextPool.AllocateOperationContext(out JsonOperationContext ctx))
using (var writer = new BlittableJsonTextWriter(ctx, connection))
{
WriteOperationHeaderToRemote(writer);
using (var responseJson = await ctx.ReadForMemoryAsync(connection, _readStatusUpdateDebugString + "/Read-Handshake-Response", _token))
{
var headerResponse = JsonDeserializationServer.TcpConnectionHeaderResponse(responseJson);
switch (headerResponse.Status)
{
case TcpConnectionStatus.Ok:
break;
case TcpConnectionStatus.AuthorizationFailed:
throw new UnauthorizedAccessException(
$"Node with ClusterTag = {ClusterTag} replied to initial handshake with authorization failure {headerResponse.Message}");
case TcpConnectionStatus.TcpVersionMissmatch:
throw new InvalidOperationException($"Node with ClusterTag = {ClusterTag} replied to initial handshake with missmatching tcp version {headerResponse.Message}");
}
}
WriteClusterMaintenanceConnectionHeader(writer);
}
return(tcpClient, connection);
}
private async Task ListenToMaintenanceWorker()
{
bool needToWait = false;
var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan;
var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
TcpConnectionInfo tcpConnection = null;
try
{
tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor",
_parent._server.RavenServer.ClusterCertificateHolder?.Certificate);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"ClusterMaintenanceSupervisor() => Failed to add to cluster node key = {ClusterTag}", e);
}
}
while (_token.IsCancellationRequested == false)
{
var internalTaskCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(_token);
try
{
if (needToWait)
{
needToWait = false; // avoid tight loop if there was timeout / error
await TimeoutManager.WaitFor(onErrorDelayTime, _token);
tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor",
_parent._server.RavenServer.ClusterCertificateHolder.Certificate);
}
if (tcpConnection == null)
{
needToWait = true;
continue;
}
var(tcpClient, connection) = await ConnectToClientNodeAsync(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
using (tcpClient)
using (_cts.Token.Register(tcpClient.Dispose))
using (connection)
{
while (_token.IsCancellationRequested == false)
{
using (_contextPool.AllocateOperationContext(out JsonOperationContext context))
{
var readResponseTask = context.ReadForMemoryAsync(connection, _readStatusUpdateDebugString, internalTaskCancellationToken.Token);
var timeout = TimeoutManager.WaitFor(receiveFromWorkerTimeout, _token);
if (await Task.WhenAny(readResponseTask.AsTask(), timeout) == timeout)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Timeout occurred while collecting info from {ClusterTag}");
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Timeout,
null,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
internalTaskCancellationToken.Cancel();
break;
}
using (var statusUpdateJson = await readResponseTask)
{
var report = new Dictionary <string, DatabaseStatusReport>();
foreach (var property in statusUpdateJson.GetPropertyNames())
{
var value = (BlittableJsonReaderObject)statusUpdateJson[property];
report.Add(property, JsonDeserializationServer.DatabaseStatusReport(value));
}
ReceivedReport = new ClusterNodeStatusReport(
report,
ClusterNodeStatusReport.ReportStatus.Ok,
null,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
_lastSuccessfulReceivedReport = ReceivedReport;
}
}
}
}
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
}
finally
{
internalTaskCancellationToken.Dispose();
}
}
}
internal static async Task <(TcpClient Client, string Url)> ConnectAsyncWithPriority(TcpConnectionInfo info, TimeSpan?tcpConnectionTimeout)
{
TcpClient tcpClient;
if (info.Urls != null)
{
foreach (var url in info.Urls)
{
try
{
tcpClient = await ConnectAsync(url, tcpConnectionTimeout).ConfigureAwait(false);
return(tcpClient, url);
}
catch
{
// ignored
}
}
}
tcpClient = await ConnectAsync(info.Url, tcpConnectionTimeout).ConfigureAwait(false);
return(tcpClient, info.Url);
}
public static async Task <Stream> WrapStreamWithSslAsync(TcpClient tcpClient, TcpConnectionInfo info, X509Certificate2 storeCertificate,
TimeSpan?timeout)
{
var networkStream = tcpClient.GetStream();
if (timeout != null)
{
networkStream.ReadTimeout =
networkStream.WriteTimeout = (int)timeout.Value.TotalMilliseconds;
}
Stream stream = networkStream;
if (info.Certificate == null)
{
return(stream);
}
var expectedCert = new X509Certificate2(Convert.FromBase64String(info.Certificate));
var sslStream = new SslStream(stream, false, (sender, actualCert, chain, errors) => expectedCert.Equals(actualCert));
await sslStream.AuthenticateAsClientAsync(new Uri(info.Url).Host, new X509CertificateCollection(new X509Certificate[] { storeCertificate }), SslProtocols.Tls12, false).ConfigureAwait(false);
stream = sslStream;
return(stream);
}
private (TcpClient TcpClient, Stream Connection) ConnectToClientNode(TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout)
{
return(AsyncHelpers.RunSync(() => ConnectToClientNodeAsync(tcpConnectionInfo, timeout)));
}
private ClusterMaintenanceConnection ConnectToClientNode(TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout)
{
return(AsyncHelpers.RunSync(() => ConnectToClientNodeAsync(tcpConnectionInfo, timeout)));
}
private void ListenToMaintenanceWorker()
{
var needToWait = false;
var firstIteration = true;
var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan;
var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
var tcpTimeout = _parent.Config.TcpConnectionTimeout.AsTimeSpan;
if (tcpTimeout < receiveFromWorkerTimeout)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Warning: TCP timeout is lower than the receive from worker timeout ({tcpTimeout} < {receiveFromWorkerTimeout}), " +
"this could affect the cluster observer's decisions.");
}
}
TcpConnectionInfo tcpConnection = null;
while (_token.IsCancellationRequested == false)
{
try
{
if (needToWait)
{
needToWait = false; // avoid tight loop if there was timeout / error
if (firstIteration == false)
{
_token.WaitHandle.WaitOne(onErrorDelayTime);
}
firstIteration = false;
using (var timeout = new CancellationTokenSource(tcpTimeout))
using (var combined = CancellationTokenSource.CreateLinkedTokenSource(_token, timeout.Token))
{
tcpConnection = ReplicationUtils.GetTcpInfo(Url, null, "Supervisor", _parent._server.Server.Certificate.Certificate, combined.Token);
}
}
if (tcpConnection == null)
{
needToWait = true;
continue;
}
var connection = ConnectToClientNode(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
var tcpClient = connection.TcpClient;
var stream = connection.Stream;
using (tcpClient)
using (_cts.Token.Register(tcpClient.Dispose))
using (_contextPool.AllocateOperationContext(out JsonOperationContext context))
using (var timeoutEvent = new TimeoutEvent(receiveFromWorkerTimeout, $"Timeout event for: {_name}", singleShot: false))
{
timeoutEvent.Start(OnTimeout);
while (_token.IsCancellationRequested == false)
{
BlittableJsonReaderObject rawReport;
try
{
// even if there is a timeout event, we will keep waiting on the same connection until the TCP timeout occurs.
rawReport = context.ReadForMemory(stream, _readStatusUpdateDebugString);
timeoutEvent.Defer(_parent._leaderClusterTag);
}
catch (Exception e)
{
if (_token.IsCancellationRequested)
{
return;
}
if (_log.IsInfoEnabled)
{
_log.Info("Exception occurred while reading the report from the connection", e);
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
break;
}
var report = BuildReport(rawReport);
timeoutEvent.Defer(_parent._leaderClusterTag);
ReceivedReport = _lastSuccessfulReceivedReport = report;
}
}
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
}
}
}
internal static async Task <Stream> WrapStreamWithSslAsync(TcpClient tcpClient, TcpConnectionInfo info, X509Certificate2 storeCertificate)
{
Stream stream = tcpClient.GetStream();
if (info.Certificate == null)
{
return(stream);
}
var expectedCert = new X509Certificate2(Convert.FromBase64String(info.Certificate));
var sslStream = new SslStream(stream, false, (sender, actualCert, chain, errors) => expectedCert.Equals(actualCert));
await sslStream.AuthenticateAsClientAsync(new Uri(info.Url).Host, new X509CertificateCollection(new X509Certificate[] { storeCertificate }), SslProtocols.Tls12, false).ConfigureAwait(false);
stream = sslStream;
return(stream);
}
private async Task <TcpConnectionHeaderMessage.SupportedFeatures> NegotiateProtocolVersionAsyncForClusterSupervisor(string url, TcpConnectionInfo info, Stream stream, JsonOperationContext context, List <string> _)
{
var parameters = new AsyncTcpNegotiateParameters
{
Database = null,
Operation = TcpConnectionHeaderMessage.OperationTypes.Heartbeats,
Version = TcpConnectionHeaderMessage.HeartbeatsTcpVersion,
ReadResponseAndGetVersionCallbackAsync = SupervisorReadResponseAndGetVersionAsync,
DestinationUrl = url,
DestinationNodeTag = ClusterTag,
DestinationServerId = info.ServerId
};
return(await TcpNegotiation.NegotiateProtocolVersionAsync(context, stream, parameters));
}
private async Task <ClusterMaintenanceConnection> ConnectToClientNodeAsync(TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout)
{
TcpConnectionHeaderMessage.SupportedFeatures supportedFeatures;
Stream connection;
TcpClient tcpClient;
using (_contextPool.AllocateOperationContext(out JsonOperationContext ctx))
{
var result = await TcpUtils.ConnectSecuredTcpSocket(
tcpConnectionInfo,
_parent._server.Server.Certificate.Certificate,
_parent._server.Server.CipherSuitesPolicy,
TcpConnectionHeaderMessage.OperationTypes.Heartbeats,
NegotiateProtocolVersionAsyncForClusterSupervisor,
ctx,
timeout, null, _token);
tcpClient = result.TcpClient;
connection = result.Stream;
supportedFeatures = result.SupportedFeatures;
await using (var writer = new AsyncBlittableJsonTextWriter(ctx, connection))
{
await WriteClusterMaintenanceConnectionHeaderAsync(writer);
}
}
return(new ClusterMaintenanceConnection
{
TcpClient = tcpClient,
Stream = connection,
SupportedFeatures = supportedFeatures
});
}
private void ListenToMaintenanceWorker()
{
var firstIteration = true;
var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan;
var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
var tcpTimeout = _parent.Config.TcpConnectionTimeout.AsTimeSpan;
if (tcpTimeout < receiveFromWorkerTimeout)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Warning: TCP timeout is lower than the receive from worker timeout ({tcpTimeout} < {receiveFromWorkerTimeout}), " +
"this could affect the cluster observer's decisions.");
}
}
while (_token.IsCancellationRequested == false)
{
try
{
if (firstIteration == false)
{
// avoid tight loop if there was timeout / error
_token.WaitHandle.WaitOne(onErrorDelayTime);
if (_token.IsCancellationRequested)
{
return;
}
}
firstIteration = false;
TcpConnectionInfo tcpConnection = null;
using (var timeout = new CancellationTokenSource(tcpTimeout))
using (var combined = CancellationTokenSource.CreateLinkedTokenSource(_token, timeout.Token))
{
tcpConnection = ReplicationUtils.GetTcpInfo(Url, null, "Supervisor", _parent._server.Server.Certificate.Certificate, combined.Token);
if (tcpConnection == null)
{
continue;
}
}
var connection = ConnectToClientNode(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
var tcpClient = connection.TcpClient;
var stream = connection.Stream;
using (tcpClient)
using (_cts.Token.Register(tcpClient.Dispose))
using (_contextPool.AllocateOperationContext(out JsonOperationContext contextForParsing))
using (_contextPool.AllocateOperationContext(out JsonOperationContext contextForBuffer))
using (contextForBuffer.GetMemoryBuffer(out var readBuffer))
using (var timeoutEvent = new TimeoutEvent(receiveFromWorkerTimeout, $"Timeout event for: {_name}", singleShot: false))
{
timeoutEvent.Start(OnTimeout);
var unchangedReports = new List <DatabaseStatusReport>();
while (_token.IsCancellationRequested == false)
{
contextForParsing.Reset();
contextForParsing.Renew();
BlittableJsonReaderObject rawReport;
try
{
// even if there is a timeout event, we will keep waiting on the same connection until the TCP timeout occurs.
rawReport = contextForParsing.Sync.ParseToMemory(stream, _readStatusUpdateDebugString, BlittableJsonDocumentBuilder.UsageMode.None, readBuffer);
timeoutEvent.Defer(_parent._leaderClusterTag);
}
catch (Exception e)
{
if (_token.IsCancellationRequested)
{
return;
}
if (_log.IsInfoEnabled)
{
_log.Info("Exception occurred while reading the report from the connection", e);
}
ReceivedReport = new ClusterNodeStatusReport(new ServerReport(), new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
break;
}
_parent.ForTestingPurposes?.BeforeReportBuildAction(this);
var nodeReport = BuildReport(rawReport, connection.SupportedFeatures);
timeoutEvent.Defer(_parent._leaderClusterTag);
UpdateNodeReportIfNeeded(nodeReport, unchangedReports);
unchangedReports.Clear();
ReceivedReport = _lastSuccessfulReceivedReport = nodeReport;
_parent.ForTestingPurposes?.AfterSettingReportAction(this);
}
}
}
catch (Exception e)
{
if (_token.IsCancellationRequested)
{
return;
}
if (_log.IsInfoEnabled)
{
_log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
}
ReceivedReport = new ClusterNodeStatusReport(new ServerReport(), new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
}
}
}
private async Task <(TcpClient TcpClient, Stream Connection)> ConnectAndGetNetworkStreamAsync(TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout, Logger log)
{
var tcpClient = await TcpUtils.ConnectSocketAsync(tcpConnectionInfo, timeout, log);
var connection = await TcpUtils.WrapStreamWithSslAsync(tcpClient, tcpConnectionInfo, Server.ClusterCertificateHolder.Certificate);
return(tcpClient, connection);
}
private async Task <ClusterMaintenanceConnection> ConnectToClientNodeAsync(TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout)
{
TcpConnectionHeaderMessage.SupportedFeatures supportedFeatures;
var tcpClient = await TcpUtils.ConnectSocketAsync(tcpConnectionInfo, timeout, _log);
var connection = await TcpUtils.WrapStreamWithSslAsync(tcpClient, tcpConnectionInfo, _parent._server.Server.Certificate.Certificate, timeout);
using (_contextPool.AllocateOperationContext(out JsonOperationContext ctx))
using (var writer = new BlittableJsonTextWriter(ctx, connection))
{
var paramaters = new TcpNegotiateParamaters
{
Database = null,
Operation = TcpConnectionHeaderMessage.OperationTypes.Heartbeats,
Version = TcpConnectionHeaderMessage.HeartbeatsTcpVersion,
ReadResponseAndGetVersionAsync = SupervisorReadResponseAndGetVersionAsync,
Url = tcpConnectionInfo.Url
};
supportedFeatures = await TcpNegotiation.NegotiateProtocolVersionAsync(ctx, connection, paramaters).ConfigureAwait(false);
WriteClusterMaintenanceConnectionHeader(writer);
}
return(new ClusterMaintenanceConnection
{
TcpClient = tcpClient,
Stream = connection,
SupportedFeatures = supportedFeatures
});
}
private async Task <ClusterMaintenanceConnection> ConnectToClientNodeAsync(TcpConnectionInfo tcpConnectionInfo, TimeSpan timeout)
{
TcpConnectionHeaderMessage.SupportedFeatures supportedFeatures;
TcpClient tcpClient;
string url;
(tcpClient, url) = await TcpUtils.ConnectSocketAsync(tcpConnectionInfo, timeout, _log);
var connection = await TcpUtils.WrapStreamWithSslAsync(tcpClient, tcpConnectionInfo, _parent._server.Server.Certificate.Certificate, _parent._server.Server.CipherSuitesPolicy, timeout);
using (_contextPool.AllocateOperationContext(out JsonOperationContext ctx))
using (var writer = new BlittableJsonTextWriter(ctx, connection))
{
var parameters = new TcpNegotiateParameters
{
Database = null,
Operation = TcpConnectionHeaderMessage.OperationTypes.Heartbeats,
Version = TcpConnectionHeaderMessage.HeartbeatsTcpVersion,
ReadResponseAndGetVersionCallback = SupervisorReadResponseAndGetVersion,
DestinationUrl = url,
DestinationNodeTag = ClusterTag
};
supportedFeatures = TcpNegotiation.NegotiateProtocolVersion(ctx, connection, parameters);
WriteClusterMaintenanceConnectionHeader(writer);
}
return(new ClusterMaintenanceConnection
{
TcpClient = tcpClient,
Stream = connection,
SupportedFeatures = supportedFeatures
});
}
