public ChangeVectorEntry[] GetIndexesAndTransformersChangeVector(Transaction tx)
{
var globalChangeVectorTree = tx.ReadTree(SchemaNameConstants.GlobalChangeVectorTree);
Debug.Assert(globalChangeVectorTree != null);
return(ReplicationUtils.ReadChangeVectorFrom(globalChangeVectorTree));
}
private IReadOnlyList <ChangeVectorEntry[]> DeleteConflictsFor(Transaction tx, Slice name)
{
var table = tx.OpenTable(ConflictsTableSchema, SchemaNameConstants.ConflictMetadataTable);
Debug.Assert(table != null);
var list = new List <ChangeVectorEntry[]>();
while (true)
{
bool deleted = false;
// deleting a value might cause other ids to change, so we can't just pass the list
// of ids to be deleted, because they wouldn't remain stable during the deletions
foreach (var seekResult in table.SeekForwardFrom(ConflictsTableSchema.Indexes[NameAndEtagIndexName], name, true))
{
foreach (var tvr in seekResult.Results)
{
deleted = true;
list.Add(ReplicationUtils.GetChangeVectorEntriesFromTableValueReader(tvr, (int)MetadataFields.ChangeVector));
//Ids might change due to delete
table.Delete(tvr.Id);
break;
}
}
if (deleted == false)
{
return(list);
}
}
}
public async Task TestConnection()
{
var url = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
DynamicJsonValue result;
try
{
var timeout = TimeoutManager.WaitFor(ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
var connectionInfo = ReplicationUtils.GetTcpInfoAsync(url, null, "Test-Connection", Server.ClusterCertificateHolder.Certificate);
if (await Task.WhenAny(timeout, connectionInfo) == timeout)
{
throw new TimeoutException($"Waited for {ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan} to receive tcp info from {url} and got no response");
}
result = await ConnectToClientNodeAsync(connectionInfo.Result, ServerStore.Engine.TcpConnectionTimeout, LoggingSource.Instance.GetLogger("testing-connection", "testing-connection"));
}
catch (Exception e)
{
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = false,
[nameof(NodeConnectionTestResult.Error)] = $"An exception was thrown while trying to connect to {url} : {e}"
};
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, result);
}
}
public void SetGlobalChangeVector(Transaction tx, ByteStringContext context, Dictionary <Guid, long> changeVector)
{
var tree = tx.CreateTree(SchemaNameConstants.GlobalChangeVectorTree);
Debug.Assert(tree != null);
ReplicationUtils.WriteChangeVectorTo(context, changeVector, tree);
}
private ChangeVectorEntry[] SetIndexTransformerChangeVectorForLocalChange(Transaction tx, TransactionOperationContext context, Slice loweredName, TableValueReader oldValue, long newEtag, ChangeVectorEntry[] vector)
{
if (oldValue != null)
{
var changeVector = ReplicationUtils.GetChangeVectorEntriesFromTableValueReader(oldValue, (int)MetadataFields.ChangeVector);
return(ReplicationUtils.UpdateChangeVectorWithNewEtag(_environment.DbId, newEtag, changeVector));
}
return(GetMergedConflictChangeVectorsAndDeleteConflicts(tx, context, loweredName, newEtag));
}
private ChangeVectorEntry[] GetMergedConflictChangeVectorsAndDeleteConflicts(Transaction tx, TransactionOperationContext context, Slice name, long newEtag, ChangeVectorEntry[] existing = null)
{
var conflictChangeVectors = DeleteConflictsFor(tx, name);
//no conflicts, no need to merge
if (conflictChangeVectors.Count == 0)
{
if (existing != null)
{
return(ReplicationUtils.UpdateChangeVectorWithNewEtag(_environment.DbId, newEtag, existing));
}
return(new[]
{
new ChangeVectorEntry
{
Etag = newEtag,
DbId = _environment.DbId
}
});
}
// need to merge the conflict change vectors
var maxEtags = new Dictionary <Guid, long>
{
[_environment.DbId] = newEtag
};
foreach (var conflictChangeVector in conflictChangeVectors)
{
foreach (var entry in conflictChangeVector)
{
long etag;
if (maxEtags.TryGetValue(entry.DbId, out etag) == false ||
etag < entry.Etag)
{
maxEtags[entry.DbId] = entry.Etag;
}
}
}
var changeVector = new ChangeVectorEntry[maxEtags.Count];
var index = 0;
foreach (var maxEtag in maxEtags)
{
changeVector[index].DbId = maxEtag.Key;
changeVector[index].Etag = maxEtag.Value;
index++;
}
return(changeVector);
}
public Task Establish(X509Certificate2 certificate)
{
if (certificate != null)
{
var tcpConnection = ReplicationUtils.GetTcpInfo(_nodeUrl, null, $"{nameof(ProxyWebSocketConnection)} to {_nodeUrl}", certificate, _cts.Token);
var expectedCert = new X509Certificate2(Convert.FromBase64String(tcpConnection.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);
_remoteWebSocket.Options.ClientCertificates.Add(certificate);
_remoteWebSocket.Options.RemoteCertificateValidationCallback += (sender, actualCert, chain, errors) => expectedCert.Equals(actualCert);
}
return(_remoteWebSocket.ConnectAsync(_remoteWebSocketUri, _cts.Token));
}
private TcpConnectionInfo GetConnectionInfo(ReplicationNode node, bool external)
{
var shutdownInfo = new ConnectionShutdownInfo
{
Node = node,
External = external
};
_outgoingFailureInfo.TryAdd(node, shutdownInfo);
try
{
if (node is ExternalReplication exNode)
{
using (var requestExecutor = RequestExecutor.Create(exNode.ConnectionString.TopologyDiscoveryUrls, exNode.ConnectionString.Database,
_server.Server.Certificate.Certificate,
DocumentConventions.Default))
using (_server.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
var database = exNode.ConnectionString.Database;
var cmd = new GetTcpInfoCommand("extrenal-replication", database);
requestExecutor.Execute(cmd, ctx);
node.Database = database;
node.Url = requestExecutor.Url;
return(cmd.Result);
}
}
if (node is InternalReplication internalNode)
{
using (var cts = new CancellationTokenSource(_server.Engine.TcpConnectionTimeout))
{
return(ReplicationUtils.GetTcpInfo(internalNode.Url, internalNode.Database, "Replication", _server.Server.Certificate.Certificate, cts.Token));
}
}
throw new InvalidOperationException(
$"Unexpected replication node type, Expected to be '{typeof(ExternalReplication)}' or '{typeof(InternalReplication)}', but got '{node.GetType()}'");
}
catch (Exception e)
{
// will try to fetch it again later
if (_log.IsInfoEnabled)
{
_log.Info($"Failed to fetch tcp connection information for the destination '{node.FromString()}' , the connection will be retried later.", e);
}
_reconnectQueue.TryAdd(shutdownInfo);
}
return(null);
}
private async Task <PingResult> PingOnce(string url)
{
var sp = Stopwatch.StartNew();
var result = new PingResult
{
Url = url
};
try
{
using (var cts = new CancellationTokenSource(ServerStore.Engine.TcpConnectionTimeout))
{
var info = await ReplicationUtils.GetTcpInfoAsync(url, null, "PingTest", ServerStore.Engine.ClusterCertificate, cts.Token);
result.TcpInfoTime = sp.ElapsedMilliseconds;
using (var tcpClient = await TcpUtils.ConnectAsync(info.Url, ServerStore.Engine.TcpConnectionTimeout).ConfigureAwait(false))
using (var stream = await TcpUtils
.WrapStreamWithSslAsync(tcpClient, info, ServerStore.Engine.ClusterCertificate, ServerStore.Engine.TcpConnectionTimeout).ConfigureAwait(false))
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var msg = new DynamicJsonValue
{
[nameof(TcpConnectionHeaderMessage.DatabaseName)] = null,
[nameof(TcpConnectionHeaderMessage.Operation)] = TcpConnectionHeaderMessage.OperationTypes.Ping,
[nameof(TcpConnectionHeaderMessage.OperationVersion)] = -1
};
using (var writer = new BlittableJsonTextWriter(context, stream))
using (var msgJson = context.ReadObject(msg, "message"))
{
result.SendTime = sp.ElapsedMilliseconds;
context.Write(writer, msgJson);
}
using (var response = context.ReadForMemory(stream, "cluster-ConnectToPeer-header-response"))
{
JsonDeserializationServer.TcpConnectionHeaderResponse(response);
result.RecieveTime = sp.ElapsedMilliseconds;
}
}
}
}
catch (Exception e)
{
result.Error = e.ToString();
}
return(result);
}
private IndexConflictEntry TableValueToConflict(TableValueReader tvr,
JsonOperationContext context)
{
var data = new IndexConflictEntry();
int size;
data.Name = new LazyStringValue(null, tvr.Read((int)ConflictFields.Name, out size), size, context).ToString();
data.ChangeVector = ReplicationUtils.GetChangeVectorEntriesFromTableValueReader(tvr, (int)ConflictFields.ChangeVector);
data.Type = (IndexEntryType)(*tvr.Read((int)ConflictFields.Type, out size));
data.Etag = Bits.SwapBytes(*(long *)tvr.Read((int)ConflictFields.Etag, out size));
var ptr = tvr.Read((int)ConflictFields.Definition, out size);
data.Definition = new BlittableJsonReaderObject(ptr, size, context);
return(data);
}
private IndexEntryMetadata TableValueToMetadata(TableValueReader tvr,
JsonOperationContext context,
bool returnNullIfTombstone)
{
var metadata = new IndexEntryMetadata();
int size;
metadata.Id = Bits.SwapBytes(*(int *)tvr.Read((int)MetadataFields.Id, out size));
if (returnNullIfTombstone && metadata.Id == -1)
{
return(null);
}
metadata.Name = new LazyStringValue(null, tvr.Read((int)MetadataFields.Name, out size), size, context).ToString();
metadata.ChangeVector = ReplicationUtils.GetChangeVectorEntriesFromTableValueReader(tvr, (int)MetadataFields.ChangeVector);
metadata.Type = (IndexEntryType)(*tvr.Read((int)MetadataFields.Type, out size));
metadata.Etag = Bits.SwapBytes(*(long *)tvr.Read((int)MetadataFields.Etag, out size));
metadata.IsConflicted = *(bool *)tvr.Read((int)MetadataFields.IsConflicted, out size);
return(metadata);
}
private void MergeEntryVectorWithGlobal(Transaction tx,
ByteStringContext context,
ChangeVectorEntry[] changeVectorForWrite)
{
var globalChangeVectorTree = tx.ReadTree(SchemaNameConstants.GlobalChangeVectorTree);
Debug.Assert(globalChangeVectorTree != null);
var globalChangeVector = ReplicationUtils.ReadChangeVectorFrom(globalChangeVectorTree);
// merge metadata change vector into global change vector
// --> if we have any entry in global vector smaller than in metadata vector,
// update the entry in global vector to a larger one
foreach (var item in ReplicationUtils.MergeVectors(globalChangeVector, changeVectorForWrite))
{
var dbId = item.DbId;
var etagBigEndian = Bits.SwapBytes(item.Etag);
Slice key;
Slice value;
using (Slice.External(context, (byte *)&dbId, sizeof(Guid), out key))
using (Slice.External(context, (byte *)&etagBigEndian, sizeof(long), out value))
globalChangeVectorTree.Add(key, value);
}
}
public async Task TestConnection()
{
var url = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
url = UrlHelper.TryGetLeftPart(url);
DynamicJsonValue result = null;
Task <TcpConnectionInfo> connectionInfo = null;
try
{
try
{
var timeout = TimeoutManager.WaitFor(ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
using (var cts = new CancellationTokenSource(Server.Configuration.Cluster.OperationTimeout.AsTimeSpan))
{
connectionInfo = ReplicationUtils.GetTcpInfoAsync(url, null, "Test-Connection", Server.Certificate.Certificate,
cts.Token);
}
Task timeoutTask = await Task.WhenAny(timeout, connectionInfo);
if (timeoutTask == timeout)
{
throw new TimeoutException($"Waited for {ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan} to receive tcp info from {url} and got no response");
}
await connectionInfo;
}
catch (Exception e)
{
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = false,
[nameof(NodeConnectionTestResult.HTTPSuccess)] = false,
[nameof(NodeConnectionTestResult.Error)] = $"An exception was thrown while trying to connect to {url} : {e}"
};
return;
}
try
{
result = await ConnectToClientNodeAsync(connectionInfo.Result, ServerStore.Engine.TcpConnectionTimeout, LoggingSource.Instance.GetLogger("testing-connection", "testing-connection"));
}
catch (Exception e)
{
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = false,
[nameof(NodeConnectionTestResult.HTTPSuccess)] = true,
[nameof(NodeConnectionTestResult.TcpServerUrl)] = connectionInfo.Result.Url,
[nameof(NodeConnectionTestResult.Error)] = $"Was able to connect to url {url}, but exception was thrown while trying to connect to TCP port {connectionInfo.Result.Url}: {e}"
};
return;
}
}
finally
{
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, result);
}
}
}
private void ListenToMaintenanceWorker()
{
var needToWait = false;
var firstIteration = true;
var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan;
var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
var tcpTimeout = _parent.Config.TcpConnectionTimeout.AsTimeSpan;
if (tcpTimeout < receiveFromWorkerTimeout)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Warning: TCP timeout is lower than the receive from worker timeout ({tcpTimeout} < {receiveFromWorkerTimeout}), " +
"this could affect the cluster observer's decisions.");
}
}
TcpConnectionInfo tcpConnection = null;
while (_token.IsCancellationRequested == false)
{
try
{
if (needToWait)
{
needToWait = false; // avoid tight loop if there was timeout / error
if (firstIteration == false)
{
_token.WaitHandle.WaitOne(onErrorDelayTime);
}
firstIteration = false;
using (var timeout = new CancellationTokenSource(tcpTimeout))
using (var combined = CancellationTokenSource.CreateLinkedTokenSource(_token, timeout.Token))
{
tcpConnection = ReplicationUtils.GetTcpInfo(Url, null, "Supervisor", _parent._server.Server.Certificate.Certificate, combined.Token);
}
}
if (tcpConnection == null)
{
needToWait = true;
continue;
}
var connection = ConnectToClientNode(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
var tcpClient = connection.TcpClient;
var stream = connection.Stream;
using (tcpClient)
using (_cts.Token.Register(tcpClient.Dispose))
using (_contextPool.AllocateOperationContext(out JsonOperationContext context))
using (var timeoutEvent = new TimeoutEvent(receiveFromWorkerTimeout, $"Timeout event for: {_name}", singleShot: false))
{
timeoutEvent.Start(OnTimeout);
while (_token.IsCancellationRequested == false)
{
BlittableJsonReaderObject rawReport;
try
{
// even if there is a timeout event, we will keep waiting on the same connection until the TCP timeout occurs.
rawReport = context.ReadForMemory(stream, _readStatusUpdateDebugString);
timeoutEvent.Defer(_parent._leaderClusterTag);
}
catch (Exception e)
{
if (_token.IsCancellationRequested)
{
return;
}
if (_log.IsInfoEnabled)
{
_log.Info("Exception occurred while reading the report from the connection", e);
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
break;
}
var report = BuildReport(rawReport);
timeoutEvent.Defer(_parent._leaderClusterTag);
ReceivedReport = _lastSuccessfulReceivedReport = report;
}
}
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
}
}
}
private async Task RequestHandler(HttpContext context)
{
try
{
context.Response.StatusCode = 200;
var sp = Stopwatch.StartNew();
var tenant = await _router.HandlePath(context, context.Request.Method, context.Request.Path.Value);
sp.Stop();
if (_logger.IsInfoEnabled)
{
_logger.Info($"{context.Request.Method} {context.Request.Path.Value}?{context.Request.QueryString.Value} - {context.Response.StatusCode} - {sp.ElapsedMilliseconds:#,#;;0} ms");
}
if (TrafficWatchManager.HasRegisteredClients)
{
var requestId = Interlocked.Increment(ref _requestId);
var twn = new TrafficWatchNotification
{
TimeStamp = DateTime.UtcNow,
RequestId = requestId, // counted only for traffic watch
HttpMethod = context.Request.Method ?? "N/A", // N/A ?
ElapsedMilliseconds = sp.ElapsedMilliseconds,
ResponseStatusCode = context.Response.StatusCode,
RequestUri = context.Request.GetEncodedUrl(),
AbsoluteUri = $@"{context.Request.Scheme}://{context.Request.Host}",
TenantName = tenant ?? "N/A",
CustomInfo = "", // TODO: Implement
InnerRequestsCount = 0, // TODO: Implement
QueryTimings = null, // TODO: Implement
};
TrafficWatchManager.DispatchMessage(twn);
}
}
catch (Exception e)
{
if (context.RequestAborted.IsCancellationRequested)
{
return;
}
//TODO: special handling for argument exception (400 bad request)
//TODO: database not found (503)
//TODO: operaton cancelled (timeout)
//TODO: Invalid data exception 422
//TODO: Proper json output, not like this
var response = context.Response;
var documentConflictException = e as DocumentConflictException;
if (response.HasStarted == false)
{
if (documentConflictException != null)
{
response.StatusCode = 409;
}
else if (response.StatusCode < 400)
{
response.StatusCode = 500;
}
}
JsonOperationContext ctx;
using (_server.ServerStore.ContextPool.AllocateOperationContext(out ctx))
{
var djv = new DynamicJsonValue
{
["Url"] = $"{context.Request.Path}?{context.Request.QueryString}",
["Type"] = e.GetType().FullName,
["Message"] = e.Message
};
string errorString;
try
{
errorString = e.ToAsyncString();
}
catch (Exception)
{
errorString = e.ToString();
}
djv["Error"] = errorString;
var indexCompilationException = e as IndexCompilationException;
if (indexCompilationException != null)
{
djv[nameof(IndexCompilationException.IndexDefinitionProperty)] =
indexCompilationException.IndexDefinitionProperty;
djv[nameof(IndexCompilationException.ProblematicText)] =
indexCompilationException.ProblematicText;
}
if (documentConflictException != null)
{
djv["ConflictInfo"] = ReplicationUtils.GetJsonForConflicts(
documentConflictException.DocId,
documentConflictException.Conflicts);
}
using (var writer = new BlittableJsonTextWriter(ctx, response.Body))
{
var json = ctx.ReadObject(djv, "exception");
writer.WriteObject(json);
}
}
}
}
private void ListenToMaintenanceWorker()
{
var firstIteration = true;
var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan;
var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
var tcpTimeout = _parent.Config.TcpConnectionTimeout.AsTimeSpan;
if (tcpTimeout < receiveFromWorkerTimeout)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Warning: TCP timeout is lower than the receive from worker timeout ({tcpTimeout} < {receiveFromWorkerTimeout}), " +
"this could affect the cluster observer's decisions.");
}
}
while (_token.IsCancellationRequested == false)
{
try
{
if (firstIteration == false)
{
// avoid tight loop if there was timeout / error
_token.WaitHandle.WaitOne(onErrorDelayTime);
if (_token.IsCancellationRequested)
{
return;
}
}
firstIteration = false;
TcpConnectionInfo tcpConnection = null;
using (var timeout = new CancellationTokenSource(tcpTimeout))
using (var combined = CancellationTokenSource.CreateLinkedTokenSource(_token, timeout.Token))
{
tcpConnection = ReplicationUtils.GetTcpInfo(Url, null, "Supervisor", _parent._server.Server.Certificate.Certificate, combined.Token);
if (tcpConnection == null)
{
continue;
}
}
var connection = ConnectToClientNode(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
var tcpClient = connection.TcpClient;
var stream = connection.Stream;
using (tcpClient)
using (_cts.Token.Register(tcpClient.Dispose))
using (_contextPool.AllocateOperationContext(out JsonOperationContext contextForParsing))
using (_contextPool.AllocateOperationContext(out JsonOperationContext contextForBuffer))
using (contextForBuffer.GetMemoryBuffer(out var readBuffer))
using (var timeoutEvent = new TimeoutEvent(receiveFromWorkerTimeout, $"Timeout event for: {_name}", singleShot: false))
{
timeoutEvent.Start(OnTimeout);
var unchangedReports = new List <DatabaseStatusReport>();
while (_token.IsCancellationRequested == false)
{
contextForParsing.Reset();
contextForParsing.Renew();
BlittableJsonReaderObject rawReport;
try
{
// even if there is a timeout event, we will keep waiting on the same connection until the TCP timeout occurs.
rawReport = contextForParsing.Sync.ParseToMemory(stream, _readStatusUpdateDebugString, BlittableJsonDocumentBuilder.UsageMode.None, readBuffer);
timeoutEvent.Defer(_parent._leaderClusterTag);
}
catch (Exception e)
{
if (_token.IsCancellationRequested)
{
return;
}
if (_log.IsInfoEnabled)
{
_log.Info("Exception occurred while reading the report from the connection", e);
}
ReceivedReport = new ClusterNodeStatusReport(new ServerReport(), new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
break;
}
_parent.ForTestingPurposes?.BeforeReportBuildAction(this);
var nodeReport = BuildReport(rawReport, connection.SupportedFeatures);
timeoutEvent.Defer(_parent._leaderClusterTag);
UpdateNodeReportIfNeeded(nodeReport, unchangedReports);
unchangedReports.Clear();
ReceivedReport = _lastSuccessfulReceivedReport = nodeReport;
_parent.ForTestingPurposes?.AfterSettingReportAction(this);
}
}
}
catch (Exception e)
{
if (_token.IsCancellationRequested)
{
return;
}
if (_log.IsInfoEnabled)
{
_log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
}
ReceivedReport = new ClusterNodeStatusReport(new ServerReport(), new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
}
}
}
private async Task ListenToMaintenanceWorker()
{
bool needToWait = false;
var onErrorDelayTime = _parent.Config.OnErrorDelayTime.AsTimeSpan;
var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
TcpConnectionInfo tcpConnection = null;
try
{
tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor",
_parent._server.RavenServer.ClusterCertificateHolder?.Certificate);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"ClusterMaintenanceSupervisor() => Failed to add to cluster node key = {ClusterTag}", e);
}
}
while (_token.IsCancellationRequested == false)
{
var internalTaskCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(_token);
try
{
if (needToWait)
{
needToWait = false; // avoid tight loop if there was timeout / error
await TimeoutManager.WaitFor(onErrorDelayTime, _token);
tcpConnection = await ReplicationUtils.GetTcpInfoAsync(Url, null, "Supervisor",
_parent._server.RavenServer.ClusterCertificateHolder.Certificate);
}
if (tcpConnection == null)
{
needToWait = true;
continue;
}
var(tcpClient, connection) = await ConnectToClientNodeAsync(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
using (tcpClient)
using (_cts.Token.Register(tcpClient.Dispose))
using (connection)
{
while (_token.IsCancellationRequested == false)
{
using (_contextPool.AllocateOperationContext(out JsonOperationContext context))
{
var readResponseTask = context.ReadForMemoryAsync(connection, _readStatusUpdateDebugString, internalTaskCancellationToken.Token);
var timeout = TimeoutManager.WaitFor(receiveFromWorkerTimeout, _token);
if (await Task.WhenAny(readResponseTask.AsTask(), timeout) == timeout)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Timeout occurred while collecting info from {ClusterTag}");
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Timeout,
null,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
internalTaskCancellationToken.Cancel();
break;
}
using (var statusUpdateJson = await readResponseTask)
{
var report = new Dictionary <string, DatabaseStatusReport>();
foreach (var property in statusUpdateJson.GetPropertyNames())
{
var value = (BlittableJsonReaderObject)statusUpdateJson[property];
report.Add(property, JsonDeserializationServer.DatabaseStatusReport(value));
}
ReceivedReport = new ClusterNodeStatusReport(
report,
ClusterNodeStatusReport.ReportStatus.Ok,
null,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
_lastSuccessfulReceivedReport = ReceivedReport;
}
}
}
}
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
}
ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
ClusterNodeStatusReport.ReportStatus.Error,
e,
DateTime.UtcNow,
_lastSuccessfulReceivedReport);
needToWait = true;
}
finally
{
internalTaskCancellationToken.Dispose();
}
}
}
public HttpResponseMessage ReplicationInfoGet()
{
var replicationStatistics = ReplicationUtils.GetReplicationInformation(Database);
return(GetMessageWithObject(replicationStatistics));
}
private async Task <PingResult> PingOnce(string url)
{
var sp = Stopwatch.StartNew();
var result = new PingResult
{
Url = url
};
using (var cts = new CancellationTokenSource(ServerStore.Engine.TcpConnectionTimeout))
{
var info = await ReplicationUtils.GetTcpInfoAsync(url, null, "PingTest", ServerStore.Engine.ClusterCertificate, cts.Token);
result.TcpInfoTime = sp.ElapsedMilliseconds;
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var log = new List <string>();
using (await TcpUtils.ConnectSecuredTcpSocket(info, ServerStore.Engine.ClusterCertificate, Server.CipherSuitesPolicy,
TcpConnectionHeaderMessage.OperationTypes.Ping, NegotiationCallback, context, ServerStore.Engine.TcpConnectionTimeout, log, cts.Token))
{
}
result.Log = log;
async Task <TcpConnectionHeaderMessage.SupportedFeatures> NegotiationCallback(string curUrl, TcpConnectionInfo tcpInfo, Stream stream,
JsonOperationContext ctx, List <string> logs = null)
{
try
{
var msg = new DynamicJsonValue
{
[nameof(TcpConnectionHeaderMessage.DatabaseName)] = null,
[nameof(TcpConnectionHeaderMessage.Operation)] = TcpConnectionHeaderMessage.OperationTypes.Ping,
[nameof(TcpConnectionHeaderMessage.OperationVersion)] = -1,
[nameof(TcpConnectionHeaderMessage.ServerId)] = tcpInfo.ServerId
};
await using (var writer = new AsyncBlittableJsonTextWriter(ctx, stream))
using (var msgJson = ctx.ReadObject(msg, "message"))
{
result.SendTime = sp.ElapsedMilliseconds;
logs?.Add($"message sent to url {curUrl} at {result.SendTime} ms.");
ctx.Write(writer, msgJson);
}
using (var rawResponse = await ctx.ReadForMemoryAsync(stream, "cluster-ConnectToPeer-header-response"))
{
TcpConnectionHeaderResponse response = JsonDeserializationServer.TcpConnectionHeaderResponse(rawResponse);
result.ReceiveTime = sp.ElapsedMilliseconds;
logs?.Add($"response received from url {curUrl} at {result.ReceiveTime} ms.");
switch (response.Status)
{
case TcpConnectionStatus.Ok:
result.Error = null;
logs?.Add($"Successfully negotiated with {url}.");
break;
case TcpConnectionStatus.AuthorizationFailed:
result.Error = $"Connection to {url} failed because of authorization failure: {response.Message}";
logs?.Add(result.Error);
throw new AuthorizationException(result.Error);
case TcpConnectionStatus.TcpVersionMismatch:
result.Error = $"Connection to {url} failed because of mismatching tcp version: {response.Message}";
logs?.Add(result.Error);
throw new AuthorizationException(result.Error);
case TcpConnectionStatus.InvalidNetworkTopology:
result.Error = $"Connection to {url} failed because of {nameof(TcpConnectionStatus.InvalidNetworkTopology)} error: {response.Message}";
logs?.Add(result.Error);
throw new InvalidNetworkTopologyException(result.Error);
}
}
}
catch (Exception e)
{
result.Error = e.ToString();
logs?.Add($"Error occurred while attempting to negotiate with the server. {e.Message}");
throw;
}
return(null);
}
}
}
return(result);
}
public static void CreateInfoPackageForDatabase(ZipArchive package, DocumentDatabase database, RequestManager requestManager, string zipEntryPrefix = null)
{
zipEntryPrefix = zipEntryPrefix ?? string.Empty;
var databaseName = database.Name;
if (string.IsNullOrWhiteSpace(databaseName))
{
databaseName = Constants.SystemDatabase;
}
var jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer();
jsonSerializer.Formatting = Formatting.Indented;
if (database.StartupTasks.OfType <ReplicationTask>().Any())
{
var replication = package.CreateEntry(zipEntryPrefix + "replication.json", compressionLevel);
using (var statsStream = replication.Open())
using (var streamWriter = new StreamWriter(statsStream))
{
jsonSerializer.Serialize(streamWriter, ReplicationUtils.GetReplicationInformation(database));
streamWriter.Flush();
}
}
var sqlReplicationTask = database.StartupTasks.OfType <SqlReplicationTask>().FirstOrDefault();
if (sqlReplicationTask != null)
{
var replication = package.CreateEntry(zipEntryPrefix + "sql_replication.json", compressionLevel);
using (var statsStream = replication.Open())
using (var streamWriter = new StreamWriter(statsStream))
{
jsonSerializer.Serialize(streamWriter, sqlReplicationTask.Statistics);
streamWriter.Flush();
}
}
var stats = package.CreateEntry(zipEntryPrefix + "stats.json", compressionLevel);
using (var statsStream = stats.Open())
using (var streamWriter = new StreamWriter(statsStream))
{
jsonSerializer.Serialize(streamWriter, database.Statistics);
streamWriter.Flush();
}
var metrics = package.CreateEntry(zipEntryPrefix + "metrics.json", compressionLevel);
using (var metricsStream = metrics.Open())
using (var streamWriter = new StreamWriter(metricsStream))
{
jsonSerializer.Serialize(streamWriter, database.CreateMetrics());
streamWriter.Flush();
}
var logs = package.CreateEntry(zipEntryPrefix + "logs.csv", compressionLevel);
using (var logsStream = logs.Open())
using (var streamWriter = new StreamWriter(logsStream))
{
var target = LogManager.GetTarget <DatabaseMemoryTarget>();
if (target == null)
{
streamWriter.WriteLine("DatabaseMemoryTarget was not registered in the log manager, logs are not available");
}
else
{
var boundedMemoryTarget = target[databaseName];
var log = boundedMemoryTarget.GeneralLog;
streamWriter.WriteLine("time,logger,level,message,exception");
foreach (var logEvent in log)
{
streamWriter.WriteLine("{0:O},{1},{2},{3},{4}", logEvent.TimeStamp, logEvent.LoggerName, logEvent.Level, logEvent.FormattedMessage, logEvent.Exception);
}
}
streamWriter.Flush();
}
var config = package.CreateEntry(zipEntryPrefix + "config.json", compressionLevel);
using (var configStream = config.Open())
using (var streamWriter = new StreamWriter(configStream))
using (var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
})
{
GetConfigForDebug(database).WriteTo(jsonWriter, new EtagJsonConverter());
jsonWriter.Flush();
}
var indexes = package.CreateEntry(zipEntryPrefix + "indexes.json", compressionLevel);
using (var indexesStream = indexes.Open())
using (var streamWriter = new StreamWriter(indexesStream))
{
jsonSerializer.Serialize(streamWriter, database.IndexDefinitionStorage.IndexDefinitions.ToDictionary(x => x.Key, x => x.Value));
streamWriter.Flush();
}
var currentlyIndexing = package.CreateEntry(zipEntryPrefix + "currently-indexing.json", compressionLevel);
using (var currentlyIndexingStream = currentlyIndexing.Open())
using (var streamWriter = new StreamWriter(currentlyIndexingStream))
{
jsonSerializer.Serialize(streamWriter, GetCurrentlyIndexingForDebug(database));
streamWriter.Flush();
}
var queries = package.CreateEntry(zipEntryPrefix + "queries.json", compressionLevel);
using (var queriesStream = queries.Open())
using (var streamWriter = new StreamWriter(queriesStream))
{
jsonSerializer.Serialize(streamWriter, database.WorkContext.CurrentlyRunningQueries);
streamWriter.Flush();
}
var version = package.CreateEntry(zipEntryPrefix + "version.json", compressionLevel);
using (var versionStream = version.Open())
using (var streamWriter = new StreamWriter(versionStream))
{
jsonSerializer.Serialize(streamWriter, new
{
DocumentDatabase.ProductVersion,
DocumentDatabase.BuildVersion
});
streamWriter.Flush();
}
var prefetchStatus = package.CreateEntry(zipEntryPrefix + "prefetch-status.json", compressionLevel);
using (var prefetchStatusStream = prefetchStatus.Open())
using (var streamWriter = new StreamWriter(prefetchStatusStream))
{
jsonSerializer.Serialize(streamWriter, GetPrefetchingQueueStatusForDebug(database));
streamWriter.Flush();
}
var requestTracking = package.CreateEntry(zipEntryPrefix + "request-tracking.json", compressionLevel);
using (var requestTrackingStream = requestTracking.Open())
using (var streamWriter = new StreamWriter(requestTrackingStream))
{
jsonSerializer.Serialize(streamWriter, GetRequestTrackingForDebug(requestManager, databaseName));
streamWriter.Flush();
}
var tasks = package.CreateEntry(zipEntryPrefix + "tasks.json", compressionLevel);
using (var tasksStream = tasks.Open())
using (var streamWriter = new StreamWriter(tasksStream))
{
jsonSerializer.Serialize(streamWriter, GetTasksForDebug(database));
streamWriter.Flush();
}
var systemUtilization = package.CreateEntry(zipEntryPrefix + "system-utilization.json", compressionLevel);
using (var systemUtilizationStream = systemUtilization.Open())
using (var streamWriter = new StreamWriter(systemUtilizationStream))
{
long totalPhysicalMemory = -1;
long availableMemory = -1;
object cpuTimes = null;
try
{
totalPhysicalMemory = MemoryStatistics.TotalPhysicalMemory;
availableMemory = MemoryStatistics.AvailableMemoryInMb;
using (var searcher = new ManagementObjectSearcher("select * from Win32_PerfFormattedData_PerfOS_Processor"))
{
cpuTimes = searcher.Get()
.Cast <ManagementObject>()
.Select(mo => new
{
Name = mo["Name"],
Usage = string.Format("{0} %", mo["PercentProcessorTime"])
}).ToArray();
}
}
catch (Exception e)
{
cpuTimes = "Could not get CPU times" + Environment.NewLine + e;
}
jsonSerializer.Serialize(streamWriter, new
{
TotalPhysicalMemory = string.Format("{0:#,#.##;;0} MB", totalPhysicalMemory),
AvailableMemory = string.Format("{0:#,#.##;;0} MB", availableMemory),
CurrentCpuUsage = cpuTimes
});
streamWriter.Flush();
}
}
private void ReplicateToDestination()
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiate);
NativeMemory.EnsureRegistered();
try
{
var connectionInfo = ReplicationUtils.GetTcpInfo(Destination.Url, GetNode(), "Replication",
_parent._server.RavenServer.ClusterCertificateHolder.Certificate);
if (_log.IsInfoEnabled)
{
_log.Info($"Will replicate to {Destination.FromString()} via {connectionInfo.Url}");
}
using (_parent._server.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var record = _parent.LoadDatabaseRecord();
if (record == null)
{
throw new InvalidOperationException($"The database record for {_parent.Database.Name} does not exist?!");
}
if (record.Encrypted && Destination.Url.StartsWith("https:", StringComparison.OrdinalIgnoreCase) == false)
{
throw new InvalidOperationException(
$"{record.DatabaseName} is encrypted, and require HTTPS for replication, but had endpoint with url {Destination.Url} to database {Destination.Database}");
}
}
var task = TcpUtils.ConnectSocketAsync(connectionInfo, _parent._server.Engine.TcpConnectionTimeout, _log);
task.Wait(CancellationToken);
using (_tcpClient = task.Result)
{
var wrapSsl = TcpUtils.WrapStreamWithSslAsync(_tcpClient, connectionInfo, _parent._server.RavenServer.ClusterCertificateHolder.Certificate);
wrapSsl.Wait(CancellationToken);
using (_stream = wrapSsl.Result)
using (_interruptibleRead = new InterruptibleRead(_database.DocumentsStorage.ContextPool, _stream))
using (_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance())
{
var documentSender = new ReplicationDocumentSender(_stream, this, _log);
WriteHeaderToRemotePeer();
//handle initial response to last etag and staff
try
{
var response = HandleServerResponse(getFullResponse: true);
if (response.ReplyType == ReplicationMessageReply.ReplyType.Error)
{
var exception = new InvalidOperationException(response.Reply.Exception);
if (response.Reply.Exception.Contains(nameof(DatabaseDoesNotExistException)) ||
response.Reply.Exception.Contains(nameof(DatabaseNotRelevantException)))
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, "Database does not exist");
DatabaseDoesNotExistException.ThrowWithMessageAndException(Destination.Database, response.Reply.Message, exception);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, $"Got error: {response.Reply.Exception}");
throw exception;
}
}
catch (DatabaseDoesNotExistException e)
{
var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} " +
"on the other end. ";
if (_external)
{
msg += "In order for the replication to work, a database with the same name needs to be created at the destination";
}
var young = (DateTime.UtcNow - _startedAt).TotalSeconds < 30;
if (young)
{
msg += "This can happen if the other node wasn't yet notified about being assigned this database and should be resolved shortly.";
}
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
// won't add an alert on young connections
// because it may take a few seconds for the other side to be notified by
// the cluster that it has this db.
if (young == false)
{
AddAlertOnFailureToReachOtherSide(msg, e);
}
throw;
}
catch (OperationCanceledException e)
{
const string msg = "Got operation canceled notification while opening outgoing replication channel. " +
"Aborting and closing the channel.";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
throw;
}
catch (Exception e)
{
const string msg = "Failed to parse initial server response. This is definitely not supposed to happen.";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
AddAlertOnFailureToReachOtherSide(msg, e);
throw;
}
while (_cts.IsCancellationRequested == false)
{
while (true)
{
if (_parent.DebugWaitAndRunReplicationOnce != null)
{
_parent.DebugWaitAndRunReplicationOnce.Wait(_cts.Token);
_parent.DebugWaitAndRunReplicationOnce.Reset();
}
var sp = Stopwatch.StartNew();
var stats = _lastStats = new OutgoingReplicationStatsAggregator(_parent.GetNextReplicationStatsId(), _lastStats);
AddReplicationPerformance(stats);
AddReplicationPulse(ReplicationPulseDirection.OutgoingBegin);
try
{
using (var scope = stats.CreateScope())
{
try
{
var didWork = documentSender.ExecuteReplicationOnce(scope);
if (didWork == false)
{
break;
}
DocumentsSend?.Invoke(this);
if (sp.ElapsedMilliseconds > 60 * 1000)
{
_waitForChanges.Set();
break;
}
}
catch (OperationCanceledException)
{
// cancellation is not an actual error,
// it is a "notification" that we need to cancel current operation
const string msg = "Operation was canceled.";
AddReplicationPulse(ReplicationPulseDirection.OutgoingError, msg);
throw;
}
catch (Exception e)
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingError, e.Message);
scope.AddError(e);
throw;
}
}
}
finally
{
stats.Complete();
AddReplicationPulse(ReplicationPulseDirection.OutgoingEnd);
}
}
//if this returns false, this means either timeout or canceled token is activated
while (WaitForChanges(_parent.MinimalHeartbeatInterval, _cts.Token) == false)
{
// open tx
// read current change vector compare to last sent
// if okay, send cv
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (var tx = ctx.OpenReadTransaction())
{
var etag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
if (etag == _lastSentDocumentEtag)
{
SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(ctx));
}
else
{
// we have updates that we need to send to the other side
// let's do that..
// this can happen if we got replication from another node
// that we need to send to it. Note that we typically
// will wait for the other node to send the data directly to
// our destination, but if it doesn't, we'll step in.
// In this case, we try to limit congestion in the network and
// only send updates that we have gotten from someone else after
// a certain time, to let the other side tell us that it already
// got it. Note that this is merely an optimization to reduce network
// traffic. It is fine to have the same data come from different sources.
break;
}
}
}
_waitForChanges.Reset();
}
}
}
}
catch (OperationCanceledException e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Operation canceled on replication thread ({FromToString}). This is not necessary due to an issue. Stopped the thread.");
}
Failed?.Invoke(this, e);
}
catch (IOException e)
{
if (_log.IsInfoEnabled)
{
if (e.InnerException is SocketException)
{
_log.Info(
$"SocketException was thrown from the connection to remote node ({FromToString}). This might mean that the remote node is done or there is a network issue.",
e);
}
else
{
_log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e);
}
}
Failed?.Invoke(this, e);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Unexpected exception occurred on replication thread ({FromToString}). Replication stopped (will be retried later).",
e);
}
Failed?.Invoke(this, e);
}
}
