public Dictionary <string, long> GetLastProcessedDocumentTombstonesPerCollection() { var minEtag = MinimalEtagForReplication; var result = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase) { { Constants.Documents.Collections.AllDocumentsCollection, minEtag } }; if (Destinations == null) { return(result); } ReplicationNode disabledReplicationNode = null; bool hasDisabled = false; foreach (var replicationDocumentDestination in Destinations) { if (replicationDocumentDestination.Disabled) { disabledReplicationNode = replicationDocumentDestination; hasDisabled = true; break; } } if (hasDisabled == false) { return(result); } const int maxTombstones = 16 * 1024; bool tooManyTombstones; using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { tooManyTombstones = Database.DocumentsStorage.HasMoreOfTombstonesAfter(context, minEtag, maxTombstones); } if (!tooManyTombstones) { return(result); } Database.NotificationCenter.Add( PerformanceHint.Create( database: Database.Name, title: "Large number of tombstones because of disabled replication destination", msg: $"The disabled replication destination {disabledReplicationNode.FromString()} prevents from cleaning large number of tombstones.", type: PerformanceHintType.Replication, notificationSeverity: NotificationSeverity.Warning, source: disabledReplicationNode.FromString() )); return(result); }
private TcpConnectionInfo GetConnectionInfo(ReplicationNode node, bool external) { var shutdownInfo = new ConnectionShutdownInfo { Node = node, External = external }; _outgoingFailureInfo.TryAdd(node, shutdownInfo); try { if (node is ExternalReplication exNode) { using (var requestExecutor = RequestExecutor.Create(exNode.ConnectionString.TopologyDiscoveryUrls, exNode.Database, _server.Server.ClusterCertificateHolder.Certificate, DocumentConventions.Default)) using (_server.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) { var cmd = new GetTcpInfoCommand("extrenal-replication"); requestExecutor.Execute(cmd, ctx); node.Url = requestExecutor.Url; return(cmd.Result); } } if (node is InternalReplication internalNode) { return(ReplicationUtils.GetTcpInfo(internalNode.Url, internalNode.NodeTag, "Replication", _server.Server.ClusterCertificateHolder.Certificate)); } throw new InvalidOperationException( $"Unexpected replication node type, Expected to be '{typeof(ExternalReplication)}' or '{typeof(InternalReplication)}', but got '{node.GetType()}'"); } catch (Exception e) { // will try to fetch it again later if (_log.IsInfoEnabled) { _log.Info($"Failed to fetch tcp connection information for the destination '{node.FromString()}' , the connection will be retried later.", e); } _reconnectQueue.TryAdd(shutdownInfo); } return(null); }
private void ReplicateToDestination() { try { AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiate); NativeMemory.EnsureRegistered(); if (_log.IsInfoEnabled) { _log.Info($"Will replicate to {Destination.FromString()} via {_connectionInfo.Url}"); } using (_parent._server.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) { var record = _parent.LoadDatabaseRecord(); if (record == null) { throw new InvalidOperationException($"The database record for {_parent.Database.Name} does not exist?!"); } if (record.Encrypted && Destination.Url.StartsWith("https:", StringComparison.OrdinalIgnoreCase) == false) { throw new InvalidOperationException( $"{record.DatabaseName} is encrypted, and require HTTPS for replication, but had endpoint with url {Destination.Url} to database {Destination.Database}"); } } var task = TcpUtils.ConnectSocketAsync(_connectionInfo, _parent._server.Engine.TcpConnectionTimeout, _log); task.Wait(CancellationToken); using (Interlocked.Exchange(ref _tcpClient, task.Result)) { var wrapSsl = TcpUtils.WrapStreamWithSslAsync(_tcpClient, _connectionInfo, _parent._server.Server.Certificate.Certificate, _parent._server.Engine.TcpConnectionTimeout); wrapSsl.Wait(CancellationToken); using (_stream = wrapSsl.Result) // note that _stream is being disposed by the interruptible read using (_interruptibleRead = new InterruptibleRead(_database.DocumentsStorage.ContextPool, _stream)) using (_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance()) { var documentSender = new ReplicationDocumentSender(_stream, this, _log); WriteHeaderToRemotePeer(); //handle initial response to last etag and staff try { var response = HandleServerResponse(getFullResponse: true); switch (response.ReplyType) { //The first time we start replication we need to register the destination current CV case ReplicationMessageReply.ReplyType.Ok: LastAcceptedChangeVector = response.Reply.DatabaseChangeVector; break; case ReplicationMessageReply.ReplyType.Error: var exception = new InvalidOperationException(response.Reply.Exception); if (response.Reply.Exception.Contains(nameof(DatabaseDoesNotExistException)) || response.Reply.Exception.Contains(nameof(DatabaseNotRelevantException))) { AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, "Database does not exist"); DatabaseDoesNotExistException.ThrowWithMessageAndException(Destination.Database, response.Reply.Message, exception); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, $"Got error: {response.Reply.Exception}"); throw exception; } } catch (DatabaseDoesNotExistException e) { var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} " + "on the other end. "; if (_external) { msg += "In order for the replication to work, a database with the same name needs to be created at the destination"; } var young = (DateTime.UtcNow - _startedAt).TotalSeconds < 30; if (young) { msg += "This can happen if the other node wasn't yet notified about being assigned this database and should be resolved shortly."; } if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); // won't add an alert on young connections // because it may take a few seconds for the other side to be notified by // the cluster that it has this db. if (young == false) { AddAlertOnFailureToReachOtherSide(msg, e); } throw; } catch (OperationCanceledException e) { const string msg = "Got operation canceled notification while opening outgoing replication channel. " + "Aborting and closing the channel."; if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); throw; } catch (Exception e) { var msg = $"{OutgoingReplicationThreadName} got an unexpected exception during initial handshake"; if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); AddAlertOnFailureToReachOtherSide(msg, e); throw; } DateTime nextReplicateAt = default(DateTime); while (_cts.IsCancellationRequested == false) { while (_database.Time.GetUtcNow() > nextReplicateAt) { if (_parent.DebugWaitAndRunReplicationOnce != null) { _parent.DebugWaitAndRunReplicationOnce.Wait(_cts.Token); _parent.DebugWaitAndRunReplicationOnce.Reset(); } var sp = Stopwatch.StartNew(); var stats = _lastStats = new OutgoingReplicationStatsAggregator(_parent.GetNextReplicationStatsId(), _lastStats); AddReplicationPerformance(stats); AddReplicationPulse(ReplicationPulseDirection.OutgoingBegin); try { using (var scope = stats.CreateScope()) { try { if (Destination is InternalReplication dest) { _parent.EnsureNotDeleted(dest.NodeTag); } var didWork = documentSender.ExecuteReplicationOnce(scope, ref nextReplicateAt); if (didWork == false) { break; } if (Destination is ExternalReplication externalReplication) { var taskId = externalReplication.TaskId; UpdateExternalReplicationInfo(taskId); } DocumentsSend?.Invoke(this); if (sp.ElapsedMilliseconds > 60 * 1000) { _waitForChanges.Set(); break; } } catch (OperationCanceledException) { // cancellation is not an actual error, // it is a "notification" that we need to cancel current operation const string msg = "Operation was canceled."; AddReplicationPulse(ReplicationPulseDirection.OutgoingError, msg); throw; } catch (Exception e) { AddReplicationPulse(ReplicationPulseDirection.OutgoingError, e.Message); scope.AddError(e); throw; } } } finally { stats.Complete(); AddReplicationPulse(ReplicationPulseDirection.OutgoingEnd); } } //if this returns false, this means either timeout or canceled token is activated while (WaitForChanges(_parent.MinimalHeartbeatInterval, _cts.Token) == false) { //If we got cancelled we need to break right away if (_cts.IsCancellationRequested) { break; } // open tx // read current change vector compare to last sent // if okay, send cv using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (var tx = ctx.OpenReadTransaction()) { var etag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); if (etag == _lastSentDocumentEtag) { SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(ctx)); _parent.CompleteDeletionIfNeeded(); } else if (nextReplicateAt > DateTime.UtcNow) { SendHeartbeat(null); } else { //Send a heartbeat first so we will get an updated CV of the destination var currentChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); SendHeartbeat(null); //If our previous CV is already merged to the destination wait a bit more if (ChangeVectorUtils.GetConflictStatus(LastAcceptedChangeVector, currentChangeVector) == ConflictStatus.AlreadyMerged) { continue; } // we have updates that we need to send to the other side // let's do that.. // this can happen if we got replication from another node // that we need to send to it. Note that we typically // will wait for the other node to send the data directly to // our destination, but if it doesn't, we'll step in. // In this case, we try to limit congestion in the network and // only send updates that we have gotten from someone else after // a certain time, to let the other side tell us that it already // got it. Note that this is merely an optimization to reduce network // traffic. It is fine to have the same data come from different sources. break; } } } _waitForChanges.Reset(); } } } } catch (AggregateException e) { if (e.InnerExceptions.Count == 1) { if (e.InnerException is OperationCanceledException oce) { HandleOperationCancelException(oce); } if (e.InnerException is IOException ioe) { HandleIOException(ioe); } } HandleException(e); } catch (OperationCanceledException e) { HandleOperationCancelException(e); } catch (IOException e) { HandleIOException(e); } catch (Exception e) { HandleException(e); } void HandleOperationCancelException(OperationCanceledException e) { if (_log.IsInfoEnabled) { _log.Info($"Operation canceled on replication thread ({FromToString}). " + $"This is not necessary due to an issue. Stopped the thread."); } if (_cts.IsCancellationRequested == false) { Failed?.Invoke(this, e); } } void HandleIOException(IOException e) { if (_log.IsInfoEnabled) { if (e.InnerException is SocketException) { _log.Info($"SocketException was thrown from the connection to remote node ({FromToString}). " + $"This might mean that the remote node is done or there is a network issue.", e); } else { _log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e); } } Failed?.Invoke(this, e); } void HandleException(Exception e) { if (_log.IsInfoEnabled) { _log.Info($"Unexpected exception occurred on replication thread ({FromToString}). " + $"Replication stopped (will be retried later).", e); } Failed?.Invoke(this, e); } }