private void ReplicateToDestination() { try { AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiate); NativeMemory.EnsureRegistered(); if (_log.IsInfoEnabled) { _log.Info($"Will replicate to {Destination.FromString()} via {_connectionInfo.Url}"); } using (_parent._server.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) { var record = _parent.LoadDatabaseRecord(); if (record == null) { throw new InvalidOperationException($"The database record for {_parent.Database.Name} does not exist?!"); } if (record.Encrypted && Destination.Url.StartsWith("https:", StringComparison.OrdinalIgnoreCase) == false) { throw new InvalidOperationException( $"{record.DatabaseName} is encrypted, and require HTTPS for replication, but had endpoint with url {Destination.Url} to database {Destination.Database}"); } } var task = TcpUtils.ConnectSocketAsync(_connectionInfo, _parent._server.Engine.TcpConnectionTimeout, _log); task.Wait(CancellationToken); using (Interlocked.Exchange(ref _tcpClient, task.Result)) { var wrapSsl = TcpUtils.WrapStreamWithSslAsync(_tcpClient, _connectionInfo, _parent._server.Server.Certificate.Certificate, _parent._server.Engine.TcpConnectionTimeout); wrapSsl.Wait(CancellationToken); using (_stream = wrapSsl.Result) // note that _stream is being disposed by the interruptible read using (_interruptibleRead = new InterruptibleRead(_database.DocumentsStorage.ContextPool, _stream)) using (_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance()) { var documentSender = new ReplicationDocumentSender(_stream, this, _log); WriteHeaderToRemotePeer(); //handle initial response to last etag and staff try { var response = HandleServerResponse(getFullResponse: true); switch (response.ReplyType) { //The first time we start replication we need to register the destination current CV case ReplicationMessageReply.ReplyType.Ok: LastAcceptedChangeVector = response.Reply.DatabaseChangeVector; break; case ReplicationMessageReply.ReplyType.Error: var exception = new InvalidOperationException(response.Reply.Exception); if (response.Reply.Exception.Contains(nameof(DatabaseDoesNotExistException)) || response.Reply.Exception.Contains(nameof(DatabaseNotRelevantException))) { AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, "Database does not exist"); DatabaseDoesNotExistException.ThrowWithMessageAndException(Destination.Database, response.Reply.Message, exception); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, $"Got error: {response.Reply.Exception}"); throw exception; } } catch (DatabaseDoesNotExistException e) { var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} " + "on the other end. "; if (_external) { msg += "In order for the replication to work, a database with the same name needs to be created at the destination"; } var young = (DateTime.UtcNow - _startedAt).TotalSeconds < 30; if (young) { msg += "This can happen if the other node wasn't yet notified about being assigned this database and should be resolved shortly."; } if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); // won't add an alert on young connections // because it may take a few seconds for the other side to be notified by // the cluster that it has this db. if (young == false) { AddAlertOnFailureToReachOtherSide(msg, e); } throw; } catch (OperationCanceledException e) { const string msg = "Got operation canceled notification while opening outgoing replication channel. " + "Aborting and closing the channel."; if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); throw; } catch (Exception e) { var msg = $"{OutgoingReplicationThreadName} got an unexpected exception during initial handshake"; if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); AddAlertOnFailureToReachOtherSide(msg, e); throw; } DateTime nextReplicateAt = default(DateTime); while (_cts.IsCancellationRequested == false) { while (_database.Time.GetUtcNow() > nextReplicateAt) { if (_parent.DebugWaitAndRunReplicationOnce != null) { _parent.DebugWaitAndRunReplicationOnce.Wait(_cts.Token); _parent.DebugWaitAndRunReplicationOnce.Reset(); } var sp = Stopwatch.StartNew(); var stats = _lastStats = new OutgoingReplicationStatsAggregator(_parent.GetNextReplicationStatsId(), _lastStats); AddReplicationPerformance(stats); AddReplicationPulse(ReplicationPulseDirection.OutgoingBegin); try { using (var scope = stats.CreateScope()) { try { if (Destination is InternalReplication dest) { _parent.EnsureNotDeleted(dest.NodeTag); } var didWork = documentSender.ExecuteReplicationOnce(scope, ref nextReplicateAt); if (didWork == false) { break; } if (Destination is ExternalReplication externalReplication) { var taskId = externalReplication.TaskId; UpdateExternalReplicationInfo(taskId); } DocumentsSend?.Invoke(this); if (sp.ElapsedMilliseconds > 60 * 1000) { _waitForChanges.Set(); break; } } catch (OperationCanceledException) { // cancellation is not an actual error, // it is a "notification" that we need to cancel current operation const string msg = "Operation was canceled."; AddReplicationPulse(ReplicationPulseDirection.OutgoingError, msg); throw; } catch (Exception e) { AddReplicationPulse(ReplicationPulseDirection.OutgoingError, e.Message); scope.AddError(e); throw; } } } finally { stats.Complete(); AddReplicationPulse(ReplicationPulseDirection.OutgoingEnd); } } //if this returns false, this means either timeout or canceled token is activated while (WaitForChanges(_parent.MinimalHeartbeatInterval, _cts.Token) == false) { //If we got cancelled we need to break right away if (_cts.IsCancellationRequested) { break; } // open tx // read current change vector compare to last sent // if okay, send cv using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (var tx = ctx.OpenReadTransaction()) { var etag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); if (etag == _lastSentDocumentEtag) { SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(ctx)); _parent.CompleteDeletionIfNeeded(); } else if (nextReplicateAt > DateTime.UtcNow) { SendHeartbeat(null); } else { //Send a heartbeat first so we will get an updated CV of the destination var currentChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); SendHeartbeat(null); //If our previous CV is already merged to the destination wait a bit more if (ChangeVectorUtils.GetConflictStatus(LastAcceptedChangeVector, currentChangeVector) == ConflictStatus.AlreadyMerged) { continue; } // we have updates that we need to send to the other side // let's do that.. // this can happen if we got replication from another node // that we need to send to it. Note that we typically // will wait for the other node to send the data directly to // our destination, but if it doesn't, we'll step in. // In this case, we try to limit congestion in the network and // only send updates that we have gotten from someone else after // a certain time, to let the other side tell us that it already // got it. Note that this is merely an optimization to reduce network // traffic. It is fine to have the same data come from different sources. break; } } } _waitForChanges.Reset(); } } } } catch (AggregateException e) { if (e.InnerExceptions.Count == 1) { if (e.InnerException is OperationCanceledException oce) { HandleOperationCancelException(oce); } if (e.InnerException is IOException ioe) { HandleIOException(ioe); } } HandleException(e); } catch (OperationCanceledException e) { HandleOperationCancelException(e); } catch (IOException e) { HandleIOException(e); } catch (Exception e) { HandleException(e); } void HandleOperationCancelException(OperationCanceledException e) { if (_log.IsInfoEnabled) { _log.Info($"Operation canceled on replication thread ({FromToString}). " + $"This is not necessary due to an issue. Stopped the thread."); } if (_cts.IsCancellationRequested == false) { Failed?.Invoke(this, e); } } void HandleIOException(IOException e) { if (_log.IsInfoEnabled) { if (e.InnerException is SocketException) { _log.Info($"SocketException was thrown from the connection to remote node ({FromToString}). " + $"This might mean that the remote node is done or there is a network issue.", e); } else { _log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e); } } Failed?.Invoke(this, e); } void HandleException(Exception e) { if (_log.IsInfoEnabled) { _log.Info($"Unexpected exception occurred on replication thread ({FromToString}). " + $"Replication stopped (will be retried later).", e); } Failed?.Invoke(this, e); } }
private void ReplicateToDestination() { try { var connectionInfo = GetTcpInfo(); using (_tcpClient = new TcpClient()) { ConnectSocket(connectionInfo, _tcpClient); using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out _documentsContext)) using (_database.ConfigurationStorage.ContextPool.AllocateOperationContext(out _configurationContext)) using (var stream = _tcpClient.GetStream()) { var documentSender = new ReplicationDocumentSender(stream, this, _log); var indexAndTransformerSender = new ReplicationIndexTransformerSender(stream, this, _log); using (_writer = new BlittableJsonTextWriter(_documentsContext, stream)) using (_parser = _documentsContext.ParseMultiFrom(stream)) { //send initial connection information _documentsContext.Write(_writer, new DynamicJsonValue { [nameof(TcpConnectionHeaderMessage.DatabaseName)] = _destination.Database, [nameof(TcpConnectionHeaderMessage.Operation)] = TcpConnectionHeaderMessage.OperationTypes.Replication.ToString(), }); //start request/response for fetching last etag _documentsContext.Write(_writer, new DynamicJsonValue { ["Type"] = "GetLastEtag", ["SourceDatabaseId"] = _database.DbId.ToString(), ["SourceDatabaseName"] = _database.Name, ["SourceUrl"] = _database.Configuration.Core.ServerUrl, ["MachineName"] = Environment.MachineName, }); _writer.Flush(); //handle initial response to last etag and staff try { using (_configurationContext.OpenReadTransaction()) using (_documentsContext.OpenReadTransaction()) { var response = HandleServerResponse(); if (response.Item1 == ReplicationMessageReply.ReplyType.Error) { if (response.Item2.Contains("DatabaseDoesNotExistsException")) { throw new DatabaseDoesNotExistsException(); } throw new InvalidOperationException(response.Item2); } } } catch (DatabaseDoesNotExistsException e) { var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} on the other end. " + "In order for the replication to work, a database with the same name needs to be created at the destination"; if (_log.IsInfoEnabled) { _log.Info(msg, e); } using (var txw = _configurationContext.OpenWriteTransaction()) { _database.Alerts.AddAlert(new Alert { Key = FromToString, Type = AlertType.Replication, Message = msg, CreatedAt = DateTime.UtcNow, Severity = AlertSeverity.Warning }, _configurationContext, txw); txw.Commit(); } throw; } catch (Exception e) { var msg = $"Failed to parse initial server response. This is definitely not supposed to happen. Exception thrown: {e}"; if (_log.IsInfoEnabled) { _log.Info(msg, e); } using (var txw = _configurationContext.OpenWriteTransaction()) { _database.Alerts.AddAlert(new Alert { Key = FromToString, Type = AlertType.Replication, Message = msg, CreatedAt = DateTime.UtcNow, Severity = AlertSeverity.Error }, _configurationContext, txw); txw.Commit(); } throw; } while (_cts.IsCancellationRequested == false) { _documentsContext.ResetAndRenew(); long currentEtag; Debug.Assert(_database.IndexMetadataPersistence.IsInitialized); using (_configurationContext.OpenReadTransaction()) currentEtag = _database.IndexMetadataPersistence.ReadLastEtag(_configurationContext.Transaction.InnerTransaction); if (_destination.SkipIndexReplication == false && currentEtag != indexAndTransformerSender.LastEtag) { indexAndTransformerSender.ExecuteReplicationOnce(); } var sp = Stopwatch.StartNew(); while (documentSender.ExecuteReplicationOnce()) { if (sp.ElapsedMilliseconds > 60 * 1000) { _waitForChanges.Set(); break; } } //if this returns false, this means either timeout or canceled token is activated while (WaitForChanges(_minimalHeartbeatInterval, _cts.Token) == false) { _configurationContext.ResetAndRenew(); _documentsContext.ResetAndRenew(); using (_documentsContext.OpenReadTransaction()) using (_configurationContext.OpenReadTransaction()) { SendHeartbeat(); } } _waitForChanges.Reset(); } } } } } catch (OperationCanceledException) { if (_log.IsInfoEnabled) { _log.Info($"Operation canceled on replication thread ({FromToString}). Stopped the thread."); } } catch (IOException e) { if (_log.IsInfoEnabled) { if (e.InnerException is SocketException) { _log.Info( $"SocketException was thrown from the connection to remote node ({FromToString}). This might mean that the remote node is done or there is a network issue.", e); } else { _log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e); } } } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info($"Unexpected exception occured on replication thread ({FromToString}). Replication stopped (will be retried later).", e); } Failed?.Invoke(this, e); } }