private void ReplicateToDestination() { try { var connectionInfo = GetTcpInfo(); using (_tcpClient = new TcpClient()) { ConnectSocket(connectionInfo, _tcpClient); using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out _documentsContext)) using (_database.ConfigurationStorage.ContextPool.AllocateOperationContext(out _configurationContext)) using (var stream = _tcpClient.GetStream()) { var documentSender = new ReplicationDocumentSender(stream, this, _log); var indexAndTransformerSender = new ReplicationIndexTransformerSender(stream, this, _log); using (_writer = new BlittableJsonTextWriter(_documentsContext, stream)) using (_parser = _documentsContext.ParseMultiFrom(stream)) { //send initial connection information _documentsContext.Write(_writer, new DynamicJsonValue { [nameof(TcpConnectionHeaderMessage.DatabaseName)] = _destination.Database, [nameof(TcpConnectionHeaderMessage.Operation)] = TcpConnectionHeaderMessage.OperationTypes.Replication.ToString(), }); //start request/response for fetching last etag _documentsContext.Write(_writer, new DynamicJsonValue { ["Type"] = "GetLastEtag", ["SourceDatabaseId"] = _database.DbId.ToString(), ["SourceDatabaseName"] = _database.Name, ["SourceUrl"] = _database.Configuration.Core.ServerUrl, ["MachineName"] = Environment.MachineName, }); _writer.Flush(); //handle initial response to last etag and staff try { using (_configurationContext.OpenReadTransaction()) using (_documentsContext.OpenReadTransaction()) { var response = HandleServerResponse(); if (response.Item1 == ReplicationMessageReply.ReplyType.Error) { if (response.Item2.Contains("DatabaseDoesNotExistsException")) { throw new DatabaseDoesNotExistsException(); } throw new InvalidOperationException(response.Item2); } } } catch (DatabaseDoesNotExistsException e) { var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} on the other end. " + "In order for the replication to work, a database with the same name needs to be created at the destination"; if (_log.IsInfoEnabled) { _log.Info(msg, e); } using (var txw = _configurationContext.OpenWriteTransaction()) { _database.Alerts.AddAlert(new Alert { Key = FromToString, Type = AlertType.Replication, Message = msg, CreatedAt = DateTime.UtcNow, Severity = AlertSeverity.Warning }, _configurationContext, txw); txw.Commit(); } throw; } catch (Exception e) { var msg = $"Failed to parse initial server response. This is definitely not supposed to happen. Exception thrown: {e}"; if (_log.IsInfoEnabled) { _log.Info(msg, e); } using (var txw = _configurationContext.OpenWriteTransaction()) { _database.Alerts.AddAlert(new Alert { Key = FromToString, Type = AlertType.Replication, Message = msg, CreatedAt = DateTime.UtcNow, Severity = AlertSeverity.Error }, _configurationContext, txw); txw.Commit(); } throw; } while (_cts.IsCancellationRequested == false) { _documentsContext.ResetAndRenew(); long currentEtag; Debug.Assert(_database.IndexMetadataPersistence.IsInitialized); using (_configurationContext.OpenReadTransaction()) currentEtag = _database.IndexMetadataPersistence.ReadLastEtag(_configurationContext.Transaction.InnerTransaction); if (_destination.SkipIndexReplication == false && currentEtag != indexAndTransformerSender.LastEtag) { indexAndTransformerSender.ExecuteReplicationOnce(); } var sp = Stopwatch.StartNew(); while (documentSender.ExecuteReplicationOnce()) { if (sp.ElapsedMilliseconds > 60 * 1000) { _waitForChanges.Set(); break; } } //if this returns false, this means either timeout or canceled token is activated while (WaitForChanges(_minimalHeartbeatInterval, _cts.Token) == false) { _configurationContext.ResetAndRenew(); _documentsContext.ResetAndRenew(); using (_documentsContext.OpenReadTransaction()) using (_configurationContext.OpenReadTransaction()) { SendHeartbeat(); } } _waitForChanges.Reset(); } } } } } catch (OperationCanceledException) { if (_log.IsInfoEnabled) { _log.Info($"Operation canceled on replication thread ({FromToString}). Stopped the thread."); } } catch (IOException e) { if (_log.IsInfoEnabled) { if (e.InnerException is SocketException) { _log.Info( $"SocketException was thrown from the connection to remote node ({FromToString}). This might mean that the remote node is done or there is a network issue.", e); } else { _log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e); } } } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info($"Unexpected exception occured on replication thread ({FromToString}). Replication stopped (will be retried later).", e); } Failed?.Invoke(this, e); } }