private void ReplicateToDestination()
{
try
{
var connectionInfo = GetTcpInfo();
using (_tcpClient = new TcpClient())
{
ConnectSocket(connectionInfo, _tcpClient);
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out _documentsContext))
using (_database.ConfigurationStorage.ContextPool.AllocateOperationContext(out _configurationContext))
using (var stream = _tcpClient.GetStream())
{
var documentSender = new ReplicationDocumentSender(stream, this, _log);
var indexAndTransformerSender = new ReplicationIndexTransformerSender(stream, this, _log);
using (_writer = new BlittableJsonTextWriter(_documentsContext, stream))
using (_parser = _documentsContext.ParseMultiFrom(stream))
{
//send initial connection information
_documentsContext.Write(_writer, new DynamicJsonValue
{
[nameof(TcpConnectionHeaderMessage.DatabaseName)] = _destination.Database,
[nameof(TcpConnectionHeaderMessage.Operation)] = TcpConnectionHeaderMessage.OperationTypes.Replication.ToString(),
});
//start request/response for fetching last etag
_documentsContext.Write(_writer, new DynamicJsonValue
{
["Type"] = "GetLastEtag",
["SourceDatabaseId"] = _database.DbId.ToString(),
["SourceDatabaseName"] = _database.Name,
["SourceUrl"] = _database.Configuration.Core.ServerUrl,
["MachineName"] = Environment.MachineName,
});
_writer.Flush();
//handle initial response to last etag and staff
try
{
using (_configurationContext.OpenReadTransaction())
using (_documentsContext.OpenReadTransaction())
{
var response = HandleServerResponse();
if (response.Item1 == ReplicationMessageReply.ReplyType.Error)
{
if (response.Item2.Contains("DatabaseDoesNotExistsException"))
{
throw new DatabaseDoesNotExistsException();
}
throw new InvalidOperationException(response.Item2);
}
}
}
catch (DatabaseDoesNotExistsException e)
{
var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} on the other end. " +
"In order for the replication to work, a database with the same name needs to be created at the destination";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
using (var txw = _configurationContext.OpenWriteTransaction())
{
_database.Alerts.AddAlert(new Alert
{
Key = FromToString,
Type = AlertType.Replication,
Message = msg,
CreatedAt = DateTime.UtcNow,
Severity = AlertSeverity.Warning
}, _configurationContext, txw);
txw.Commit();
}
throw;
}
catch (Exception e)
{
var msg =
$"Failed to parse initial server response. This is definitely not supposed to happen. Exception thrown: {e}";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
using (var txw = _configurationContext.OpenWriteTransaction())
{
_database.Alerts.AddAlert(new Alert
{
Key = FromToString,
Type = AlertType.Replication,
Message = msg,
CreatedAt = DateTime.UtcNow,
Severity = AlertSeverity.Error
}, _configurationContext, txw);
txw.Commit();
}
throw;
}
while (_cts.IsCancellationRequested == false)
{
_documentsContext.ResetAndRenew();
long currentEtag;
Debug.Assert(_database.IndexMetadataPersistence.IsInitialized);
using (_configurationContext.OpenReadTransaction())
currentEtag = _database.IndexMetadataPersistence.ReadLastEtag(_configurationContext.Transaction.InnerTransaction);
if (_destination.SkipIndexReplication == false &&
currentEtag != indexAndTransformerSender.LastEtag)
{
indexAndTransformerSender.ExecuteReplicationOnce();
}
var sp = Stopwatch.StartNew();
while (documentSender.ExecuteReplicationOnce())
{
if (sp.ElapsedMilliseconds > 60 * 1000)
{
_waitForChanges.Set();
break;
}
}
//if this returns false, this means either timeout or canceled token is activated
while (WaitForChanges(_minimalHeartbeatInterval, _cts.Token) == false)
{
_configurationContext.ResetAndRenew();
_documentsContext.ResetAndRenew();
using (_documentsContext.OpenReadTransaction())
using (_configurationContext.OpenReadTransaction())
{
SendHeartbeat();
}
}
_waitForChanges.Reset();
}
}
}
}
}
catch (OperationCanceledException)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Operation canceled on replication thread ({FromToString}). Stopped the thread.");
}
}
catch (IOException e)
{
if (_log.IsInfoEnabled)
{
if (e.InnerException is SocketException)
{
_log.Info(
$"SocketException was thrown from the connection to remote node ({FromToString}). This might mean that the remote node is done or there is a network issue.",
e);
}
else
{
_log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e);
}
}
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Unexpected exception occured on replication thread ({FromToString}). Replication stopped (will be retried later).", e);
}
Failed?.Invoke(this, e);
}
}