private void IncomingHandlerRemoved(IncomingReplicationHandler handler)
{
if (_incoming.TryRemove(handler.ConnectionInfo.SourceDatabaseId, out var stats))
{
stats.Handler.DocumentsReceived -= IncomingDocumentsReceived;
}
}
public void All_local_etags_lower_than_remote_should_return_Update_at_conflict_status()
{
var dbIds = new List <Guid> {
Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid()
};
var local = new[]
{
new ChangeVectorEntry {
DbId = dbIds[0], Etag = 1
},
new ChangeVectorEntry {
DbId = dbIds[1], Etag = 2
},
new ChangeVectorEntry {
DbId = dbIds[2], Etag = 3
},
};
var remote = new[]
{
new ChangeVectorEntry {
DbId = dbIds[0], Etag = 10
},
new ChangeVectorEntry {
DbId = dbIds[1], Etag = 20
},
new ChangeVectorEntry {
DbId = dbIds[2], Etag = 30
},
};
Assert.Equal(IncomingReplicationHandler.ConflictStatus.Update, IncomingReplicationHandler.GetConflictStatus(remote, local));
}
public void Some_remote_etags_lower_than_local_and_some_higher_should_return_Conflict_at_conflict_status_with_different_order()
{
var dbIds = new List <Guid> {
Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid()
};
var local = new[]
{
new ChangeVectorEntry {
DbId = dbIds[1], Etag = 75
},
new ChangeVectorEntry {
DbId = dbIds[0], Etag = 10
},
new ChangeVectorEntry {
DbId = dbIds[2], Etag = 3
},
};
var remote = new[]
{
new ChangeVectorEntry {
DbId = dbIds[1], Etag = 95
},
new ChangeVectorEntry {
DbId = dbIds[2], Etag = 2
},
new ChangeVectorEntry {
DbId = dbIds[0], Etag = 10
},
};
Assert.Equal(IncomingReplicationHandler.ConflictStatus.Conflict, IncomingReplicationHandler.GetConflictStatus(remote, local));
}
private void IncomingHandlerAdded(IncomingReplicationHandler handler)
{
_incoming.GetOrAdd(handler.ConnectionInfo.SourceDatabaseId, key =>
{
handler.DocumentsReceived += IncomingDocumentsReceived;
return(new ReplicationHandlerAndPerformanceStatsList <IncomingReplicationHandler, IncomingReplicationStatsAggregator>(handler));
});
}
private void OnIncomingReceiveSucceeded(IncomingReplicationHandler instance)
{
_incomingLastActivityTime.AddOrUpdate(instance.ConnectionInfo, DateTime.UtcNow, (_, __) => DateTime.UtcNow);
foreach (var handler in _incoming.Values)
{
if (handler != instance)
{
handler.OnReplicationFromAnotherSource();
}
}
}
private void OnIncomingReceiveSucceeded(IncomingReplicationHandler instance)
{
_incomingLastActivityTime.AddOrUpdate(instance.ConnectionInfo, DateTime.UtcNow, (_, __) => DateTime.UtcNow);
// PERF: _incoming locks if you do _incoming.Values. Using .Select
// directly and fetching the Value avoids this problem.
foreach (var kv in _incoming)
{
var handler = kv.Value;
if (handler != instance)
{
handler.OnReplicationFromAnotherSource();
}
}
}
private void IncomingDocumentsReceived(IncomingReplicationHandler handler)
{
if (_incoming.TryGetValue(handler.ConnectionInfo.SourceDatabaseId, out var stats) == false)
{
// possible?
return;
}
var latestStat = stats.Handler.GetLatestReplicationPerformance();
if (latestStat != null)
{
stats.Performance.Add(latestStat, _cts.Token);
}
}
private void OnIncomingReceiveFailed(IncomingReplicationHandler instance, Exception e)
{
using (instance)
{
IncomingReplicationHandler _;
_incoming.TryRemove(instance.ConnectionInfo.SourceDatabaseId, out _);
instance.Failed -= OnIncomingReceiveFailed;
instance.DocumentsReceived -= OnIncomingReceiveSucceeded;
if (_log.IsInfoEnabled)
{
_log.Info($"Incoming replication handler has thrown an unhandled exception. ({instance.FromToString})", e);
}
}
}
public void Remote_change_vector_with_different_dbId_set_than_local_should_return_Conflict_at_conflict_status()
{
var dbIds = new List <Guid> {
Guid.NewGuid(), Guid.NewGuid()
};
var local = new[]
{
new ChangeVectorEntry {
DbId = dbIds[0], Etag = 10
},
};
var remote = new[]
{
new ChangeVectorEntry {
DbId = dbIds[1], Etag = 10
}
};
Assert.Equal(IncomingReplicationHandler.ConflictStatus.Conflict, IncomingReplicationHandler.GetConflictStatus(remote, local));
}
public void Remote_change_vector_smaller_than_local_and_some_remote_etags_higher_than_local_should_return_Conflict_at_conflict_status()
{
var dbIds = new List <Guid> {
Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid()
};
var local = new[]
{
new ChangeVectorEntry {
DbId = dbIds[0], Etag = 10
},
new ChangeVectorEntry {
DbId = dbIds[1], Etag = 20
},
new ChangeVectorEntry {
DbId = dbIds[2], Etag = 3000
},
new ChangeVectorEntry {
DbId = dbIds[3], Etag = 40
}
};
var remote = new[]
{
new ChangeVectorEntry {
DbId = dbIds[0], Etag = 100
},
new ChangeVectorEntry {
DbId = dbIds[1], Etag = 200
},
new ChangeVectorEntry {
DbId = dbIds[2], Etag = 300
}
};
Assert.Equal(IncomingReplicationHandler.ConflictStatus.Conflict, IncomingReplicationHandler.GetConflictStatus(remote, local));
}
private void IncomingHandlerAdded(IncomingReplicationHandler handler)
{
handler.HandleReplicationPulse += HandleReplicationPulse;
}
private void IncomingHandlerRemoved(IncomingReplicationHandler handler)
{
handler.HandleReplicationPulse -= HandleReplicationPulse;
}
public void AcceptIncomingConnection(TcpConnectionOptions tcpConnectionOptions)
{
ReplicationLatestEtagRequest getLatestEtagMessage;
using (tcpConnectionOptions.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var readerObject = context.ParseToMemory(
tcpConnectionOptions.Stream,
"IncomingReplication/get-last-etag-message read",
BlittableJsonDocumentBuilder.UsageMode.None,
tcpConnectionOptions.PinnedBuffer))
{
getLatestEtagMessage = JsonDeserializationServer.ReplicationLatestEtagRequest(readerObject);
if (_log.IsInfoEnabled)
{
_log.Info(
$"GetLastEtag: {getLatestEtagMessage.SourceTag}({getLatestEtagMessage.SourceMachineName}) / {getLatestEtagMessage.SourceDatabaseName} ({getLatestEtagMessage.SourceDatabaseId}) - {getLatestEtagMessage.SourceUrl}");
}
}
var connectionInfo = IncomingConnectionInfo.FromGetLatestEtag(getLatestEtagMessage);
try
{
AssertValidConnection(connectionInfo);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Connection from [{connectionInfo}] is rejected.", e);
}
var incomingConnectionRejectionInfos = _incomingRejectionStats.GetOrAdd(connectionInfo,
_ => new ConcurrentQueue <IncomingConnectionRejectionInfo>());
incomingConnectionRejectionInfos.Enqueue(new IncomingConnectionRejectionInfo {
Reason = e.ToString()
});
try
{
tcpConnectionOptions.Dispose();
}
catch
{
// do nothing
}
throw;
}
try
{
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsOperationContext))
using (Database.ConfigurationStorage.ContextPool.AllocateOperationContext(out TransactionOperationContext configurationContext))
using (var writer = new BlittableJsonTextWriter(documentsOperationContext, tcpConnectionOptions.Stream))
using (documentsOperationContext.OpenReadTransaction())
using (configurationContext.OpenReadTransaction())
{
var changeVector = DocumentsStorage.GetDatabaseChangeVector(documentsOperationContext);
var lastEtagFromSrc = Database.DocumentsStorage.GetLastReplicateEtagFrom(
documentsOperationContext, getLatestEtagMessage.SourceDatabaseId);
if (_log.IsInfoEnabled)
{
_log.Info($"GetLastEtag response, last etag: {lastEtagFromSrc}");
}
var response = new DynamicJsonValue
{
[nameof(ReplicationMessageReply.Type)] = "Ok",
[nameof(ReplicationMessageReply.MessageType)] = ReplicationMessageType.Heartbeat,
[nameof(ReplicationMessageReply.LastEtagAccepted)] = lastEtagFromSrc,
[nameof(ReplicationMessageReply.NodeTag)] = _server.NodeTag,
[nameof(ReplicationMessageReply.DatabaseChangeVector)] = changeVector
};
documentsOperationContext.Write(writer, response);
writer.Flush();
}
}
catch (Exception)
{
try
{
tcpConnectionOptions.Dispose();
}
catch (Exception)
{
// do nothing
}
throw;
}
var newIncoming = new IncomingReplicationHandler(
tcpConnectionOptions,
getLatestEtagMessage,
this);
newIncoming.Failed += OnIncomingReceiveFailed;
newIncoming.DocumentsReceived += OnIncomingReceiveSucceeded;
if (_log.IsInfoEnabled)
{
_log.Info(
$"Initialized document replication connection from {connectionInfo.SourceDatabaseName} located at {connectionInfo.SourceUrl}");
}
// need to safeguard against two concurrent connection attempts
var newConnection = _incoming.GetOrAdd(newIncoming.ConnectionInfo.SourceDatabaseId, newIncoming);
if (newConnection == newIncoming)
{
newIncoming.Start();
IncomingReplicationAdded?.Invoke(newIncoming);
ForceTryReconnectAll();
}
else
{
newIncoming.Dispose();
}
}
public void AcceptIncomingConnection(TcpConnectionOptions tcpConnectionOptions)
{
ReplicationLatestEtagRequest getLatestEtagMessage;
using (var readerObject = tcpConnectionOptions.MultiDocumentParser.ParseToMemory("IncomingReplication/get-last-etag-message read"))
{
getLatestEtagMessage = JsonDeserializationServer.ReplicationLatestEtagRequest(readerObject);
if (_log.IsInfoEnabled)
{
_log.Info($"GetLastEtag: {getLatestEtagMessage.SourceMachineName} / {getLatestEtagMessage.SourceDatabaseName} ({getLatestEtagMessage.SourceDatabaseId}) - {getLatestEtagMessage.SourceUrl}");
}
}
var connectionInfo = IncomingConnectionInfo.FromGetLatestEtag(getLatestEtagMessage);
try
{
AssertValidConnection(connectionInfo);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Connection from [{connectionInfo}] is rejected.", e);
}
var incomingConnectionRejectionInfos = _incomingRejectionStats.GetOrAdd(connectionInfo,
_ => new ConcurrentQueue <IncomingConnectionRejectionInfo>());
incomingConnectionRejectionInfos.Enqueue(new IncomingConnectionRejectionInfo {
Reason = e.ToString()
});
throw;
}
DocumentsOperationContext documentsOperationContext;
TransactionOperationContext configurationContext;
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out documentsOperationContext))
using (_database.ConfigurationStorage.ContextPool.AllocateOperationContext(out configurationContext))
using (var writer = new BlittableJsonTextWriter(documentsOperationContext, tcpConnectionOptions.Stream))
using (var docTx = documentsOperationContext.OpenReadTransaction())
using (var configTx = configurationContext.OpenReadTransaction())
{
var documentsChangeVector = new DynamicJsonArray();
foreach (var changeVectorEntry in _database.DocumentsStorage.GetDatabaseChangeVector(documentsOperationContext))
{
documentsChangeVector.Add(new DynamicJsonValue
{
[nameof(ChangeVectorEntry.DbId)] = changeVectorEntry.DbId.ToString(),
[nameof(ChangeVectorEntry.Etag)] = changeVectorEntry.Etag
});
}
var indexesChangeVector = new DynamicJsonArray();
var changeVectorAsArray = _database.IndexMetadataPersistence.GetIndexesAndTransformersChangeVector(configTx.InnerTransaction);
foreach (var changeVectorEntry in changeVectorAsArray)
{
indexesChangeVector.Add(new DynamicJsonValue
{
[nameof(ChangeVectorEntry.DbId)] = changeVectorEntry.DbId.ToString(),
[nameof(ChangeVectorEntry.Etag)] = changeVectorEntry.Etag
});
}
var lastEtagFromSrc = _database.DocumentsStorage.GetLastReplicateEtagFrom(documentsOperationContext, getLatestEtagMessage.SourceDatabaseId);
if (_log.IsInfoEnabled)
{
_log.Info($"GetLastEtag response, last etag: {lastEtagFromSrc}");
}
documentsOperationContext.Write(writer, new DynamicJsonValue
{
[nameof(ReplicationMessageReply.Type)] = "Ok",
[nameof(ReplicationMessageReply.MessageType)] = ReplicationMessageType.Heartbeat,
[nameof(ReplicationMessageReply.LastEtagAccepted)] = lastEtagFromSrc,
[nameof(ReplicationMessageReply.LastIndexTransformerEtagAccepted)] = _database.IndexMetadataPersistence.GetLastReplicateEtagFrom(configTx.InnerTransaction, getLatestEtagMessage.SourceDatabaseId),
[nameof(ReplicationMessageReply.DocumentsChangeVector)] = documentsChangeVector,
[nameof(ReplicationMessageReply.IndexTransformerChangeVector)] = indexesChangeVector
});
writer.Flush();
}
var newIncoming = new IncomingReplicationHandler(
tcpConnectionOptions.MultiDocumentParser,
_database,
tcpConnectionOptions.TcpClient,
tcpConnectionOptions.Stream,
getLatestEtagMessage,
this);
newIncoming.Failed += OnIncomingReceiveFailed;
newIncoming.DocumentsReceived += OnIncomingReceiveSucceeded;
if (_log.IsInfoEnabled)
{
_log.Info($"Initialized document replication connection from {connectionInfo.SourceDatabaseName} located at {connectionInfo.SourceUrl}", null);
}
// need to safeguard against two concurrent connection attempts
var newConnection = _incoming.GetOrAdd(newIncoming.ConnectionInfo.SourceDatabaseId, newIncoming);
if (newConnection == newIncoming)
{
newIncoming.Start();
}
else
{
newIncoming.Dispose();
}
}
