private static async Task UnlikelyWaitForDatabaseToLoad(RequestHandlerContext context, Task <DocumentDatabase> database,
DatabasesLandlord databasesLandlord, StringSegment databaseName)
{
var time = databasesLandlord.DatabaseLoadTimeout;
await Task.WhenAny(database, Task.Delay(time));
if (database.IsCompleted == false)
{
if (databasesLandlord.InitLog.TryGetValue(databaseName.Value, out var initLogQueue))
{
var sb = new StringBuilder();
foreach (var logline in initLogQueue)
{
sb.AppendLine(logline);
}
ThrowDatabaseLoadTimeoutWithLog(databaseName, databasesLandlord.DatabaseLoadTimeout, sb.ToString());
}
ThrowDatabaseLoadTimeout(databaseName, databasesLandlord.DatabaseLoadTimeout);
}
context.Database = await database;
if (context.Database == null)
{
DatabaseDoesNotExistException.Throw(databaseName.Value);
}
}
public Task CreateDatabase(RequestHandlerContext context)
{
var databaseName = context.RouteMatch.GetCapture();
if (context.RavenServer.ServerStore.IsPassive())
{
throw new NodeIsPassiveException($"Can't perform actions on the database '{databaseName}' while the node is passive.");
}
var databasesLandlord = context.RavenServer.ServerStore.DatabasesLandlord;
var database = databasesLandlord.TryGetOrCreateResourceStore(databaseName);
if (database.IsCompletedSuccessfully)
{
context.Database = database.Result;
if (context.Database == null)
{
DatabaseDoesNotExistException.Throw(databaseName.Value);
}
return(context.Database?.DatabaseShutdown.IsCancellationRequested == false
? Task.CompletedTask
: UnlikelyWaitForDatabaseToUnload(context, context.Database, databasesLandlord, databaseName));
}
return(UnlikelyWaitForDatabaseToLoad(context, database, databasesLandlord, databaseName));
}
public override string UpdateDatabaseRecord(DatabaseRecord record, long etag)
{
var deletionInProgressStatus = HardDelete ? DeletionInProgressStatus.HardDelete
: DeletionInProgressStatus.SoftDelete;
if (record.DeletionInProgress == null)
{
record.DeletionInProgress = new Dictionary <string, DeletionInProgressStatus>();
}
if (FromNodes != null && FromNodes.Length > 0)
{
foreach (var node in FromNodes)
{
if (record.Topology.RelevantFor(node) == false)
{
DatabaseDoesNotExistException.ThrowWithMessage(record.DatabaseName, $"Request to delete database from node '{node}' failed.");
}
// rehabs will be removed only once the replication sent all the documents to the mentor
if (record.Topology.Rehabs.Contains(node) == false)
{
record.Topology.RemoveFromTopology(node);
}
if (UpdateReplicationFactor)
{
record.Topology.ReplicationFactor--;
}
if (ClusterNodes.Contains(node))
{
record.DeletionInProgress[node] = deletionInProgressStatus;
}
}
}
else
{
var allNodes = record.Topology.Members.Select(m => m)
.Concat(record.Topology.Promotables.Select(p => p))
.Concat(record.Topology.Rehabs.Select(r => r));
foreach (var node in allNodes)
{
if (ClusterNodes.Contains(node))
{
record.DeletionInProgress[node] = deletionInProgressStatus;
}
}
record.Topology = new DatabaseTopology
{
Stamp = record.Topology.Stamp,
ReplicationFactor = 0
};
}
record.Topology.Stamp.Index = etag;
return(null);
}
private async Task <bool> DispatchDatabaseTcpConnection(TcpConnectionOptions tcp, TcpConnectionHeaderMessage header)
{
var databaseLoadingTask = ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(header.DatabaseName);
if (databaseLoadingTask == null)
{
DatabaseDoesNotExistException.Throw(header.DatabaseName);
return(true);
}
var databaseLoadTimeout = ServerStore.DatabasesLandlord.DatabaseLoadTimeout;
if (databaseLoadingTask.IsCompleted == false)
{
var resultingTask = await Task.WhenAny(databaseLoadingTask, Task.Delay(databaseLoadTimeout));
if (resultingTask != databaseLoadingTask)
{
ThrowTimeoutOnDatabaseLoad(header);
}
}
tcp.DocumentDatabase = await databaseLoadingTask;
if (tcp.DocumentDatabase == null)
{
DatabaseDoesNotExistException.Throw(header.DatabaseName);
}
Debug.Assert(tcp.DocumentDatabase != null);
if (tcp.DocumentDatabase.DatabaseShutdown.IsCancellationRequested)
{
ThrowDatabaseShutdown(tcp.DocumentDatabase);
}
tcp.DocumentDatabase.RunningTcpConnections.Add(tcp);
switch (header.Operation)
{
case TcpConnectionHeaderMessage.OperationTypes.Subscription:
SubscriptionConnection.SendSubscriptionDocuments(tcp);
break;
case TcpConnectionHeaderMessage.OperationTypes.Replication:
var documentReplicationLoader = tcp.DocumentDatabase.ReplicationLoader;
documentReplicationLoader.AcceptIncomingConnection(tcp);
break;
default:
throw new InvalidOperationException("Unknown operation for TCP " + header.Operation);
}
//since the responses to TCP connections mostly continue to run
//beyond this point, no sense to dispose the connection now, so set it to null.
//this way the responders are responsible to dispose the connection and the context
// ReSharper disable once RedundantAssignment
tcp = null;
return(false);
}
public Task CreateDatabase(RequestHandlerContext context)
{
var databaseName = context.RouteMatch.GetCapture();
// todo: think if we need to pass this check to the landlord
if (context.RavenServer.ServerStore.IsPassive())
{
throw new NodeIsPassiveException($"Can't perform actions on the database '{databaseName}' while the node is passive.");
}
if (context.RavenServer.ServerStore.IdleDatabases.TryGetValue(databaseName.Value, out var replicationsDictionary))
{
if (context.HttpContext.Request.Query.TryGetValue("from-outgoing", out var dbId) && context.HttpContext.Request.Query.TryGetValue("etag", out var replicationEtag))
{
var hasChanges = false;
var etag = Convert.ToInt64(replicationEtag);
if (replicationsDictionary.TryGetValue(dbId, out var storedEtag))
{
if (storedEtag < etag)
{
hasChanges = true;
}
}
else
{
if (etag > 0)
{
hasChanges = true;
}
}
if (hasChanges == false)
{
throw new DatabaseIdleException($"Replication attempt doesn't have changes to database {databaseName.Value}, which is currently idle.");
}
}
}
var databasesLandlord = context.RavenServer.ServerStore.DatabasesLandlord;
var database = databasesLandlord.TryGetOrCreateResourceStore(databaseName);
if (database.IsCompletedSuccessfully)
{
context.Database = database.Result;
if (context.Database == null)
{
DatabaseDoesNotExistException.Throw(databaseName.Value);
}
return(context.Database?.DatabaseShutdown.IsCancellationRequested == false
? Task.CompletedTask
: UnlikelyWaitForDatabaseToUnload(context, context.Database, databasesLandlord, databaseName));
}
return(UnlikelyWaitForDatabaseToLoad(context, database, databasesLandlord, databaseName));
}
public MigrationService(MigrationRunner runner, IDocumentStore store)
{
_runner = runner;
_store = store;
if (string.IsNullOrEmpty(_store.Database))
{
throw DatabaseDoesNotExistException.CreateWithMessage(_store.Database, "No database specified");
}
}
public void Initialize(InitializeOptions options = InitializeOptions.None)
{
try
{
NotificationCenter.Initialize(this);
DocumentsStorage.Initialize((options & InitializeOptions.GenerateNewDatabaseId) == InitializeOptions.GenerateNewDatabaseId);
TxMerger.Start();
ConfigurationStorage.Initialize();
if ((options & InitializeOptions.SkipLoadingDatabaseRecord) == InitializeOptions.SkipLoadingDatabaseRecord)
{
return;
}
long index;
DatabaseRecord record;
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
record = _serverStore.Cluster.ReadDatabase(context, Name, out index);
if (record == null)
{
DatabaseDoesNotExistException.Throw(Name);
}
PeriodicBackupRunner = new PeriodicBackupRunner(this, _serverStore);
_indexStoreTask = IndexStore.InitializeAsync(record);
ReplicationLoader?.Initialize(record);
EtlLoader.Initialize(record);
DocumentTombstoneCleaner.Start();
try
{
_indexStoreTask.Wait(DatabaseShutdown);
}
finally
{
_indexStoreTask = null;
}
SubscriptionStorage.Initialize();
NotifyFeaturesAboutStateChange(record, index);
}
catch (Exception)
{
Dispose();
throw;
}
}
private static async Task UnlikelyWaitForDatabaseToLoad(RequestHandlerContext context, Task <DocumentDatabase> database,
DatabasesLandlord databasesLandlord, StringSegment databaseName)
{
var time = databasesLandlord.DatabaseLoadTimeout;
await Task.WhenAny(database, Task.Delay(time));
if (database.IsCompleted == false)
{
ThrowDatabaseLoadTimeout(databaseName, databasesLandlord.DatabaseLoadTimeout);
}
context.Database = await database;
if (context.Database == null)
{
DatabaseDoesNotExistException.Throw(databaseName);
}
}
public Task CreateDatabase(RequestHandlerContext context)
{
var databaseName = context.RouteMatch.GetCapture();
var databasesLandlord = context.RavenServer.ServerStore.DatabasesLandlord;
var database = databasesLandlord.TryGetOrCreateResourceStore(databaseName);
if (database.IsCompletedSuccessfully)
{
context.Database = database.Result;
if (context.Database == null)
{
DatabaseDoesNotExistException.Throw(databaseName);
}
return(context.Database?.DatabaseShutdown.IsCancellationRequested == false
? Task.CompletedTask
: UnlikelyWaitForDatabaseToUnload(context, context.Database, databasesLandlord, databaseName));
}
return(UnlikelyWaitForDatabaseToLoad(context, database, databasesLandlord, databaseName));
}
private void ReplicateToDestination()
{
try
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiate);
NativeMemory.EnsureRegistered();
if (_log.IsInfoEnabled)
{
_log.Info($"Will replicate to {Destination.FromString()} via {_connectionInfo.Url}");
}
using (_parent._server.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var record = _parent.LoadDatabaseRecord();
if (record == null)
{
throw new InvalidOperationException($"The database record for {_parent.Database.Name} does not exist?!");
}
if (record.Encrypted && Destination.Url.StartsWith("https:", StringComparison.OrdinalIgnoreCase) == false)
{
throw new InvalidOperationException(
$"{record.DatabaseName} is encrypted, and require HTTPS for replication, but had endpoint with url {Destination.Url} to database {Destination.Database}");
}
}
var task = TcpUtils.ConnectSocketAsync(_connectionInfo, _parent._server.Engine.TcpConnectionTimeout, _log);
task.Wait(CancellationToken);
using (Interlocked.Exchange(ref _tcpClient, task.Result))
{
var wrapSsl = TcpUtils.WrapStreamWithSslAsync(_tcpClient, _connectionInfo, _parent._server.Server.Certificate.Certificate, _parent._server.Engine.TcpConnectionTimeout);
wrapSsl.Wait(CancellationToken);
using (_stream = wrapSsl.Result) // note that _stream is being disposed by the interruptible read
using (_interruptibleRead = new InterruptibleRead(_database.DocumentsStorage.ContextPool, _stream))
using (_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance())
{
var documentSender = new ReplicationDocumentSender(_stream, this, _log);
WriteHeaderToRemotePeer();
//handle initial response to last etag and staff
try
{
var response = HandleServerResponse(getFullResponse: true);
switch (response.ReplyType)
{
//The first time we start replication we need to register the destination current CV
case ReplicationMessageReply.ReplyType.Ok:
LastAcceptedChangeVector = response.Reply.DatabaseChangeVector;
break;
case ReplicationMessageReply.ReplyType.Error:
var exception = new InvalidOperationException(response.Reply.Exception);
if (response.Reply.Exception.Contains(nameof(DatabaseDoesNotExistException)) ||
response.Reply.Exception.Contains(nameof(DatabaseNotRelevantException)))
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, "Database does not exist");
DatabaseDoesNotExistException.ThrowWithMessageAndException(Destination.Database, response.Reply.Message, exception);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, $"Got error: {response.Reply.Exception}");
throw exception;
}
}
catch (DatabaseDoesNotExistException e)
{
var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} " +
"on the other end. ";
if (_external)
{
msg += "In order for the replication to work, a database with the same name needs to be created at the destination";
}
var young = (DateTime.UtcNow - _startedAt).TotalSeconds < 30;
if (young)
{
msg += "This can happen if the other node wasn't yet notified about being assigned this database and should be resolved shortly.";
}
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
// won't add an alert on young connections
// because it may take a few seconds for the other side to be notified by
// the cluster that it has this db.
if (young == false)
{
AddAlertOnFailureToReachOtherSide(msg, e);
}
throw;
}
catch (OperationCanceledException e)
{
const string msg = "Got operation canceled notification while opening outgoing replication channel. " +
"Aborting and closing the channel.";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
throw;
}
catch (Exception e)
{
var msg = $"{OutgoingReplicationThreadName} got an unexpected exception during initial handshake";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
AddAlertOnFailureToReachOtherSide(msg, e);
throw;
}
DateTime nextReplicateAt = default(DateTime);
while (_cts.IsCancellationRequested == false)
{
while (_database.Time.GetUtcNow() > nextReplicateAt)
{
if (_parent.DebugWaitAndRunReplicationOnce != null)
{
_parent.DebugWaitAndRunReplicationOnce.Wait(_cts.Token);
_parent.DebugWaitAndRunReplicationOnce.Reset();
}
var sp = Stopwatch.StartNew();
var stats = _lastStats = new OutgoingReplicationStatsAggregator(_parent.GetNextReplicationStatsId(), _lastStats);
AddReplicationPerformance(stats);
AddReplicationPulse(ReplicationPulseDirection.OutgoingBegin);
try
{
using (var scope = stats.CreateScope())
{
try
{
if (Destination is InternalReplication dest)
{
_parent.EnsureNotDeleted(dest.NodeTag);
}
var didWork = documentSender.ExecuteReplicationOnce(scope, ref nextReplicateAt);
if (didWork == false)
{
break;
}
if (Destination is ExternalReplication externalReplication)
{
var taskId = externalReplication.TaskId;
UpdateExternalReplicationInfo(taskId);
}
DocumentsSend?.Invoke(this);
if (sp.ElapsedMilliseconds > 60 * 1000)
{
_waitForChanges.Set();
break;
}
}
catch (OperationCanceledException)
{
// cancellation is not an actual error,
// it is a "notification" that we need to cancel current operation
const string msg = "Operation was canceled.";
AddReplicationPulse(ReplicationPulseDirection.OutgoingError, msg);
throw;
}
catch (Exception e)
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingError, e.Message);
scope.AddError(e);
throw;
}
}
}
finally
{
stats.Complete();
AddReplicationPulse(ReplicationPulseDirection.OutgoingEnd);
}
}
//if this returns false, this means either timeout or canceled token is activated
while (WaitForChanges(_parent.MinimalHeartbeatInterval, _cts.Token) == false)
{
//If we got cancelled we need to break right away
if (_cts.IsCancellationRequested)
{
break;
}
// open tx
// read current change vector compare to last sent
// if okay, send cv
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (var tx = ctx.OpenReadTransaction())
{
var etag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
if (etag == _lastSentDocumentEtag)
{
SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(ctx));
_parent.CompleteDeletionIfNeeded();
}
else if (nextReplicateAt > DateTime.UtcNow)
{
SendHeartbeat(null);
}
else
{
//Send a heartbeat first so we will get an updated CV of the destination
var currentChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
SendHeartbeat(null);
//If our previous CV is already merged to the destination wait a bit more
if (ChangeVectorUtils.GetConflictStatus(LastAcceptedChangeVector, currentChangeVector) ==
ConflictStatus.AlreadyMerged)
{
continue;
}
// we have updates that we need to send to the other side
// let's do that..
// this can happen if we got replication from another node
// that we need to send to it. Note that we typically
// will wait for the other node to send the data directly to
// our destination, but if it doesn't, we'll step in.
// In this case, we try to limit congestion in the network and
// only send updates that we have gotten from someone else after
// a certain time, to let the other side tell us that it already
// got it. Note that this is merely an optimization to reduce network
// traffic. It is fine to have the same data come from different sources.
break;
}
}
}
_waitForChanges.Reset();
}
}
}
}
catch (AggregateException e)
{
if (e.InnerExceptions.Count == 1)
{
if (e.InnerException is OperationCanceledException oce)
{
HandleOperationCancelException(oce);
}
if (e.InnerException is IOException ioe)
{
HandleIOException(ioe);
}
}
HandleException(e);
}
catch (OperationCanceledException e)
{
HandleOperationCancelException(e);
}
catch (IOException e)
{
HandleIOException(e);
}
catch (Exception e)
{
HandleException(e);
}
void HandleOperationCancelException(OperationCanceledException e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Operation canceled on replication thread ({FromToString}). " +
$"This is not necessary due to an issue. Stopped the thread.");
}
if (_cts.IsCancellationRequested == false)
{
Failed?.Invoke(this, e);
}
}
void HandleIOException(IOException e)
{
if (_log.IsInfoEnabled)
{
if (e.InnerException is SocketException)
{
_log.Info($"SocketException was thrown from the connection to remote node ({FromToString}). " +
$"This might mean that the remote node is done or there is a network issue.", e);
}
else
{
_log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e);
}
}
Failed?.Invoke(this, e);
}
void HandleException(Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Unexpected exception occurred on replication thread ({FromToString}). " +
$"Replication stopped (will be retried later).", e);
}
Failed?.Invoke(this, e);
}
}
public void Initialize(InitializeOptions options = InitializeOptions.None)
{
try
{
_addToInitLog("Initializing NotificationCenter");
NotificationCenter.Initialize(this);
_addToInitLog("Initializing DocumentStorage");
DocumentsStorage.Initialize((options & InitializeOptions.GenerateNewDatabaseId) == InitializeOptions.GenerateNewDatabaseId);
_addToInitLog("Starting Transaction Merger");
TxMerger.Start();
_addToInitLog("Initializing ConfigurationStorage");
ConfigurationStorage.Initialize();
if ((options & InitializeOptions.SkipLoadingDatabaseRecord) == InitializeOptions.SkipLoadingDatabaseRecord)
{
return;
}
_addToInitLog("Loading Database");
long index;
DatabaseRecord record;
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
record = _serverStore.Cluster.ReadDatabase(context, Name, out index);
if (record == null)
{
DatabaseDoesNotExistException.Throw(Name);
}
PeriodicBackupRunner = new PeriodicBackupRunner(this, _serverStore);
_addToInitLog("Initializing IndexStore (async)");
_indexStoreTask = IndexStore.InitializeAsync(record);
_addToInitLog("Initializing Replication");
ReplicationLoader?.Initialize(record);
_addToInitLog("Initializing ETL");
EtlLoader.Initialize(record);
DocumentTombstoneCleaner.Start();
try
{
_indexStoreTask.Wait(DatabaseShutdown);
}
finally
{
_addToInitLog("Initializing IndexStore completed");
_indexStoreTask = null;
}
SubscriptionStorage.Initialize();
_addToInitLog("Initializing SubscriptionStorage completed");
TaskExecutor.Execute((state) =>
{
try
{
NotifyFeaturesAboutStateChange(record, index);
}
catch
{
// We ignore the exception since it was caught in the function itself
}
}, null);
}
catch (Exception)
{
Dispose();
throw;
}
}
public Task CreateDatabase(RequestHandlerContext context)
{
var databaseName = context.RouteMatch.GetCapture();
// todo: think if we need to pass this check to the landlord
if (context.RavenServer.ServerStore.IsPassive())
{
throw new NodeIsPassiveException($"Can't perform actions on the database '{databaseName}' while the node is passive.");
}
if (context.RavenServer.ServerStore.IdleDatabases.TryGetValue(databaseName.Value, out var replicationsDictionary))
{
if (context.HttpContext.Request.Query.TryGetValue("from-outgoing", out var dbId) && context.HttpContext.Request.Query.TryGetValue("etag", out var replicationEtag))
{
var hasChanges = false;
var etag = Convert.ToInt64(replicationEtag);
if (replicationsDictionary.TryGetValue(dbId, out var storedEtag))
{
if (storedEtag < etag)
{
hasChanges = true;
}
}
else
{
if (etag > 0)
{
hasChanges = true;
}
}
if (hasChanges == false)
{
throw new DatabaseIdleException($"Replication attempt doesn't have changes to database {databaseName.Value}, which is currently idle.");
}
}
}
var databasesLandlord = context.RavenServer.ServerStore.DatabasesLandlord;
var database = databasesLandlord.TryGetOrCreateResourceStore(databaseName);
if (database.IsCompletedSuccessfully)
{
context.Database = database.Result;
if (context.Database == null)
{
DatabaseDoesNotExistException.Throw(databaseName.Value);
}
// ReSharper disable once PossibleNullReferenceException
if (context.Database.DatabaseShutdownCompleted.IsSet)
{
using (context.RavenServer.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (ctx.OpenReadTransaction())
{
if (context.RavenServer.ServerStore.Cluster.DatabaseExists(ctx, databaseName.Value))
{
// db got disabled during loading
throw new DatabaseDisabledException($"Cannot complete the request, because {databaseName.Value} has been disabled.");
}
}
// db got deleted during loading
DatabaseDoesNotExistException.ThrowWithMessage(databaseName.Value, "Cannot complete the request.");
}
return(context.Database.DatabaseShutdown.IsCancellationRequested == false
? Task.CompletedTask
: UnlikelyWaitForDatabaseToUnload(context, context.Database, databasesLandlord, databaseName));
}
return(UnlikelyWaitForDatabaseToLoad(context, database, databasesLandlord, databaseName));
}
public override string UpdateDatabaseRecord(DatabaseRecord record, long etag)
{
var deletionInProgressStatus = HardDelete ? DeletionInProgressStatus.HardDelete
: DeletionInProgressStatus.SoftDelete;
if (MentorChangeVector != null)
{
if (record.DeletionInProgressChangeVector == null)
{
record.DeletionInProgressChangeVector = new Dictionary <string, string>();
}
foreach (var entry in MentorChangeVector)
{
var key = entry.Key;
var value = entry.Value;
record.DeletionInProgressChangeVector[key] = value;
}
}
if (record.DeletionInProgress == null)
{
record.DeletionInProgress = new Dictionary <string, DeletionInProgressStatus>();
}
if (FromNodes != null && FromNodes.Length > 0)
{
foreach (var node in FromNodes)
{
if (record.Topology.RelevantFor(node) == false)
{
DatabaseDoesNotExistException.ThrowWithMessage(record.DatabaseName, $"Request to delete database from node '{node}' failed.");
}
if (UpdateReplicationFactor)
{
record.Topology.RemoveFromTopology(node);
record.Topology.ReplicationFactor--;
}
if (ClusterNodes.Contains(node))
{
record.DeletionInProgress[node] = deletionInProgressStatus;
}
}
}
else
{
var allNodes = record.Topology.Members.Select(m => m)
.Concat(record.Topology.Promotables.Select(p => p))
.Concat(record.Topology.Rehabs.Select(r => r));
foreach (var node in allNodes)
{
if (ClusterNodes.Contains(node))
{
record.DeletionInProgress[node] = deletionInProgressStatus;
}
}
record.Topology.ReplicationFactor = 0;
record.Topology = new DatabaseTopology
{
Stamp = record.Topology.Stamp
};
}
return(null);
}
