private static InternalReplication GetNode(string databaseName, ClusterTopology clusterTopology, string rehab, string mentor, out PromotableTask promotableTask)
{
var url = clusterTopology.GetUrlFromTag(rehab);
var node = new InternalReplication
{
Database = databaseName,
NodeTag = rehab,
Url = url
};
promotableTask = new PromotableTask(rehab, url, databaseName, mentor);
return(node);
}
private static IEnumerable <OngoingTask> CollectBackupTasks(
DatabaseRecord databaseRecord,
DatabaseTopology dbTopology,
ClusterTopology clusterTopology,
ServerStore store)
{
if (dbTopology == null)
{
yield break;
}
if (databaseRecord.PeriodicBackups == null)
{
yield break;
}
if (databaseRecord.PeriodicBackups.Count == 0)
{
yield break;
}
var database = store.DatabasesLandlord.TryGetOrCreateResourceStore(databaseRecord.DatabaseName).Result;
foreach (var backupConfiguration in databaseRecord.PeriodicBackups)
{
var tag = dbTopology.WhoseTaskIsIt(backupConfiguration, store.IsPassive());
var backupDestinations = GetBackupDestinations(backupConfiguration);
var backupStatus = database.PeriodicBackupRunner.GetBackupStatus(backupConfiguration.TaskId);
var nextBackup = database.PeriodicBackupRunner.GetNextBackupDetails(databaseRecord, backupConfiguration, backupStatus);
yield return(new OngoingTaskBackup
{
TaskId = backupConfiguration.TaskId,
BackupType = backupConfiguration.BackupType,
TaskName = backupConfiguration.Name,
TaskState = backupConfiguration.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
LastFullBackup = backupStatus.LastFullBackup,
LastIncrementalBackup = backupStatus.LastIncrementalBackup,
NextBackup = nextBackup,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
BackupDestinations = backupDestinations
});
}
}
private bool TryGetMentorNode(string dbName, DatabaseTopology topology, ClusterTopology clusterTopology, string promotable, out string mentorNode)
{
var url = clusterTopology.GetUrlFromTag(promotable);
var task = new PromotableTask(promotable, url, dbName);
mentorNode = topology.WhoseTaskIsIt(task, _server.IsPassive());
if (mentorNode == null)
{
// We are in passive mode and were kicked out of the cluster.
return(false);
}
return(true);
}
private string GetUrl(string tag, ClusterTopology clusterTopology)
{
string url = null;
if (Server.ServerStore.NodeTag == tag)
{
url = ServerStore.GetNodeHttpServerUrl(HttpContext.Request.GetClientRequestedNodeUrl());
}
if (url == null)
{
url = clusterTopology.GetUrlFromTag(tag);
}
return(url);
}
private string GetUrl(string tag, ClusterTopology clusterTopology)
{
string url = null;
if (Server.ServerStore.NodeTag == tag)
{
url = ServerStore.NodeHttpServerUrl;
}
if (url == null)
{
url = clusterTopology.GetUrlFromTag(tag);
}
return(url);
}
private bool TryGetMentorNode(string dbName, DatabaseTopology topology, ClusterTopology clusterTopology, string promotable, out string mentorNode)
{
var url = clusterTopology.GetUrlFromTag(promotable);
topology.PredefinedMentors.TryGetValue(promotable, out var mentor);
var task = new PromotableTask(promotable, url, dbName, mentor);
mentorNode = topology.WhoseTaskIsIt(_server.Engine.CurrentState, task, null);
if (mentorNode == null)
{
// We are in passive mode and were kicked out of the cluster.
return(false);
}
return(true);
}
private async Task UpdateNodesDirectoryResult(IEnumerable <string> nodes, ClusterTopology clusterTopology, DataDirectoryResult dataDirectoryResult)
{
var tasks = new List <Task <SingleNodeDataDirectoryResult> >();
foreach (var nodeTag in nodes)
{
_serverStore.ServerShutdown.ThrowIfCancellationRequested();
if (nodeTag.Equals(_serverStore.NodeTag, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var serverUrl = clusterTopology.GetUrlFromTag(nodeTag);
if (serverUrl == null)
{
continue;
}
tasks.Add(Task.Run(async() =>
{
var singleNodeResult = await GetSingleNodeDataDirectoryInfo(serverUrl);
return(singleNodeResult);
}, _serverStore.ServerShutdown));
}
await Task.WhenAll(tasks);
foreach (var task in tasks)
{
if (task.IsCompletedSuccessfully == false)
{
continue;
}
var singleNodeResult = await task;
if (singleNodeResult == null)
{
continue;
}
dataDirectoryResult.List.Add(singleNodeResult);
}
}
private IEnumerable <OngoingTask> CollectSubscriptionTasks(TransactionOperationContext context, DatabaseRecord databaseRecord, ClusterTopology clusterTopology)
{
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(databaseRecord.DatabaseName)))
{
var subscriptionState = JsonDeserializationClient.SubscriptionState(keyValue.Value);
var tag = databaseRecord.Topology.WhoseTaskIsIt(subscriptionState, ServerStore.Engine.CurrentState);
yield return(new OngoingTaskSubscription
{
// Supply only needed fields for List View
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
TaskName = subscriptionState.SubscriptionName,
TaskState = subscriptionState.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
TaskId = subscriptionState.SubscriptionId,
Query = subscriptionState.Query
});
}
}
private OngoingTaskReplication GetExternalReplicationInfo(DatabaseTopology dbTopology, ClusterTopology clusterTopology,
ExternalReplication watcher)
{
NodeId responsibale = null;
var tag = dbTopology.WhoseTaskIsIt(watcher, ServerStore.Engine.CurrentState);
if (tag != null)
{
responsibale = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
};
}
(string Url, OngoingTaskConnectionStatus Status)res = (null, OngoingTaskConnectionStatus.None);
if (tag == ServerStore.NodeTag)
{
res = Database.ReplicationLoader.GetExternalReplicationDestination(watcher.TaskId);
}
else
{
res.Status = OngoingTaskConnectionStatus.NotOnThisNode;
}
var taskInfo = new OngoingTaskReplication
{
TaskId = watcher.TaskId,
TaskName = watcher.Name,
ResponsibleNode = responsibale,
DestinationDatabase = watcher.Database,
TaskState = watcher.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
DestinationUrl = res.Url,
TaskConnectionStatus = res.Status,
};
return(taskInfo);
}
private static void AssertCanAddNodeWithTopologyId(ClusterTopology clusterTopology, NodeInfo nodeInfo, string nodeUrl)
{
if (clusterTopology.TopologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. " +
$"Expected topology id: {clusterTopology.TopologyId}, but we get {nodeInfo.TopologyId}");
}
if (nodeInfo.NodeTag != RachisConsensus.InitialTag && clusterTopology.Contains(nodeInfo.NodeTag) == false)
{
// this is fine, since we probably adding back a node that we just removed
return;
}
if (nodeInfo.CurrentState != RachisState.Passive)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " +
$"because it's already in the cluster under tag :{nodeInfo.NodeTag} " +
$"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}");
}
}
private IEnumerable <OngoingTask> CollectEtlTasks(DatabaseRecord databaseRecord, DatabaseTopology dbTopology, ClusterTopology clusterTopology)
{
if (dbTopology == null)
{
yield break;
}
if (databaseRecord.RavenEtls != null)
{
foreach (var ravenEtl in databaseRecord.RavenEtls)
{
var tag = dbTopology.WhoseTaskIsIt(ravenEtl, ServerStore.Engine.CurrentState);
var taskState = GetEtlTaskState(ravenEtl);
if (databaseRecord.RavenConnectionStrings.TryGetValue(ravenEtl.ConnectionStringName, out var connection) == false)
{
throw new InvalidOperationException(
$"Could not find connection string named '{ravenEtl.ConnectionStringName}' in the database record for '{ravenEtl.Name}' ETL");
}
(string Url, OngoingTaskConnectionStatus Status)res = (null, OngoingTaskConnectionStatus.None);
string error = null;
if (tag == ServerStore.NodeTag)
{
foreach (var process in Database.EtlLoader.Processes)
{
if (process is RavenEtl etlProcess)
{
if (etlProcess.Name == ravenEtl.Name)
{
res.Url = etlProcess.Url;
res.Status = OngoingTaskConnectionStatus.Active;
break;
}
}
}
if (res.Status == OngoingTaskConnectionStatus.None)
{
error = $"The raven etl process'{ravenEtl.Name}' was not found.";
}
}
else
{
res.Status = OngoingTaskConnectionStatus.NotOnThisNode;
}
yield return(new OngoingTaskRavenEtlListView()
{
TaskId = ravenEtl.TaskId,
TaskName = ravenEtl.Name,
// TODO arek TaskConnectionStatus =
TaskState = taskState,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
DestinationUrl = res.Url,
TaskConnectionStatus = res.Status,
DestinationDatabase = connection.Database,
ConnectionStringName = ravenEtl.ConnectionStringName,
Error = error
});
}
}
if (databaseRecord.SqlEtls != null)
{
foreach (var sqlEtl in databaseRecord.SqlEtls)
{
var tag = dbTopology.WhoseTaskIsIt(sqlEtl, ServerStore.Engine.CurrentState);
var taskState = GetEtlTaskState(sqlEtl);
if (databaseRecord.SqlConnectionStrings.TryGetValue(sqlEtl.ConnectionStringName, out var sqlConnection) == false)
{
throw new InvalidOperationException(
$"Could not find connection string named '{sqlEtl.ConnectionStringName}' in the database record for '{sqlEtl.Name}' ETL");
}
var(database, server) =
SqlConnectionStringParser.GetDatabaseAndServerFromConnectionString(sqlEtl.FactoryName, sqlConnection.ConnectionString);
yield return(new OngoingTaskSqlEtlListView()
{
TaskId = sqlEtl.TaskId,
TaskName = sqlEtl.Name,
// TODO arek TaskConnectionStatus =
TaskState = taskState,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
DestinationServer = server,
DestinationDatabase = database,
ConnectionStringName = sqlEtl.ConnectionStringName
});
}
}
}
protected async Task WaitForExecutionOnRelevantNodes(JsonOperationContext context, string database, ClusterTopology clusterTopology, List <string> members, long index)
{
await ServerStore.Cluster.WaitForIndexNotification(index); // first let see if we commit this in the leader
if (members.Count == 0)
{
throw new InvalidOperationException("Cannot wait for execution when there are no nodes to execute ON.");
}
var executors = new List <ClusterRequestExecutor>();
try
{
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(ServerStore.ServerShutdown))
{
cts.CancelAfter(ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
var waitingTasks = new List <Task <Exception> >();
List <Exception> exceptions = null;
foreach (var member in members)
{
var url = clusterTopology.GetUrlFromTag(member);
var executor = ClusterRequestExecutor.CreateForSingleNode(url, ServerStore.Server.Certificate.Certificate);
executors.Add(executor);
waitingTasks.Add(ExecuteTask(executor, member, cts.Token));
}
while (waitingTasks.Count > 0)
{
var task = await Task.WhenAny(waitingTasks);
waitingTasks.Remove(task);
if (task.Result == null)
{
continue;
}
var exception = task.Result.ExtractSingleInnerException();
if (exceptions == null)
{
exceptions = new List <Exception>();
}
exceptions.Add(exception);
}
if (exceptions != null)
{
var allTimeouts = true;
foreach (var exception in exceptions)
{
if (exception is OperationCanceledException)
{
continue;
}
allTimeouts = false;
}
var aggregateException = new AggregateException(exceptions);
if (allTimeouts)
{
throw new TimeoutException($"Waited too long for the raft command (number {index}) to be executed on any of the relevant nodes to this command.", aggregateException);
}
throw new InvalidDataException($"The database '{database}' was created but is not accessible, because all of the nodes on which this database was supposed to reside on, threw an exception.", aggregateException);
}
}
}
finally
{
foreach (var executor in executors)
{
executor.Dispose();
}
}
async Task <Exception> ExecuteTask(RequestExecutor executor, string nodeTag, CancellationToken token)
{
try
{
await executor.ExecuteAsync(new WaitForRaftIndexCommand(index), context, token : token);
return(null);
}
catch (RavenException re) when(re.InnerException is HttpRequestException)
{
// we want to throw for self-checks
if (nodeTag == ServerStore.NodeTag)
{
return(re);
}
// ignore - we are ok when connection with a node cannot be established (test: AddDatabaseOnDisconnectedNode)
return(null);
}
catch (Exception e)
{
return(e);
}
}
}
private static IEnumerable <OngoingTask> CollectEtlTasks(DatabaseRecord databaseRecord, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store)
{
if (dbTopology == null)
{
yield break;
}
if (databaseRecord.RavenEtls != null)
{
foreach (var ravenEtl in databaseRecord.RavenEtls)
{
var tag = dbTopology.WhoseTaskIsIt(ravenEtl, store.IsPassive());
var taskState = OngoingTaskState.Enabled;
if (ravenEtl.Disabled || ravenEtl.Transforms.All(x => x.Disabled))
{
taskState = OngoingTaskState.Disabled;
}
else if (ravenEtl.Transforms.Any(x => x.Disabled))
{
taskState = OngoingTaskState.PartiallyEnabled;
}
if (databaseRecord.RavenConnectionStrings.TryGetValue(ravenEtl.ConnectionStringName, out var connection) == false)
{
throw new InvalidOperationException(
$"Could not find connection string named '{ravenEtl.ConnectionStringName}' in the database record for '{ravenEtl.Name}' ETL");
}
yield return(new OngoingTaskRavenEtl
{
TaskId = ravenEtl.TaskId,
TaskName = ravenEtl.Name,
// TODO arek TaskConnectionStatus =
TaskState = taskState,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
DestinationUrl = connection.Url,
DestinationDatabase = connection.Database
});
}
}
if (databaseRecord.SqlEtls != null)
{
foreach (var sqlEtl in databaseRecord.SqlEtls)
{
var tag = dbTopology.WhoseTaskIsIt(sqlEtl, store.IsPassive());
var taskState = OngoingTaskState.Enabled;
if (sqlEtl.Disabled || sqlEtl.Transforms.All(x => x.Disabled))
{
taskState = OngoingTaskState.Disabled;
}
else if (sqlEtl.Transforms.Any(x => x.Disabled))
{
taskState = OngoingTaskState.PartiallyEnabled;
}
if (databaseRecord.SqlConnectionStrings.TryGetValue(sqlEtl.ConnectionStringName, out var sqlConnection) == false)
{
throw new InvalidOperationException(
$"Could not find connection string named '{sqlEtl.ConnectionStringName}' in the database record for '{sqlEtl.Name}' ETL");
}
var(database, server) =
SqlConnectionStringParser.GetDatabaseAndServerFromConnectionString(sqlEtl.FactoryName, sqlConnection.ConnectionString);
yield return(new OngoingTaskSqlEtl
{
TaskId = sqlEtl.TaskId,
TaskName = sqlEtl.Name,
// TODO arek TaskConnectionStatus =
TaskState = taskState,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
DestinationServer = server,
DestinationDatabase = database
});
}
}
}