private static InternalReplication GetNode(string databaseName, ClusterTopology clusterTopology, string rehab, string mentor, out PromotableTask promotableTask) { var url = clusterTopology.GetUrlFromTag(rehab); var node = new InternalReplication { Database = databaseName, NodeTag = rehab, Url = url }; promotableTask = new PromotableTask(rehab, url, databaseName, mentor); return(node); }
private static IEnumerable <OngoingTask> CollectBackupTasks( DatabaseRecord databaseRecord, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store) { if (dbTopology == null) { yield break; } if (databaseRecord.PeriodicBackups == null) { yield break; } if (databaseRecord.PeriodicBackups.Count == 0) { yield break; } var database = store.DatabasesLandlord.TryGetOrCreateResourceStore(databaseRecord.DatabaseName).Result; foreach (var backupConfiguration in databaseRecord.PeriodicBackups) { var tag = dbTopology.WhoseTaskIsIt(backupConfiguration, store.IsPassive()); var backupDestinations = GetBackupDestinations(backupConfiguration); var backupStatus = database.PeriodicBackupRunner.GetBackupStatus(backupConfiguration.TaskId); var nextBackup = database.PeriodicBackupRunner.GetNextBackupDetails(databaseRecord, backupConfiguration, backupStatus); yield return(new OngoingTaskBackup { TaskId = backupConfiguration.TaskId, BackupType = backupConfiguration.BackupType, TaskName = backupConfiguration.Name, TaskState = backupConfiguration.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled, LastFullBackup = backupStatus.LastFullBackup, LastIncrementalBackup = backupStatus.LastIncrementalBackup, NextBackup = nextBackup, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, BackupDestinations = backupDestinations }); } }
private bool TryGetMentorNode(string dbName, DatabaseTopology topology, ClusterTopology clusterTopology, string promotable, out string mentorNode) { var url = clusterTopology.GetUrlFromTag(promotable); var task = new PromotableTask(promotable, url, dbName); mentorNode = topology.WhoseTaskIsIt(task, _server.IsPassive()); if (mentorNode == null) { // We are in passive mode and were kicked out of the cluster. return(false); } return(true); }
private string GetUrl(string tag, ClusterTopology clusterTopology) { string url = null; if (Server.ServerStore.NodeTag == tag) { url = ServerStore.GetNodeHttpServerUrl(HttpContext.Request.GetClientRequestedNodeUrl()); } if (url == null) { url = clusterTopology.GetUrlFromTag(tag); } return(url); }
private string GetUrl(string tag, ClusterTopology clusterTopology) { string url = null; if (Server.ServerStore.NodeTag == tag) { url = ServerStore.NodeHttpServerUrl; } if (url == null) { url = clusterTopology.GetUrlFromTag(tag); } return(url); }
private bool TryGetMentorNode(string dbName, DatabaseTopology topology, ClusterTopology clusterTopology, string promotable, out string mentorNode) { var url = clusterTopology.GetUrlFromTag(promotable); topology.PredefinedMentors.TryGetValue(promotable, out var mentor); var task = new PromotableTask(promotable, url, dbName, mentor); mentorNode = topology.WhoseTaskIsIt(_server.Engine.CurrentState, task, null); if (mentorNode == null) { // We are in passive mode and were kicked out of the cluster. return(false); } return(true); }
private async Task UpdateNodesDirectoryResult(IEnumerable <string> nodes, ClusterTopology clusterTopology, DataDirectoryResult dataDirectoryResult) { var tasks = new List <Task <SingleNodeDataDirectoryResult> >(); foreach (var nodeTag in nodes) { _serverStore.ServerShutdown.ThrowIfCancellationRequested(); if (nodeTag.Equals(_serverStore.NodeTag, StringComparison.OrdinalIgnoreCase)) { continue; } var serverUrl = clusterTopology.GetUrlFromTag(nodeTag); if (serverUrl == null) { continue; } tasks.Add(Task.Run(async() => { var singleNodeResult = await GetSingleNodeDataDirectoryInfo(serverUrl); return(singleNodeResult); }, _serverStore.ServerShutdown)); } await Task.WhenAll(tasks); foreach (var task in tasks) { if (task.IsCompletedSuccessfully == false) { continue; } var singleNodeResult = await task; if (singleNodeResult == null) { continue; } dataDirectoryResult.List.Add(singleNodeResult); } }
private IEnumerable <OngoingTask> CollectSubscriptionTasks(TransactionOperationContext context, DatabaseRecord databaseRecord, ClusterTopology clusterTopology) { foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(databaseRecord.DatabaseName))) { var subscriptionState = JsonDeserializationClient.SubscriptionState(keyValue.Value); var tag = databaseRecord.Topology.WhoseTaskIsIt(subscriptionState, ServerStore.Engine.CurrentState); yield return(new OngoingTaskSubscription { // Supply only needed fields for List View ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, TaskName = subscriptionState.SubscriptionName, TaskState = subscriptionState.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled, TaskId = subscriptionState.SubscriptionId, Query = subscriptionState.Query }); } }
private OngoingTaskReplication GetExternalReplicationInfo(DatabaseTopology dbTopology, ClusterTopology clusterTopology, ExternalReplication watcher) { NodeId responsibale = null; var tag = dbTopology.WhoseTaskIsIt(watcher, ServerStore.Engine.CurrentState); if (tag != null) { responsibale = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }; } (string Url, OngoingTaskConnectionStatus Status)res = (null, OngoingTaskConnectionStatus.None); if (tag == ServerStore.NodeTag) { res = Database.ReplicationLoader.GetExternalReplicationDestination(watcher.TaskId); } else { res.Status = OngoingTaskConnectionStatus.NotOnThisNode; } var taskInfo = new OngoingTaskReplication { TaskId = watcher.TaskId, TaskName = watcher.Name, ResponsibleNode = responsibale, DestinationDatabase = watcher.Database, TaskState = watcher.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled, DestinationUrl = res.Url, TaskConnectionStatus = res.Status, }; return(taskInfo); }
private static void AssertCanAddNodeWithTopologyId(ClusterTopology clusterTopology, NodeInfo nodeInfo, string nodeUrl) { if (clusterTopology.TopologyId != nodeInfo.TopologyId) { throw new TopologyMismatchException( $"Adding a new node to cluster failed. The new node is already in another cluster. " + $"Expected topology id: {clusterTopology.TopologyId}, but we get {nodeInfo.TopologyId}"); } if (nodeInfo.NodeTag != RachisConsensus.InitialTag && clusterTopology.Contains(nodeInfo.NodeTag) == false) { // this is fine, since we probably adding back a node that we just removed return; } if (nodeInfo.CurrentState != RachisState.Passive) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " + $"because it's already in the cluster under tag :{nodeInfo.NodeTag} " + $"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}"); } }
private IEnumerable <OngoingTask> CollectEtlTasks(DatabaseRecord databaseRecord, DatabaseTopology dbTopology, ClusterTopology clusterTopology) { if (dbTopology == null) { yield break; } if (databaseRecord.RavenEtls != null) { foreach (var ravenEtl in databaseRecord.RavenEtls) { var tag = dbTopology.WhoseTaskIsIt(ravenEtl, ServerStore.Engine.CurrentState); var taskState = GetEtlTaskState(ravenEtl); if (databaseRecord.RavenConnectionStrings.TryGetValue(ravenEtl.ConnectionStringName, out var connection) == false) { throw new InvalidOperationException( $"Could not find connection string named '{ravenEtl.ConnectionStringName}' in the database record for '{ravenEtl.Name}' ETL"); } (string Url, OngoingTaskConnectionStatus Status)res = (null, OngoingTaskConnectionStatus.None); string error = null; if (tag == ServerStore.NodeTag) { foreach (var process in Database.EtlLoader.Processes) { if (process is RavenEtl etlProcess) { if (etlProcess.Name == ravenEtl.Name) { res.Url = etlProcess.Url; res.Status = OngoingTaskConnectionStatus.Active; break; } } } if (res.Status == OngoingTaskConnectionStatus.None) { error = $"The raven etl process'{ravenEtl.Name}' was not found."; } } else { res.Status = OngoingTaskConnectionStatus.NotOnThisNode; } yield return(new OngoingTaskRavenEtlListView() { TaskId = ravenEtl.TaskId, TaskName = ravenEtl.Name, // TODO arek TaskConnectionStatus = TaskState = taskState, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, DestinationUrl = res.Url, TaskConnectionStatus = res.Status, DestinationDatabase = connection.Database, ConnectionStringName = ravenEtl.ConnectionStringName, Error = error }); } } if (databaseRecord.SqlEtls != null) { foreach (var sqlEtl in databaseRecord.SqlEtls) { var tag = dbTopology.WhoseTaskIsIt(sqlEtl, ServerStore.Engine.CurrentState); var taskState = GetEtlTaskState(sqlEtl); if (databaseRecord.SqlConnectionStrings.TryGetValue(sqlEtl.ConnectionStringName, out var sqlConnection) == false) { throw new InvalidOperationException( $"Could not find connection string named '{sqlEtl.ConnectionStringName}' in the database record for '{sqlEtl.Name}' ETL"); } var(database, server) = SqlConnectionStringParser.GetDatabaseAndServerFromConnectionString(sqlEtl.FactoryName, sqlConnection.ConnectionString); yield return(new OngoingTaskSqlEtlListView() { TaskId = sqlEtl.TaskId, TaskName = sqlEtl.Name, // TODO arek TaskConnectionStatus = TaskState = taskState, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, DestinationServer = server, DestinationDatabase = database, ConnectionStringName = sqlEtl.ConnectionStringName }); } } }
protected async Task WaitForExecutionOnRelevantNodes(JsonOperationContext context, string database, ClusterTopology clusterTopology, List <string> members, long index) { await ServerStore.Cluster.WaitForIndexNotification(index); // first let see if we commit this in the leader if (members.Count == 0) { throw new InvalidOperationException("Cannot wait for execution when there are no nodes to execute ON."); } var executors = new List <ClusterRequestExecutor>(); try { using (var cts = CancellationTokenSource.CreateLinkedTokenSource(ServerStore.ServerShutdown)) { cts.CancelAfter(ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan); var waitingTasks = new List <Task <Exception> >(); List <Exception> exceptions = null; foreach (var member in members) { var url = clusterTopology.GetUrlFromTag(member); var executor = ClusterRequestExecutor.CreateForSingleNode(url, ServerStore.Server.Certificate.Certificate); executors.Add(executor); waitingTasks.Add(ExecuteTask(executor, member, cts.Token)); } while (waitingTasks.Count > 0) { var task = await Task.WhenAny(waitingTasks); waitingTasks.Remove(task); if (task.Result == null) { continue; } var exception = task.Result.ExtractSingleInnerException(); if (exceptions == null) { exceptions = new List <Exception>(); } exceptions.Add(exception); } if (exceptions != null) { var allTimeouts = true; foreach (var exception in exceptions) { if (exception is OperationCanceledException) { continue; } allTimeouts = false; } var aggregateException = new AggregateException(exceptions); if (allTimeouts) { throw new TimeoutException($"Waited too long for the raft command (number {index}) to be executed on any of the relevant nodes to this command.", aggregateException); } throw new InvalidDataException($"The database '{database}' was created but is not accessible, because all of the nodes on which this database was supposed to reside on, threw an exception.", aggregateException); } } } finally { foreach (var executor in executors) { executor.Dispose(); } } async Task <Exception> ExecuteTask(RequestExecutor executor, string nodeTag, CancellationToken token) { try { await executor.ExecuteAsync(new WaitForRaftIndexCommand(index), context, token : token); return(null); } catch (RavenException re) when(re.InnerException is HttpRequestException) { // we want to throw for self-checks if (nodeTag == ServerStore.NodeTag) { return(re); } // ignore - we are ok when connection with a node cannot be established (test: AddDatabaseOnDisconnectedNode) return(null); } catch (Exception e) { return(e); } } }
private static IEnumerable <OngoingTask> CollectEtlTasks(DatabaseRecord databaseRecord, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store) { if (dbTopology == null) { yield break; } if (databaseRecord.RavenEtls != null) { foreach (var ravenEtl in databaseRecord.RavenEtls) { var tag = dbTopology.WhoseTaskIsIt(ravenEtl, store.IsPassive()); var taskState = OngoingTaskState.Enabled; if (ravenEtl.Disabled || ravenEtl.Transforms.All(x => x.Disabled)) { taskState = OngoingTaskState.Disabled; } else if (ravenEtl.Transforms.Any(x => x.Disabled)) { taskState = OngoingTaskState.PartiallyEnabled; } if (databaseRecord.RavenConnectionStrings.TryGetValue(ravenEtl.ConnectionStringName, out var connection) == false) { throw new InvalidOperationException( $"Could not find connection string named '{ravenEtl.ConnectionStringName}' in the database record for '{ravenEtl.Name}' ETL"); } yield return(new OngoingTaskRavenEtl { TaskId = ravenEtl.TaskId, TaskName = ravenEtl.Name, // TODO arek TaskConnectionStatus = TaskState = taskState, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, DestinationUrl = connection.Url, DestinationDatabase = connection.Database }); } } if (databaseRecord.SqlEtls != null) { foreach (var sqlEtl in databaseRecord.SqlEtls) { var tag = dbTopology.WhoseTaskIsIt(sqlEtl, store.IsPassive()); var taskState = OngoingTaskState.Enabled; if (sqlEtl.Disabled || sqlEtl.Transforms.All(x => x.Disabled)) { taskState = OngoingTaskState.Disabled; } else if (sqlEtl.Transforms.Any(x => x.Disabled)) { taskState = OngoingTaskState.PartiallyEnabled; } if (databaseRecord.SqlConnectionStrings.TryGetValue(sqlEtl.ConnectionStringName, out var sqlConnection) == false) { throw new InvalidOperationException( $"Could not find connection string named '{sqlEtl.ConnectionStringName}' in the database record for '{sqlEtl.Name}' ETL"); } var(database, server) = SqlConnectionStringParser.GetDatabaseAndServerFromConnectionString(sqlEtl.FactoryName, sqlConnection.ConnectionString); yield return(new OngoingTaskSqlEtl { TaskId = sqlEtl.TaskId, TaskName = sqlEtl.Name, // TODO arek TaskConnectionStatus = TaskState = taskState, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, DestinationServer = server, DestinationDatabase = database }); } } }