private void AssertValidConnection(IncomingConnectionInfo connectionInfo) { //precaution, should never happen.. if (string.IsNullOrWhiteSpace(connectionInfo.SourceDatabaseId) || Guid.TryParse(connectionInfo.SourceDatabaseId, out Guid sourceDbId) == false) { throw new InvalidOperationException( $"Failed to parse source database Id. What I got is {(string.IsNullOrWhiteSpace(connectionInfo.SourceDatabaseId) ? "<empty string>" : Database.DbId.ToString())}. This is not supposed to happen and is likely a bug."); } if (sourceDbId == Database.DbId) { throw new InvalidOperationException( $"Cannot have replication with source and destination being the same database. They share the same db id ({connectionInfo} - {Database.DbId})"); } if (_server.IsPassive()) { throw new InvalidOperationException( $"Cannot accept the incoming replication connection from {connectionInfo.SourceUrl}, because this node is in passive state."); } if (_incoming.TryRemove(connectionInfo.SourceDatabaseId, out IncomingReplicationHandler value)) { if (_log.IsInfoEnabled) { _log.Info( $"Disconnecting existing connection from {value.FromToString} because we got a new connection from the same source db"); } IncomingReplicationRemoved?.Invoke(value); value.Dispose(); } }
private static IEnumerable <OngoingTask> CollectExternalReplicationTasks(List <ExternalReplication> watchers, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store) { if (dbTopology == null) { yield break; } foreach (var watcher in watchers) { NodeId responsibale = null; var tag = dbTopology.WhoseTaskIsIt(watcher, store.IsPassive()); if (tag != null) { responsibale = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }; } yield return(new OngoingTaskReplication { TaskId = watcher.TaskId, TaskName = watcher.Name, ResponsibleNode = responsibale, DestinationDatabase = watcher.Database, TaskState = watcher.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled, DestinationUrl = watcher.Url }); } }
public async Task <SubscriptionState> AssertSubscriptionConnectionDetails(long id, string name, TimeSpan timeout) { await _serverStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, id); using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverStoreContext)) using (serverStoreContext.OpenReadTransaction()) { var subscription = GetSubscriptionFromServerStore(serverStoreContext, name); var dbRecord = _serverStore.Cluster.ReadDatabase(serverStoreContext, _db.Name, out var _); var whoseTaskIsIt = dbRecord.Topology.WhoseTaskIsIt(subscription, _serverStore.IsPassive()); if (whoseTaskIsIt != _serverStore.NodeTag) { throw new SubscriptionDoesNotBelongToNodeException($"Subscripition with id {id} can't be proccessed on current node ({_serverStore.NodeTag}), because it belongs to {whoseTaskIsIt}") { AppropriateNode = whoseTaskIsIt }; } if (subscription.Disabled) { throw new SubscriptionClosedException($"The subscription {id} is disabled and cannot be used until enabled"); } return(subscription); } }
public async Task GetRemoteTaskTcp() { var remoteTask = GetStringQueryString("remote-task"); var database = GetStringQueryString("database"); var verifyDatabase = GetBoolValueQueryString("verify-database", false); if (ServerStore.IsPassive()) { throw new NodeIsPassiveException($"Can't fetch Tcp info from a passive node in url {this.HttpContext.Request.GetFullUrl()}"); } if (verifyDatabase.HasValue && verifyDatabase.Value) { await ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); } if (Authenticate(HttpContext, ServerStore, database, remoteTask) == false) { return; } using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { var output = Server.ServerStore.GetTcpInfoAndCertificates(HttpContext.Request.GetClientRequestedNodeUrl(), forExternalUse: true); context.Write(writer, output); } }
private static IEnumerable <OngoingTask> CollectExternalReplicationTasks(string name, List <ExternalReplication> watchers, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store) { if (dbTopology == null) { yield break; } foreach (var watcher in watchers) { NodeId responsibale = null; var tag = dbTopology.WhoseTaskIsIt(watcher, store.IsPassive()); if (tag != null) { responsibale = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }; } (string Url, OngoingTaskReplication.ReplicationStatus Status)res = (null, OngoingTaskReplication.ReplicationStatus.None); string error = null; if (tag == store.NodeTag) { try { store.DatabasesLandlord.DatabasesCache.TryGetValue(name, out var task); res = task.Result.ReplicationLoader.GetExternalReplicationDestination(watcher.TaskId); } catch (Exception e) { // failed to retrive the destination url error = e.ToString(); } } else { res.Status = OngoingTaskReplication.ReplicationStatus.NotOnThisNode; } yield return(new OngoingTaskReplication { TaskId = watcher.TaskId, TaskName = watcher.Name, ResponsibleNode = responsibale, DestinationDatabase = watcher.Database, TaskState = watcher.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled, DestinationUrl = res.Url, Status = res.Status, Error = error }); } }
private bool TryGetMentorNode(string dbName, DatabaseTopology topology, ClusterTopology clusterTopology, string promotable, out string mentorNode) { var url = clusterTopology.GetUrlFromTag(promotable); var task = new PromotableTask(promotable, url, dbName); mentorNode = topology.WhoseTaskIsIt(task, _server.IsPassive()); if (mentorNode == null) { // We are in passive mode and were kicked out of the cluster. return(false); } return(true); }
private static IEnumerable <OngoingTask> CollectBackupTasks( DatabaseRecord databaseRecord, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store) { if (dbTopology == null) { yield break; } if (databaseRecord.PeriodicBackups == null) { yield break; } if (databaseRecord.PeriodicBackups.Count == 0) { yield break; } var database = store.DatabasesLandlord.TryGetOrCreateResourceStore(databaseRecord.DatabaseName).Result; foreach (var backupConfiguration in databaseRecord.PeriodicBackups) { var tag = dbTopology.WhoseTaskIsIt(backupConfiguration, store.IsPassive()); var backupDestinations = GetBackupDestinations(backupConfiguration); var backupStatus = database.PeriodicBackupRunner.GetBackupStatus(backupConfiguration.TaskId); var nextBackup = database.PeriodicBackupRunner.GetNextBackupDetails(databaseRecord, backupConfiguration, backupStatus); yield return(new OngoingTaskBackup { TaskId = backupConfiguration.TaskId, BackupType = backupConfiguration.BackupType, TaskName = backupConfiguration.Name, TaskState = backupConfiguration.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled, LastFullBackup = backupStatus.LastFullBackup, LastIncrementalBackup = backupStatus.LastIncrementalBackup, NextBackup = nextBackup, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, BackupDestinations = backupDestinations }); } }
public async Task UpdateExternalReplication() { if (ResourceNameValidator.IsValidResourceName(Database.Name, ServerStore.Configuration.Core.DataDirectory.FullPath, out string errorMessage) == false) { throw new BadRequestException(errorMessage); } using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { var updateJson = await context.ReadForMemoryAsync(RequestBodyStream(), "read-update-replication"); if (updateJson.TryGet(nameof(UpdateExternalReplicationCommand.Watcher), out BlittableJsonReaderObject watcherBlittable) == false) { throw new InvalidDataException($"{nameof(UpdateExternalReplicationCommand.Watcher)} was not found."); } var watcher = JsonDeserializationClient.ExternalReplication(watcherBlittable); if (ServerStore.LicenseManager.CanAddExternalReplication(out var licenseLimit) == false) { SetLicenseLimitResponse(licenseLimit); return; } var(index, _) = await ServerStore.UpdateExternalReplication(Database.Name, watcher); await Database.RachisLogIndexNotifications.WaitForIndexNotification(index); string responsibleNode; using (context.OpenReadTransaction()) { var record = ServerStore.Cluster.ReadDatabase(context, Database.Name); responsibleNode = record.Topology.WhoseTaskIsIt(watcher, ServerStore.IsPassive()); } HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(ModifyOngoingTaskResult.TaskId)] = watcher.TaskId == 0 ? index : watcher.TaskId, [nameof(ModifyOngoingTaskResult.RaftCommandIndex)] = index, [nameof(OngoingTask.ResponsibleNode)] = responsibleNode }); writer.Flush(); } } }
private TaskStatus GetTaskStatus( DatabaseRecord databaseRecord, PeriodicBackupConfiguration configuration, bool skipErrorLog = false) { if (configuration.Disabled) { return(TaskStatus.Disabled); } if (configuration.HasBackup() == false) { if (skipErrorLog == false) { var message = $"All backup destinations are disabled for backup task id: {configuration.TaskId}"; _database.NotificationCenter.Add(AlertRaised.Create( "Periodic Backup", message, AlertType.PeriodicBackup, NotificationSeverity.Info)); } return(TaskStatus.Disabled); } var whoseTaskIsIt = databaseRecord.Topology.WhoseTaskIsIt(configuration, _serverStore.IsPassive()); if (whoseTaskIsIt == null) { return(TaskStatus.Disabled); } if (whoseTaskIsIt == _serverStore.NodeTag) { return(TaskStatus.ActiveByCurrentNode); } if (_logger.IsInfoEnabled) { _logger.Info($"Backup job is skipped at {SystemTime.UtcNow}, because it is managed " + $"by '{whoseTaskIsIt}' node and not the current node ({_serverStore.NodeTag})"); } return(TaskStatus.ActiveByOtherNode); }
private static IEnumerable <OngoingTask> CollectSubscriptionTasks(TransactionOperationContext context, DatabaseRecord databaseRecord, ClusterTopology clusterTopology, ServerStore store) { foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(databaseRecord.DatabaseName))) { var subscriptionState = JsonDeserializationClient.SubscriptionState(keyValue.Value); var tag = databaseRecord.Topology.WhoseTaskIsIt(subscriptionState, store.IsPassive()); yield return(new OngoingTaskSubscription { // Supply only needed fields for List View ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, TaskName = subscriptionState.SubscriptionName, TaskState = subscriptionState.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled, TaskId = subscriptionState.SubscriptionId, Query = subscriptionState.Query }); } }
private void HandleTopologyChange(DatabaseRecord newRecord) { var instancesToDispose = new List <OutgoingReplicationHandler>(); if (newRecord == null || _server.IsPassive()) { DropOutgoingConnections(Destinations, ref instancesToDispose); _internalDestinations.Clear(); _externalDestinations.Clear(); _destinations.Clear(); DisposeConnections(instancesToDispose); return; } HandleInternalReplication(newRecord, ref instancesToDispose); HandleExternalReplication(newRecord, ref instancesToDispose); var destinations = new List <ReplicationNode>(); destinations.AddRange(_internalDestinations); destinations.AddRange(_externalDestinations); _destinations = destinations; DisposeConnections(instancesToDispose); }
public Task GetTopology() { var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name"); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { var dbId = Constants.Documents.Prefix + name; using (context.OpenReadTransaction()) using (var dbBlit = ServerStore.Cluster.Read(context, dbId, out long _)) { if (TryGetAllowedDbs(name, out var _, requireAdmin: false) == false) { return(Task.CompletedTask); } if (dbBlit == null) { // here we return 503 so clients will try to failover to another server // if this is a newly created db that we haven't been notified about it yet HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; HttpContext.Response.Headers["Database-Missing"] = name; using (var writer = new BlittableJsonTextWriter(context, HttpContext.Response.Body)) { context.Write(writer, new DynamicJsonValue { ["Type"] = "Error", ["Message"] = "Database " + name + " wasn't found" }); } return(Task.CompletedTask); } var clusterTopology = ServerStore.GetClusterTopology(context); if (ServerStore.IsPassive() && clusterTopology.TopologyId != null) { // we were kicked-out from the cluster HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; return(Task.CompletedTask); } clusterTopology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext); var dbRecord = JsonDeserializationCluster.DatabaseRecord(dbBlit); using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(Topology.Nodes)] = new DynamicJsonArray( dbRecord.Topology.Members.Select(x => new DynamicJsonValue { [nameof(ServerNode.Url)] = GetUrl(x, clusterTopology), [nameof(ServerNode.ClusterTag)] = x, [nameof(ServerNode.ServerRole)] = ServerNode.Role.Member, [nameof(ServerNode.Database)] = dbRecord.DatabaseName }) .Concat(dbRecord.Topology.Rehabs.Select(x => new DynamicJsonValue { [nameof(ServerNode.Url)] = GetUrl(x, clusterTopology), [nameof(ServerNode.ClusterTag)] = x, [nameof(ServerNode.Database)] = dbRecord.DatabaseName, [nameof(ServerNode.ServerRole)] = ServerNode.Role.Rehab }) ) ), [nameof(Topology.Etag)] = dbRecord.Topology.Stamp?.Index ?? -1 }); } } } return(Task.CompletedTask); }
public async Task GetTopology() { var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name"); var applicationIdentifier = GetStringQueryString("applicationIdentifier", required: false); if (applicationIdentifier != null) { AlertIfDocumentStoreCreationRateIsNotReasonable(applicationIdentifier, name); } using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { using (context.OpenReadTransaction()) using (var rawRecord = ServerStore.Cluster.ReadRawDatabaseRecord(context, name)) { if (await CanAccessDatabaseAsync(name, requireAdmin: false) == false) { return; } if (rawRecord == null) { // here we return 503 so clients will try to failover to another server // if this is a newly created db that we haven't been notified about it yet HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; HttpContext.Response.Headers["Database-Missing"] = name; await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { ["Type"] = "Error", ["Message"] = "Database " + name + " wasn't found" }); } return; } var clusterTopology = ServerStore.GetClusterTopology(context); if (ServerStore.IsPassive() && clusterTopology.TopologyId != null) { // we were kicked-out from the cluster HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; return; } if (rawRecord.Topology.Members.Count == 0 && rawRecord.Topology.Rehabs.Count == 0 && rawRecord.DeletionInProgress.Any()) { // The database at deletion progress from all nodes HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; HttpContext.Response.Headers["Database-Missing"] = name; await using (var writer = new AsyncBlittableJsonTextWriter(context, HttpContext.Response.Body)) { context.Write(writer, new DynamicJsonValue { ["Type"] = "Error", ["Message"] = "Database " + name + " was deleted" }); } return; } clusterTopology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext); await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(Topology.Nodes)] = new DynamicJsonArray( rawRecord.Topology.Members.Select(x => new DynamicJsonValue { [nameof(ServerNode.Url)] = GetUrl(x, clusterTopology), [nameof(ServerNode.ClusterTag)] = x, [nameof(ServerNode.ServerRole)] = ServerNode.Role.Member, [nameof(ServerNode.Database)] = rawRecord.DatabaseName }) .Concat(rawRecord.Topology.Rehabs.Select(x => new DynamicJsonValue { [nameof(ServerNode.Url)] = GetUrl(x, clusterTopology), [nameof(ServerNode.ClusterTag)] = x, [nameof(ServerNode.Database)] = rawRecord.DatabaseName, [nameof(ServerNode.ServerRole)] = ServerNode.Role.Rehab }) ) ), [nameof(Topology.Etag)] = rawRecord.Topology.Stamp?.Index ?? -1 }); } } } }
public async Task AddNode() { SetupCORSHeaders(); var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url"); var watcher = GetBoolValueQueryString("watcher", false); var assignedCores = GetIntValueQueryString("assignedCores", false); if (assignedCores <= 0) { throw new ArgumentException("Assigned cores must be greater than 0!"); } nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl); var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase); if (HttpContext.Request.IsHttps != remoteIsHttps) { throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP."); } NodeInfo nodeInfo; using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate)) { requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout; var infoCmd = new GetNodeInfoCommand(); try { await requestExecutor.ExecuteAsync(infoCmd, ctx); } catch (AllTopologyNodesDownException e) { throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e); } nodeInfo = infoCmd.Result; if (ServerStore.IsPassive() && nodeInfo.TopologyId != null) { throw new TopologyMismatchException("You can't add new node to an already existing cluster"); } } if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores) { throw new ArgumentException("Cannot add node because the assigned cores is larger " + $"than the available cores on that machine: {nodeInfo.NumberOfCores}"); } ServerStore.EnsureNotPassive(); if (assignedCores == null) { assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores); } Debug.Assert(assignedCores <= nodeInfo.NumberOfCores); ServerStore.LicenseManager.AssertCanAddNode(nodeUrl, assignedCores.Value); if (ServerStore.IsLeader()) { using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) { string topologyId; ClusterTopology clusterTopology; using (ctx.OpenReadTransaction()) { clusterTopology = ServerStore.GetClusterTopology(ctx); topologyId = clusterTopology.TopologyId; } var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); if (possibleNode.HasUrl) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}"); } if (nodeInfo.ServerId == ServerStore.GetServerId()) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}"); } if (nodeInfo.TopologyId != null) { if (topologyId != nodeInfo.TopologyId) { throw new TopologyMismatchException( $"Adding a new node to cluster failed. The new node is already in another cluster. " + $"Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}"); } if (nodeInfo.CurrentState != RachisState.Passive) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " + $"because it's already in the cluster under tag :{nodeInfo.NodeTag} " + $"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}"); } } var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag; CertificateDefinition oldServerCert = null; X509Certificate2 certificate = null; if (remoteIsHttps) { if (nodeInfo.Certificate == null) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS"); } certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet); var now = DateTime.UtcNow; if (certificate.NotBefore.ToUniversalTime() > now) { // Because of time zone and time drift issues, we can't assume that the certificate generation will be // proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the // current time. Clients may still fail to work with our certificate because of timing issues, // but the admin needs to setup time sync properly and there isn't much we can do at that point if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1) { throw new InvalidOperationException( $"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}"); } } if (certificate.NotAfter.ToUniversalTime() < now) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}"); } var expected = GetStringQueryString("expectedThumbprint", required: false); if (expected != null) { if (certificate.Thumbprint != expected) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'."); } } using (ctx.OpenReadTransaction()) { var key = Constants.Certificates.Prefix + certificate.Thumbprint; var readCert = ServerStore.Cluster.Read(ctx, key); if (readCert != null) { oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert); } } if (oldServerCert == null) { var certificateDefinition = new CertificateDefinition { Certificate = nodeInfo.Certificate, Thumbprint = certificate.Thumbprint, NotAfter = certificate.NotAfter, Name = "Server Certificate for " + nodeUrl, SecurityClearance = SecurityClearance.ClusterNode }; var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition)); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } } await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false); using (ctx.OpenReadTransaction()) { clusterTopology = ServerStore.GetClusterTopology(ctx); possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null; if (certificate != null) { var key = Constants.Certificates.Prefix + certificate.Thumbprint; var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key)); if (modifiedServerCert == null) { throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through."); } if (oldServerCert == null) { modifiedServerCert.Name = "Server certificate for Node " + nodeTag; } else { var value = "Node " + nodeTag; if (modifiedServerCert.Name.Contains(value) == false) { modifiedServerCert.Name += ", " + value; } } var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert)); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } var nodeDetails = new NodeDetails { NodeTag = nodeTag, AssignedCores = assignedCores.Value, NumberOfCores = nodeInfo.NumberOfCores, InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb, UsableMemoryInGb = nodeInfo.UsableMemoryInGb, BuildInfo = nodeInfo.BuildInfo }; await ServerStore.LicenseManager.CalculateLicenseLimits(nodeDetails, forceFetchingNodeInfo : true, waitToUpdate : true); } NoContentStatus(); return; } } RedirectToLeader(); }
private IEnumerable <EtlProcess> GetRelevantProcesses <T, TConnectionString>(List <T> configurations, HashSet <string> uniqueNames) where T : EtlConfiguration <TConnectionString> where TConnectionString : ConnectionString { foreach (var config in configurations) { SqlEtlConfiguration sqlConfig = null; RavenEtlConfiguration ravenConfig = null; var connectionStringNotFound = false; switch (config.EtlType) { case EtlType.Raven: ravenConfig = config as RavenEtlConfiguration; if (_databaseRecord.RavenConnectionStrings.TryGetValue(config.ConnectionStringName, out var ravenConnection)) { ravenConfig.Initialize(ravenConnection); } else { connectionStringNotFound = true; } break; case EtlType.Sql: sqlConfig = config as SqlEtlConfiguration; if (_databaseRecord.SqlConnectionStrings.TryGetValue(config.ConnectionStringName, out var sqlConnection)) { sqlConfig.Initialize(sqlConnection); } else { connectionStringNotFound = true; } break; default: ThrownUnknownEtlConfiguration(config.GetType()); break; } if (connectionStringNotFound) { LogConfigurationError(config, new List <string> { $"Connection string named '{config.ConnectionStringName}' was not found for {config.EtlType} ETL" }); continue; } if (ValidateConfiguration(config, uniqueNames) == false) { continue; } if (config.Disabled) { continue; } if (_databaseRecord.Topology.WhoseTaskIsIt(config, _serverStore.IsPassive()) != _serverStore.NodeTag) { continue; } foreach (var transform in config.Transforms) { if (transform.Disabled) { continue; } EtlProcess process = null; if (sqlConfig != null) { process = new SqlEtl(transform, sqlConfig, _database, _serverStore); } if (ravenConfig != null) { process = new RavenEtl(transform, ravenConfig, _database, _serverStore); } yield return(process); } } }
private void WriteDatabaseInfo(string databaseName, BlittableJsonReaderObject dbRecordBlittable, TransactionOperationContext context, BlittableJsonTextWriter writer) { var online = ServerStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out Task <DocumentDatabase> dbTask) && dbTask != null && dbTask.IsCompleted; // Check for exceptions if (dbTask != null && dbTask.IsFaulted) { WriteFaultedDatabaseInfo(context, writer, dbTask, databaseName); return; } var dbRecord = JsonDeserializationCluster.DatabaseRecord(dbRecordBlittable); var db = online ? dbTask.Result : null; var indexingStatus = db?.IndexStore.Status ?? IndexRunningStatus.Running; // Looking for disabled indexing flag inside the database settings for offline database status if (dbRecord.Settings.TryGetValue(RavenConfiguration.GetKey(x => x.Indexing.Disabled), out var val) && val == "true") { indexingStatus = IndexRunningStatus.Disabled; } var disabled = dbRecord.Disabled; var topology = dbRecord.Topology; var clusterTopology = ServerStore.GetClusterTopology(context); var nodesTopology = new NodesTopology(); if (topology != null) { foreach (var member in topology.Members) { var url = clusterTopology.GetUrlFromTag(member); var node = new InternalReplication { Database = databaseName, NodeTag = member, Url = url }; nodesTopology.Members.Add(GetNodeId(node)); SetNodeStatus(topology, member, nodesTopology); } foreach (var promotable in topology.Promotables) { var node = GetNode(databaseName, clusterTopology, promotable, out var promotableTask); var mentor = topology.WhoseTaskIsIt(promotableTask, ServerStore.IsPassive()); nodesTopology.Promotables.Add(GetNodeId(node, mentor)); SetNodeStatus(topology, promotable, nodesTopology); } foreach (var rehab in topology.Rehabs) { var node = GetNode(databaseName, clusterTopology, rehab, out var promotableTask); var mentor = topology.WhoseTaskIsIt(promotableTask, ServerStore.IsPassive()); nodesTopology.Rehabs.Add(GetNodeId(node, mentor)); SetNodeStatus(topology, rehab, nodesTopology); } } if (online == false) { // If state of database is found in the cache we can continue if (ServerStore.DatabaseInfoCache.TryWriteOfflineDatabaseStatusToRequest( context, writer, databaseName, disabled, indexingStatus, nodesTopology)) { return; } // We won't find it if it is a new database or after a dirty shutdown, so just report empty values then } var size = new Size(GetTotalSize(db)); var databaseInfo = new DatabaseInfo { Name = databaseName, Disabled = disabled, TotalSize = size, IsAdmin = true, //TODO: implement me! IsEncrypted = dbRecord.Encrypted, UpTime = online ? (TimeSpan?)GetUptime(db) : null, BackupInfo = GetBackupInfo(db), Alerts = db?.NotificationCenter.GetAlertCount() ?? 0, RejectClients = false, //TODO: implement me! LoadError = null, IndexingErrors = db?.IndexStore.GetIndexes().Sum(index => index.GetErrorCount()) ?? 0, DocumentsCount = db?.DocumentsStorage.GetNumberOfDocuments() ?? 0, HasRevisionsConfiguration = db?.DocumentsStorage.RevisionsStorage.Configuration != null, HasExpirationConfiguration = db?.ExpiredDocumentsCleaner != null, IndexesCount = db?.IndexStore.GetIndexes().Count() ?? 0, IndexingStatus = indexingStatus, NodesTopology = nodesTopology, ReplicationFactor = topology?.ReplicationFactor ?? -1, DynamicNodesDistribution = topology?.DynamicNodesDistribution ?? false }; var doc = databaseInfo.ToJson(); context.Write(writer, doc); }
public async Task AddNode() { var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url"); var tag = GetStringQueryString("tag", false); var watcher = GetBoolValueQueryString("watcher", false); var raftRequestId = GetRaftRequestIdFromQuery(); var maxUtilizedCores = GetIntValueQueryString("maxUtilizedCores", false); if (maxUtilizedCores != null && maxUtilizedCores <= 0) { throw new ArgumentException("Max utilized cores cores must be greater than 0"); } nodeUrl = nodeUrl.Trim(); if (Uri.IsWellFormedUriString(nodeUrl, UriKind.Absolute) == false) { throw new InvalidOperationException($"Given node URL '{nodeUrl}' is not in a correct format."); } nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl); var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase); if (HttpContext.Request.IsHttps != remoteIsHttps) { throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP."); } tag = tag?.Trim(); NodeInfo nodeInfo; using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate)) { requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout; // test connection to remote. var result = await ServerStore.TestConnectionToRemote(nodeUrl, database : null); if (result.Success == false) { throw new InvalidOperationException(result.Error); } // test connection from remote to destination result = await ServerStore.TestConnectionFromRemote(requestExecutor, ctx, nodeUrl); if (result.Success == false) { throw new InvalidOperationException(result.Error); } var infoCmd = new GetNodeInfoCommand(); try { await requestExecutor.ExecuteAsync(infoCmd, ctx); } catch (AllTopologyNodesDownException e) { throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e); } nodeInfo = infoCmd.Result; if (SchemaUpgrader.CurrentVersion.ServerVersion != nodeInfo.ServerSchemaVersion) { var nodesVersion = nodeInfo.ServerSchemaVersion == 0 ? "Pre 4.2 version" : nodeInfo.ServerSchemaVersion.ToString(); throw new InvalidOperationException($"Can't add node with mismatched storage schema version.{Environment.NewLine}" + $"My version is {SchemaUpgrader.CurrentVersion.ServerVersion}, while node's version is {nodesVersion}"); } if (ServerStore.IsPassive() && nodeInfo.TopologyId != null) { throw new TopologyMismatchException("You can't add new node to an already existing cluster"); } } if (ServerStore.ValidateFixedPort && nodeInfo.HasFixedPort == false) { throw new InvalidOperationException($"Failed to add node '{nodeUrl}' to cluster. " + $"Node '{nodeUrl}' has port '0' in 'Configuration.Core.ServerUrls' setting. " + "Adding a node with non fixed port is forbidden. Define a fixed port for the node to enable cluster creation."); } await ServerStore.EnsureNotPassiveAsync(); ServerStore.LicenseManager.AssertCanAddNode(); if (ServerStore.IsLeader()) { using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) { var clusterTopology = ServerStore.GetClusterTopology(); var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); if (possibleNode.HasUrl) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}"); } if (nodeInfo.ServerId == ServerStore.GetServerId()) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}"); } if (nodeInfo.TopologyId != null) { AssertCanAddNodeWithTopologyId(clusterTopology, nodeInfo, nodeUrl); } var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? tag : nodeInfo.NodeTag; CertificateDefinition oldServerCert = null; X509Certificate2 certificate = null; if (remoteIsHttps) { if (nodeInfo.Certificate == null) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS"); } certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet); var now = DateTime.UtcNow; if (certificate.NotBefore.ToUniversalTime() > now) { // Because of time zone and time drift issues, we can't assume that the certificate generation will be // proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the // current time. Clients may still fail to work with our certificate because of timing issues, // but the admin needs to setup time sync properly and there isn't much we can do at that point if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1) { throw new InvalidOperationException( $"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}"); } } if (certificate.NotAfter.ToUniversalTime() < now) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}"); } var expected = GetStringQueryString("expectedThumbprint", required: false); if (expected != null) { if (certificate.Thumbprint != expected) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'."); } } // if it's the same server certificate as our own, we don't want to add it to the cluster if (certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint) { using (ctx.OpenReadTransaction()) { var readCert = ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint); if (readCert != null) { oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert); } } if (oldServerCert == null) { var certificateDefinition = new CertificateDefinition { Certificate = nodeInfo.Certificate, Thumbprint = certificate.Thumbprint, PublicKeyPinningHash = certificate.GetPublicKeyPinningHash(), NotAfter = certificate.NotAfter, Name = "Server Certificate for " + nodeUrl, SecurityClearance = SecurityClearance.ClusterNode }; var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, certificateDefinition, $"{raftRequestId}/put-new-certificate")); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } } } await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : true, asWatcher : watcher ?? false); using (ctx.OpenReadTransaction()) { clusterTopology = ServerStore.GetClusterTopology(ctx); possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null; if (certificate != null && certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint) { var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint)); if (modifiedServerCert == null) { throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through."); } if (oldServerCert == null) { modifiedServerCert.Name = "Server certificate for Node " + nodeTag; } else { var value = "Node " + nodeTag; if (modifiedServerCert.Name.Contains(value) == false) { modifiedServerCert.Name += ", " + value; } } var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, modifiedServerCert, $"{raftRequestId}/put-modified-certificate")); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } var detailsPerNode = new DetailsPerNode { MaxUtilizedCores = maxUtilizedCores, NumberOfCores = nodeInfo.NumberOfCores, InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb, UsableMemoryInGb = nodeInfo.UsableMemoryInGb, BuildInfo = nodeInfo.BuildInfo, OsInfo = nodeInfo.OsInfo }; var maxCores = ServerStore.LicenseManager.LicenseStatus.MaxCores; try { await ServerStore.PutNodeLicenseLimitsAsync(nodeTag, detailsPerNode, maxCores, $"{raftRequestId}/put-license-limits"); } catch { // we'll retry this again later } } NoContentStatus(); return; } } RedirectToLeader(); }
public async Task AddNode() { SetupCORSHeaders(); var nodeUrl = GetStringQueryString("url").TrimEnd('/'); var watcher = GetBoolValueQueryString("watcher", false); var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase); if (HttpContext.Request.IsHttps != remoteIsHttps) { throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP."); } NodeInfo nodeInfo; using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.ClusterCertificateHolder.Certificate)) { requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout; var infoCmd = new GetNodeInfoCommand(); try { await requestExecutor.ExecuteAsync(infoCmd, ctx); } catch (AllTopologyNodesDownException e) { throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e); } nodeInfo = infoCmd.Result; if (ServerStore.IsPassive() && nodeInfo.TopologyId != null) { throw new TopologyMismatchException("You can't add new node to an already existing cluster"); } } ServerStore.EnsureNotPassive(); if (ServerStore.IsLeader()) { using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) { string topologyId; using (ctx.OpenReadTransaction()) { var clusterTopology = ServerStore.GetClusterTopology(ctx); var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); if (possibleNode.HasUrl) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}"); } topologyId = clusterTopology.TopologyId; } if (nodeInfo.TopologyId != null && topologyId != nodeInfo.TopologyId) { throw new TopologyMismatchException( $"Adding a new node to cluster failed. The new node is already in another cluster. Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}"); } var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag; if (remoteIsHttps) { if (nodeInfo.Certificate == null) { throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because it has no certificate while trying to use HTTPS"); } var certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate)); if (certificate.NotBefore > DateTime.UtcNow) { throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}"); } if (certificate.NotAfter < DateTime.UtcNow) { throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}"); } var expected = GetStringQueryString("expectedThumbrpint", required: false); if (expected != null) { if (certificate.Thumbprint != expected) { throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'."); } } var certificateDefinition = new CertificateDefinition { Certificate = nodeInfo.Certificate, Thumbprint = certificate.Thumbprint }; var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition)); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false); NoContentStatus(); return; } } RedirectToLeader(); }
public async Task AddNode() { SetupCORSHeaders(); var nodeUrl = GetStringQueryString("url").TrimEnd('/'); var watcher = GetBoolValueQueryString("watcher", false); var assignedCores = GetIntValueQueryString("assignedCores", false); if (assignedCores <= 0) { throw new ArgumentException("Assigned cores must be greater than 0!"); } var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase); if (HttpContext.Request.IsHttps != remoteIsHttps) { throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP."); } NodeInfo nodeInfo; using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.ClusterCertificateHolder.Certificate)) { requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout; var infoCmd = new GetNodeInfoCommand(); try { await requestExecutor.ExecuteAsync(infoCmd, ctx); } catch (AllTopologyNodesDownException e) { throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e); } nodeInfo = infoCmd.Result; if (ServerStore.IsPassive() && nodeInfo.TopologyId != null) { throw new TopologyMismatchException("You can't add new node to an already existing cluster"); } } if (assignedCores == null) { assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores); } if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores) { throw new ArgumentException("Cannot add node because the assigned cores is larger " + $"than the available cores on that machine: {nodeInfo.NumberOfCores}"); } ServerStore.EnsureNotPassive(); if (ServerStore.LicenseManager.CanAddNode(nodeUrl, assignedCores, out var licenseLimit) == false) { SetLicenseLimitResponse(licenseLimit); return; } if (ServerStore.IsLeader()) { using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) { string topologyId; using (ctx.OpenReadTransaction()) { var clusterTopology = ServerStore.GetClusterTopology(ctx); var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); if (possibleNode.HasUrl) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}"); } topologyId = clusterTopology.TopologyId; } if (nodeInfo.TopologyId != null && topologyId != nodeInfo.TopologyId) { throw new TopologyMismatchException( $"Adding a new node to cluster failed. The new node is already in another cluster. Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}"); } var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag; CertificateDefinition oldServerCert = null; X509Certificate2 certificate = null; if (remoteIsHttps) { if (nodeInfo.Certificate == null) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS"); } certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate)); if (certificate.NotBefore > DateTime.UtcNow) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}"); } if (certificate.NotAfter < DateTime.UtcNow) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}"); } var expected = GetStringQueryString("expectedThumbrpint", required: false); if (expected != null) { if (certificate.Thumbprint != expected) { throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'."); } } using (ctx.OpenReadTransaction()) { var key = Constants.Certificates.Prefix + certificate.Thumbprint; var readCert = ServerStore.Cluster.Read(ctx, key); if (readCert != null) { oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert); } } if (oldServerCert == null) { var certificateDefinition = new CertificateDefinition { Certificate = nodeInfo.Certificate, Thumbprint = certificate.Thumbprint, Name = "Server Certificate for " + nodeUrl, SecurityClearance = SecurityClearance.ClusterNode }; var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition)); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } } await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false); using (ctx.OpenReadTransaction()) { var clusterTopology = ServerStore.GetClusterTopology(ctx); var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl); nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null; if (certificate != null) { var key = Constants.Certificates.Prefix + certificate.Thumbprint; var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key)); if (modifiedServerCert == null) { throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through."); } if (oldServerCert == null) { modifiedServerCert.Name = "Server certificate for Node " + nodeTag; } else { var value = "Node " + nodeTag; if (modifiedServerCert.Name.Contains(value) == false) { modifiedServerCert.Name += ", " + value; } } var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert)); await ServerStore.Cluster.WaitForIndexNotification(res.Index); } ServerStore.LicenseManager.CalculateLicenseLimits(nodeTag, assignedCores, nodeInfo, forceFetchingNodeInfo: true); } NoContentStatus(); return; } } RedirectToLeader(); }
public Task GetOngoingTaskInfo() { if (ResourceNameValidator.IsValidResourceName(Database.Name, ServerStore.Configuration.Core.DataDirectory.FullPath, out string errorMessage) == false) { throw new BadRequestException(errorMessage); } var key = GetLongQueryString("key"); var typeStr = GetQueryStringValueAndAssertIfSingleAndNotEmpty("type"); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { using (context.OpenReadTransaction()) { var clusterTopology = ServerStore.GetClusterTopology(context); var record = ServerStore.Cluster.ReadDatabase(context, Database.Name); var dbTopology = record?.Topology; if (Enum.TryParse <OngoingTaskType>(typeStr, true, out var type) == false) { throw new ArgumentException($"Unknown task type: {type}", "type"); } string tag; switch (type) { case OngoingTaskType.Replication: var watcher = record?.ExternalReplication.Find(x => x.TaskId == key); if (watcher == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; break; } tag = dbTopology?.WhoseTaskIsIt(watcher, ServerStore.IsPassive()); var replicationTaskInfo = new OngoingTaskReplication { TaskId = watcher.TaskId, TaskName = watcher.Name, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, DestinationDatabase = watcher.Database, TaskState = watcher.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled, DestinationUrl = watcher.Url }; WriteResult(context, replicationTaskInfo); break; case OngoingTaskType.Backup: var backupConfiguration = record?.PeriodicBackups?.Find(x => x.TaskId == key); if (backupConfiguration == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; break; } tag = dbTopology?.WhoseTaskIsIt(backupConfiguration, ServerStore.IsPassive()); var backupDestinations = GetBackupDestinations(backupConfiguration); var backupStatus = Database.PeriodicBackupRunner.GetBackupStatus(key); var nextBackup = Database.PeriodicBackupRunner.GetNextBackupDetails(record, backupConfiguration, backupStatus); var backupTaskInfo = new OngoingTaskBackup { TaskId = backupConfiguration.TaskId, BackupType = backupConfiguration.BackupType, TaskName = backupConfiguration.Name, TaskState = backupConfiguration.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, BackupDestinations = backupDestinations, LastFullBackup = backupStatus.LastFullBackup, LastIncrementalBackup = backupStatus.LastIncrementalBackup, NextBackup = nextBackup }; WriteResult(context, backupTaskInfo); break; case OngoingTaskType.SqlEtl: var sqlEtl = record?.SqlEtls?.Find(x => x.TaskId == key); if (sqlEtl == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; break; } WriteResult(context, new OngoingTaskSqlEtl { TaskId = sqlEtl.TaskId, TaskName = sqlEtl.Name, Configuration = sqlEtl }); break; case OngoingTaskType.RavenEtl: var ravenEtl = record?.RavenEtls?.Find(x => x.TaskId == key); if (ravenEtl == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; break; } WriteResult(context, new OngoingTaskRavenEtl { TaskId = ravenEtl.TaskId, TaskName = ravenEtl.Name, Configuration = ravenEtl }); break; case OngoingTaskType.Subscription: var nameKey = GetQueryStringValueAndAssertIfSingleAndNotEmpty("taskName"); var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(record.DatabaseName, nameKey); var doc = ServerStore.Cluster.Read(context, itemKey); if (doc == null) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; break; } var subscriptionState = JsonDeserializationClient.SubscriptionState(doc); tag = dbTopology?.WhoseTaskIsIt(subscriptionState, ServerStore.IsPassive()); var subscriptionStateInfo = new SubscriptionStateWithNodeDetails { Query = subscriptionState.Query, ChangeVectorForNextBatchStartingPoint = subscriptionState.ChangeVectorForNextBatchStartingPoint, SubscriptionId = subscriptionState.SubscriptionId, SubscriptionName = subscriptionState.SubscriptionName, LastTimeServerMadeProgressWithDocuments = subscriptionState.LastTimeServerMadeProgressWithDocuments, Disabled = subscriptionState.Disabled, LastClientConnectionTime = subscriptionState.LastClientConnectionTime, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) } }; // Todo: here we'll need to talk with the running node? TaskConnectionStatus = subscriptionState.Disabled ? OngoingTaskConnectionStatus.NotActive : OngoingTaskConnectionStatus.Active, WriteResult(context, subscriptionStateInfo.ToJson()); break; default: HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound; break; } } } return(Task.CompletedTask); }
private static IEnumerable <OngoingTask> CollectEtlTasks(DatabaseRecord databaseRecord, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store) { if (dbTopology == null) { yield break; } if (databaseRecord.RavenEtls != null) { foreach (var ravenEtl in databaseRecord.RavenEtls) { var tag = dbTopology.WhoseTaskIsIt(ravenEtl, store.IsPassive()); var taskState = OngoingTaskState.Enabled; if (ravenEtl.Disabled || ravenEtl.Transforms.All(x => x.Disabled)) { taskState = OngoingTaskState.Disabled; } else if (ravenEtl.Transforms.Any(x => x.Disabled)) { taskState = OngoingTaskState.PartiallyEnabled; } if (databaseRecord.RavenConnectionStrings.TryGetValue(ravenEtl.ConnectionStringName, out var connection) == false) { throw new InvalidOperationException( $"Could not find connection string named '{ravenEtl.ConnectionStringName}' in the database record for '{ravenEtl.Name}' ETL"); } yield return(new OngoingTaskRavenEtl { TaskId = ravenEtl.TaskId, TaskName = ravenEtl.Name, // TODO arek TaskConnectionStatus = TaskState = taskState, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, DestinationUrl = connection.Url, DestinationDatabase = connection.Database }); } } if (databaseRecord.SqlEtls != null) { foreach (var sqlEtl in databaseRecord.SqlEtls) { var tag = dbTopology.WhoseTaskIsIt(sqlEtl, store.IsPassive()); var taskState = OngoingTaskState.Enabled; if (sqlEtl.Disabled || sqlEtl.Transforms.All(x => x.Disabled)) { taskState = OngoingTaskState.Disabled; } else if (sqlEtl.Transforms.Any(x => x.Disabled)) { taskState = OngoingTaskState.PartiallyEnabled; } if (databaseRecord.SqlConnectionStrings.TryGetValue(sqlEtl.ConnectionStringName, out var sqlConnection) == false) { throw new InvalidOperationException( $"Could not find connection string named '{sqlEtl.ConnectionStringName}' in the database record for '{sqlEtl.Name}' ETL"); } var(database, server) = SqlConnectionStringParser.GetDatabaseAndServerFromConnectionString(sqlEtl.FactoryName, sqlConnection.ConnectionString); yield return(new OngoingTaskSqlEtl { TaskId = sqlEtl.TaskId, TaskName = sqlEtl.Name, // TODO arek TaskConnectionStatus = TaskState = taskState, ResponsibleNode = new NodeId { NodeTag = tag, NodeUrl = clusterTopology.GetUrlFromTag(tag) }, DestinationServer = server, DestinationDatabase = database }); } } }