private void AssertValidConnection(IncomingConnectionInfo connectionInfo)
{
//precaution, should never happen..
if (string.IsNullOrWhiteSpace(connectionInfo.SourceDatabaseId) ||
Guid.TryParse(connectionInfo.SourceDatabaseId, out Guid sourceDbId) == false)
{
throw new InvalidOperationException(
$"Failed to parse source database Id. What I got is {(string.IsNullOrWhiteSpace(connectionInfo.SourceDatabaseId) ? "<empty string>" : Database.DbId.ToString())}. This is not supposed to happen and is likely a bug.");
}
if (sourceDbId == Database.DbId)
{
throw new InvalidOperationException(
$"Cannot have replication with source and destination being the same database. They share the same db id ({connectionInfo} - {Database.DbId})");
}
if (_server.IsPassive())
{
throw new InvalidOperationException(
$"Cannot accept the incoming replication connection from {connectionInfo.SourceUrl}, because this node is in passive state.");
}
if (_incoming.TryRemove(connectionInfo.SourceDatabaseId, out IncomingReplicationHandler value))
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Disconnecting existing connection from {value.FromToString} because we got a new connection from the same source db");
}
IncomingReplicationRemoved?.Invoke(value);
value.Dispose();
}
}
private static IEnumerable <OngoingTask> CollectExternalReplicationTasks(List <ExternalReplication> watchers, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store)
{
if (dbTopology == null)
{
yield break;
}
foreach (var watcher in watchers)
{
NodeId responsibale = null;
var tag = dbTopology.WhoseTaskIsIt(watcher, store.IsPassive());
if (tag != null)
{
responsibale = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
};
}
yield return(new OngoingTaskReplication
{
TaskId = watcher.TaskId,
TaskName = watcher.Name,
ResponsibleNode = responsibale,
DestinationDatabase = watcher.Database,
TaskState = watcher.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
DestinationUrl = watcher.Url
});
}
}
public async Task <SubscriptionState> AssertSubscriptionConnectionDetails(long id, string name, TimeSpan timeout)
{
await _serverStore.WaitForCommitIndexChange(RachisConsensus.CommitIndexModification.GreaterOrEqual, id);
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverStoreContext))
using (serverStoreContext.OpenReadTransaction())
{
var subscription = GetSubscriptionFromServerStore(serverStoreContext, name);
var dbRecord = _serverStore.Cluster.ReadDatabase(serverStoreContext, _db.Name, out var _);
var whoseTaskIsIt = dbRecord.Topology.WhoseTaskIsIt(subscription, _serverStore.IsPassive());
if (whoseTaskIsIt != _serverStore.NodeTag)
{
throw new SubscriptionDoesNotBelongToNodeException($"Subscripition with id {id} can't be proccessed on current node ({_serverStore.NodeTag}), because it belongs to {whoseTaskIsIt}")
{
AppropriateNode = whoseTaskIsIt
};
}
if (subscription.Disabled)
{
throw new SubscriptionClosedException($"The subscription {id} is disabled and cannot be used until enabled");
}
return(subscription);
}
}
public async Task GetRemoteTaskTcp()
{
var remoteTask = GetStringQueryString("remote-task");
var database = GetStringQueryString("database");
var verifyDatabase = GetBoolValueQueryString("verify-database", false);
if (ServerStore.IsPassive())
{
throw new NodeIsPassiveException($"Can't fetch Tcp info from a passive node in url {this.HttpContext.Request.GetFullUrl()}");
}
if (verifyDatabase.HasValue && verifyDatabase.Value)
{
await ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
}
if (Authenticate(HttpContext, ServerStore, database, remoteTask) == false)
{
return;
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
var output = Server.ServerStore.GetTcpInfoAndCertificates(HttpContext.Request.GetClientRequestedNodeUrl(), forExternalUse: true);
context.Write(writer, output);
}
}
private static IEnumerable <OngoingTask> CollectExternalReplicationTasks(string name, List <ExternalReplication> watchers, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store)
{
if (dbTopology == null)
{
yield break;
}
foreach (var watcher in watchers)
{
NodeId responsibale = null;
var tag = dbTopology.WhoseTaskIsIt(watcher, store.IsPassive());
if (tag != null)
{
responsibale = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
};
}
(string Url, OngoingTaskReplication.ReplicationStatus Status)res = (null, OngoingTaskReplication.ReplicationStatus.None);
string error = null;
if (tag == store.NodeTag)
{
try
{
store.DatabasesLandlord.DatabasesCache.TryGetValue(name, out var task);
res = task.Result.ReplicationLoader.GetExternalReplicationDestination(watcher.TaskId);
}
catch (Exception e)
{
// failed to retrive the destination url
error = e.ToString();
}
}
else
{
res.Status = OngoingTaskReplication.ReplicationStatus.NotOnThisNode;
}
yield return(new OngoingTaskReplication
{
TaskId = watcher.TaskId,
TaskName = watcher.Name,
ResponsibleNode = responsibale,
DestinationDatabase = watcher.Database,
TaskState = watcher.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
DestinationUrl = res.Url,
Status = res.Status,
Error = error
});
}
}
private bool TryGetMentorNode(string dbName, DatabaseTopology topology, ClusterTopology clusterTopology, string promotable, out string mentorNode)
{
var url = clusterTopology.GetUrlFromTag(promotable);
var task = new PromotableTask(promotable, url, dbName);
mentorNode = topology.WhoseTaskIsIt(task, _server.IsPassive());
if (mentorNode == null)
{
// We are in passive mode and were kicked out of the cluster.
return(false);
}
return(true);
}
private static IEnumerable <OngoingTask> CollectBackupTasks(
DatabaseRecord databaseRecord,
DatabaseTopology dbTopology,
ClusterTopology clusterTopology,
ServerStore store)
{
if (dbTopology == null)
{
yield break;
}
if (databaseRecord.PeriodicBackups == null)
{
yield break;
}
if (databaseRecord.PeriodicBackups.Count == 0)
{
yield break;
}
var database = store.DatabasesLandlord.TryGetOrCreateResourceStore(databaseRecord.DatabaseName).Result;
foreach (var backupConfiguration in databaseRecord.PeriodicBackups)
{
var tag = dbTopology.WhoseTaskIsIt(backupConfiguration, store.IsPassive());
var backupDestinations = GetBackupDestinations(backupConfiguration);
var backupStatus = database.PeriodicBackupRunner.GetBackupStatus(backupConfiguration.TaskId);
var nextBackup = database.PeriodicBackupRunner.GetNextBackupDetails(databaseRecord, backupConfiguration, backupStatus);
yield return(new OngoingTaskBackup
{
TaskId = backupConfiguration.TaskId,
BackupType = backupConfiguration.BackupType,
TaskName = backupConfiguration.Name,
TaskState = backupConfiguration.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
LastFullBackup = backupStatus.LastFullBackup,
LastIncrementalBackup = backupStatus.LastIncrementalBackup,
NextBackup = nextBackup,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
BackupDestinations = backupDestinations
});
}
}
public async Task UpdateExternalReplication()
{
if (ResourceNameValidator.IsValidResourceName(Database.Name, ServerStore.Configuration.Core.DataDirectory.FullPath, out string errorMessage) == false)
{
throw new BadRequestException(errorMessage);
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
var updateJson = await context.ReadForMemoryAsync(RequestBodyStream(), "read-update-replication");
if (updateJson.TryGet(nameof(UpdateExternalReplicationCommand.Watcher), out BlittableJsonReaderObject watcherBlittable) == false)
{
throw new InvalidDataException($"{nameof(UpdateExternalReplicationCommand.Watcher)} was not found.");
}
var watcher = JsonDeserializationClient.ExternalReplication(watcherBlittable);
if (ServerStore.LicenseManager.CanAddExternalReplication(out var licenseLimit) == false)
{
SetLicenseLimitResponse(licenseLimit);
return;
}
var(index, _) = await ServerStore.UpdateExternalReplication(Database.Name, watcher);
await Database.RachisLogIndexNotifications.WaitForIndexNotification(index);
string responsibleNode;
using (context.OpenReadTransaction())
{
var record = ServerStore.Cluster.ReadDatabase(context, Database.Name);
responsibleNode = record.Topology.WhoseTaskIsIt(watcher, ServerStore.IsPassive());
}
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(ModifyOngoingTaskResult.TaskId)] = watcher.TaskId == 0 ? index : watcher.TaskId,
[nameof(ModifyOngoingTaskResult.RaftCommandIndex)] = index,
[nameof(OngoingTask.ResponsibleNode)] = responsibleNode
});
writer.Flush();
}
}
}
private TaskStatus GetTaskStatus(
DatabaseRecord databaseRecord,
PeriodicBackupConfiguration configuration,
bool skipErrorLog = false)
{
if (configuration.Disabled)
{
return(TaskStatus.Disabled);
}
if (configuration.HasBackup() == false)
{
if (skipErrorLog == false)
{
var message = $"All backup destinations are disabled for backup task id: {configuration.TaskId}";
_database.NotificationCenter.Add(AlertRaised.Create(
"Periodic Backup",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Info));
}
return(TaskStatus.Disabled);
}
var whoseTaskIsIt = databaseRecord.Topology.WhoseTaskIsIt(configuration, _serverStore.IsPassive());
if (whoseTaskIsIt == null)
{
return(TaskStatus.Disabled);
}
if (whoseTaskIsIt == _serverStore.NodeTag)
{
return(TaskStatus.ActiveByCurrentNode);
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Backup job is skipped at {SystemTime.UtcNow}, because it is managed " +
$"by '{whoseTaskIsIt}' node and not the current node ({_serverStore.NodeTag})");
}
return(TaskStatus.ActiveByOtherNode);
}
private static IEnumerable <OngoingTask> CollectSubscriptionTasks(TransactionOperationContext context, DatabaseRecord databaseRecord, ClusterTopology clusterTopology, ServerStore store)
{
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(databaseRecord.DatabaseName)))
{
var subscriptionState = JsonDeserializationClient.SubscriptionState(keyValue.Value);
var tag = databaseRecord.Topology.WhoseTaskIsIt(subscriptionState, store.IsPassive());
yield return(new OngoingTaskSubscription
{
// Supply only needed fields for List View
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
TaskName = subscriptionState.SubscriptionName,
TaskState = subscriptionState.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
TaskId = subscriptionState.SubscriptionId,
Query = subscriptionState.Query
});
}
}
private void HandleTopologyChange(DatabaseRecord newRecord)
{
var instancesToDispose = new List <OutgoingReplicationHandler>();
if (newRecord == null || _server.IsPassive())
{
DropOutgoingConnections(Destinations, ref instancesToDispose);
_internalDestinations.Clear();
_externalDestinations.Clear();
_destinations.Clear();
DisposeConnections(instancesToDispose);
return;
}
HandleInternalReplication(newRecord, ref instancesToDispose);
HandleExternalReplication(newRecord, ref instancesToDispose);
var destinations = new List <ReplicationNode>();
destinations.AddRange(_internalDestinations);
destinations.AddRange(_externalDestinations);
_destinations = destinations;
DisposeConnections(instancesToDispose);
}
public Task GetTopology()
{
var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
var dbId = Constants.Documents.Prefix + name;
using (context.OpenReadTransaction())
using (var dbBlit = ServerStore.Cluster.Read(context, dbId, out long _))
{
if (TryGetAllowedDbs(name, out var _, requireAdmin: false) == false)
{
return(Task.CompletedTask);
}
if (dbBlit == null)
{
// here we return 503 so clients will try to failover to another server
// if this is a newly created db that we haven't been notified about it yet
HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
HttpContext.Response.Headers["Database-Missing"] = name;
using (var writer = new BlittableJsonTextWriter(context, HttpContext.Response.Body))
{
context.Write(writer,
new DynamicJsonValue
{
["Type"] = "Error",
["Message"] = "Database " + name + " wasn't found"
});
}
return(Task.CompletedTask);
}
var clusterTopology = ServerStore.GetClusterTopology(context);
if (ServerStore.IsPassive() && clusterTopology.TopologyId != null)
{
// we were kicked-out from the cluster
HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
return(Task.CompletedTask);
}
clusterTopology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext);
var dbRecord = JsonDeserializationCluster.DatabaseRecord(dbBlit);
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(Topology.Nodes)] = new DynamicJsonArray(
dbRecord.Topology.Members.Select(x => new DynamicJsonValue
{
[nameof(ServerNode.Url)] = GetUrl(x, clusterTopology),
[nameof(ServerNode.ClusterTag)] = x,
[nameof(ServerNode.ServerRole)] = ServerNode.Role.Member,
[nameof(ServerNode.Database)] = dbRecord.DatabaseName
})
.Concat(dbRecord.Topology.Rehabs.Select(x => new DynamicJsonValue
{
[nameof(ServerNode.Url)] = GetUrl(x, clusterTopology),
[nameof(ServerNode.ClusterTag)] = x,
[nameof(ServerNode.Database)] = dbRecord.DatabaseName,
[nameof(ServerNode.ServerRole)] = ServerNode.Role.Rehab
})
)
),
[nameof(Topology.Etag)] = dbRecord.Topology.Stamp?.Index ?? -1
});
}
}
}
return(Task.CompletedTask);
}
public async Task GetTopology()
{
var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");
var applicationIdentifier = GetStringQueryString("applicationIdentifier", required: false);
if (applicationIdentifier != null)
{
AlertIfDocumentStoreCreationRateIsNotReasonable(applicationIdentifier, name);
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
using (context.OpenReadTransaction())
using (var rawRecord = ServerStore.Cluster.ReadRawDatabaseRecord(context, name))
{
if (await CanAccessDatabaseAsync(name, requireAdmin: false) == false)
{
return;
}
if (rawRecord == null)
{
// here we return 503 so clients will try to failover to another server
// if this is a newly created db that we haven't been notified about it yet
HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
HttpContext.Response.Headers["Database-Missing"] = name;
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer,
new DynamicJsonValue
{
["Type"] = "Error",
["Message"] = "Database " + name + " wasn't found"
});
}
return;
}
var clusterTopology = ServerStore.GetClusterTopology(context);
if (ServerStore.IsPassive() && clusterTopology.TopologyId != null)
{
// we were kicked-out from the cluster
HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
return;
}
if (rawRecord.Topology.Members.Count == 0 && rawRecord.Topology.Rehabs.Count == 0 && rawRecord.DeletionInProgress.Any())
{
// The database at deletion progress from all nodes
HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
HttpContext.Response.Headers["Database-Missing"] = name;
await using (var writer = new AsyncBlittableJsonTextWriter(context, HttpContext.Response.Body))
{
context.Write(writer, new DynamicJsonValue
{
["Type"] = "Error",
["Message"] = "Database " + name + " was deleted"
});
}
return;
}
clusterTopology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext);
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(Topology.Nodes)] = new DynamicJsonArray(
rawRecord.Topology.Members.Select(x => new DynamicJsonValue
{
[nameof(ServerNode.Url)] = GetUrl(x, clusterTopology),
[nameof(ServerNode.ClusterTag)] = x,
[nameof(ServerNode.ServerRole)] = ServerNode.Role.Member,
[nameof(ServerNode.Database)] = rawRecord.DatabaseName
})
.Concat(rawRecord.Topology.Rehabs.Select(x => new DynamicJsonValue
{
[nameof(ServerNode.Url)] = GetUrl(x, clusterTopology),
[nameof(ServerNode.ClusterTag)] = x,
[nameof(ServerNode.Database)] = rawRecord.DatabaseName,
[nameof(ServerNode.ServerRole)] = ServerNode.Role.Rehab
})
)
),
[nameof(Topology.Etag)] = rawRecord.Topology.Stamp?.Index ?? -1
});
}
}
}
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
var watcher = GetBoolValueQueryString("watcher", false);
var assignedCores = GetIntValueQueryString("assignedCores", false);
if (assignedCores <= 0)
{
throw new ArgumentException("Assigned cores must be greater than 0!");
}
nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
{
throw new ArgumentException("Cannot add node because the assigned cores is larger " +
$"than the available cores on that machine: {nodeInfo.NumberOfCores}");
}
ServerStore.EnsureNotPassive();
if (assignedCores == null)
{
assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
}
Debug.Assert(assignedCores <= nodeInfo.NumberOfCores);
ServerStore.LicenseManager.AssertCanAddNode(nodeUrl, assignedCores.Value);
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
ClusterTopology clusterTopology;
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
topologyId = clusterTopology.TopologyId;
}
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
if (nodeInfo.ServerId == ServerStore.GetServerId())
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
}
if (nodeInfo.TopologyId != null)
{
if (topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. " +
$"Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
if (nodeInfo.CurrentState != RachisState.Passive)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " +
$"because it's already in the cluster under tag :{nodeInfo.NodeTag} " +
$"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}");
}
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);
var now = DateTime.UtcNow;
if (certificate.NotBefore.ToUniversalTime() > now)
{
// Because of time zone and time drift issues, we can't assume that the certificate generation will be
// proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
// current time. Clients may still fail to work with our certificate because of timing issues,
// but the admin needs to setup time sync properly and there isn't much we can do at that point
if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
{
throw new InvalidOperationException(
$"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
}
if (certificate.NotAfter.ToUniversalTime() < now)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbprint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
using (ctx.OpenReadTransaction())
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var readCert = ServerStore.Cluster.Read(ctx, key);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
NotAfter = certificate.NotAfter,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null)
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
var nodeDetails = new NodeDetails
{
NodeTag = nodeTag,
AssignedCores = assignedCores.Value,
NumberOfCores = nodeInfo.NumberOfCores,
InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
UsableMemoryInGb = nodeInfo.UsableMemoryInGb,
BuildInfo = nodeInfo.BuildInfo
};
await ServerStore.LicenseManager.CalculateLicenseLimits(nodeDetails, forceFetchingNodeInfo : true, waitToUpdate : true);
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
private IEnumerable <EtlProcess> GetRelevantProcesses <T, TConnectionString>(List <T> configurations, HashSet <string> uniqueNames) where T : EtlConfiguration <TConnectionString> where TConnectionString : ConnectionString
{
foreach (var config in configurations)
{
SqlEtlConfiguration sqlConfig = null;
RavenEtlConfiguration ravenConfig = null;
var connectionStringNotFound = false;
switch (config.EtlType)
{
case EtlType.Raven:
ravenConfig = config as RavenEtlConfiguration;
if (_databaseRecord.RavenConnectionStrings.TryGetValue(config.ConnectionStringName, out var ravenConnection))
{
ravenConfig.Initialize(ravenConnection);
}
else
{
connectionStringNotFound = true;
}
break;
case EtlType.Sql:
sqlConfig = config as SqlEtlConfiguration;
if (_databaseRecord.SqlConnectionStrings.TryGetValue(config.ConnectionStringName, out var sqlConnection))
{
sqlConfig.Initialize(sqlConnection);
}
else
{
connectionStringNotFound = true;
}
break;
default:
ThrownUnknownEtlConfiguration(config.GetType());
break;
}
if (connectionStringNotFound)
{
LogConfigurationError(config,
new List <string>
{
$"Connection string named '{config.ConnectionStringName}' was not found for {config.EtlType} ETL"
});
continue;
}
if (ValidateConfiguration(config, uniqueNames) == false)
{
continue;
}
if (config.Disabled)
{
continue;
}
if (_databaseRecord.Topology.WhoseTaskIsIt(config, _serverStore.IsPassive()) != _serverStore.NodeTag)
{
continue;
}
foreach (var transform in config.Transforms)
{
if (transform.Disabled)
{
continue;
}
EtlProcess process = null;
if (sqlConfig != null)
{
process = new SqlEtl(transform, sqlConfig, _database, _serverStore);
}
if (ravenConfig != null)
{
process = new RavenEtl(transform, ravenConfig, _database, _serverStore);
}
yield return(process);
}
}
}
private void WriteDatabaseInfo(string databaseName, BlittableJsonReaderObject dbRecordBlittable,
TransactionOperationContext context, BlittableJsonTextWriter writer)
{
var online = ServerStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out Task <DocumentDatabase> dbTask) &&
dbTask != null &&
dbTask.IsCompleted;
// Check for exceptions
if (dbTask != null && dbTask.IsFaulted)
{
WriteFaultedDatabaseInfo(context, writer, dbTask, databaseName);
return;
}
var dbRecord = JsonDeserializationCluster.DatabaseRecord(dbRecordBlittable);
var db = online ? dbTask.Result : null;
var indexingStatus = db?.IndexStore.Status ?? IndexRunningStatus.Running;
// Looking for disabled indexing flag inside the database settings for offline database status
if (dbRecord.Settings.TryGetValue(RavenConfiguration.GetKey(x => x.Indexing.Disabled), out var val) && val == "true")
{
indexingStatus = IndexRunningStatus.Disabled;
}
var disabled = dbRecord.Disabled;
var topology = dbRecord.Topology;
var clusterTopology = ServerStore.GetClusterTopology(context);
var nodesTopology = new NodesTopology();
if (topology != null)
{
foreach (var member in topology.Members)
{
var url = clusterTopology.GetUrlFromTag(member);
var node = new InternalReplication
{
Database = databaseName,
NodeTag = member,
Url = url
};
nodesTopology.Members.Add(GetNodeId(node));
SetNodeStatus(topology, member, nodesTopology);
}
foreach (var promotable in topology.Promotables)
{
var node = GetNode(databaseName, clusterTopology, promotable, out var promotableTask);
var mentor = topology.WhoseTaskIsIt(promotableTask, ServerStore.IsPassive());
nodesTopology.Promotables.Add(GetNodeId(node, mentor));
SetNodeStatus(topology, promotable, nodesTopology);
}
foreach (var rehab in topology.Rehabs)
{
var node = GetNode(databaseName, clusterTopology, rehab, out var promotableTask);
var mentor = topology.WhoseTaskIsIt(promotableTask, ServerStore.IsPassive());
nodesTopology.Rehabs.Add(GetNodeId(node, mentor));
SetNodeStatus(topology, rehab, nodesTopology);
}
}
if (online == false)
{
// If state of database is found in the cache we can continue
if (ServerStore.DatabaseInfoCache.TryWriteOfflineDatabaseStatusToRequest(
context, writer, databaseName, disabled, indexingStatus, nodesTopology))
{
return;
}
// We won't find it if it is a new database or after a dirty shutdown, so just report empty values then
}
var size = new Size(GetTotalSize(db));
var databaseInfo = new DatabaseInfo
{
Name = databaseName,
Disabled = disabled,
TotalSize = size,
IsAdmin = true, //TODO: implement me!
IsEncrypted = dbRecord.Encrypted,
UpTime = online ? (TimeSpan?)GetUptime(db) : null,
BackupInfo = GetBackupInfo(db),
Alerts = db?.NotificationCenter.GetAlertCount() ?? 0,
RejectClients = false, //TODO: implement me!
LoadError = null,
IndexingErrors = db?.IndexStore.GetIndexes().Sum(index => index.GetErrorCount()) ?? 0,
DocumentsCount = db?.DocumentsStorage.GetNumberOfDocuments() ?? 0,
HasRevisionsConfiguration = db?.DocumentsStorage.RevisionsStorage.Configuration != null,
HasExpirationConfiguration = db?.ExpiredDocumentsCleaner != null,
IndexesCount = db?.IndexStore.GetIndexes().Count() ?? 0,
IndexingStatus = indexingStatus,
NodesTopology = nodesTopology,
ReplicationFactor = topology?.ReplicationFactor ?? -1,
DynamicNodesDistribution = topology?.DynamicNodesDistribution ?? false
};
var doc = databaseInfo.ToJson();
context.Write(writer, doc);
}
public async Task AddNode()
{
var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
var tag = GetStringQueryString("tag", false);
var watcher = GetBoolValueQueryString("watcher", false);
var raftRequestId = GetRaftRequestIdFromQuery();
var maxUtilizedCores = GetIntValueQueryString("maxUtilizedCores", false);
if (maxUtilizedCores != null && maxUtilizedCores <= 0)
{
throw new ArgumentException("Max utilized cores cores must be greater than 0");
}
nodeUrl = nodeUrl.Trim();
if (Uri.IsWellFormedUriString(nodeUrl, UriKind.Absolute) == false)
{
throw new InvalidOperationException($"Given node URL '{nodeUrl}' is not in a correct format.");
}
nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
tag = tag?.Trim();
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
// test connection to remote.
var result = await ServerStore.TestConnectionToRemote(nodeUrl, database : null);
if (result.Success == false)
{
throw new InvalidOperationException(result.Error);
}
// test connection from remote to destination
result = await ServerStore.TestConnectionFromRemote(requestExecutor, ctx, nodeUrl);
if (result.Success == false)
{
throw new InvalidOperationException(result.Error);
}
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (SchemaUpgrader.CurrentVersion.ServerVersion != nodeInfo.ServerSchemaVersion)
{
var nodesVersion = nodeInfo.ServerSchemaVersion == 0 ? "Pre 4.2 version" : nodeInfo.ServerSchemaVersion.ToString();
throw new InvalidOperationException($"Can't add node with mismatched storage schema version.{Environment.NewLine}" +
$"My version is {SchemaUpgrader.CurrentVersion.ServerVersion}, while node's version is {nodesVersion}");
}
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (ServerStore.ValidateFixedPort && nodeInfo.HasFixedPort == false)
{
throw new InvalidOperationException($"Failed to add node '{nodeUrl}' to cluster. " +
$"Node '{nodeUrl}' has port '0' in 'Configuration.Core.ServerUrls' setting. " +
"Adding a node with non fixed port is forbidden. Define a fixed port for the node to enable cluster creation.");
}
await ServerStore.EnsureNotPassiveAsync();
ServerStore.LicenseManager.AssertCanAddNode();
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
var clusterTopology = ServerStore.GetClusterTopology();
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
if (nodeInfo.ServerId == ServerStore.GetServerId())
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
}
if (nodeInfo.TopologyId != null)
{
AssertCanAddNodeWithTopologyId(clusterTopology, nodeInfo, nodeUrl);
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? tag : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);
var now = DateTime.UtcNow;
if (certificate.NotBefore.ToUniversalTime() > now)
{
// Because of time zone and time drift issues, we can't assume that the certificate generation will be
// proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
// current time. Clients may still fail to work with our certificate because of timing issues,
// but the admin needs to setup time sync properly and there isn't much we can do at that point
if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
{
throw new InvalidOperationException(
$"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
}
if (certificate.NotAfter.ToUniversalTime() < now)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbprint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
// if it's the same server certificate as our own, we don't want to add it to the cluster
if (certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
{
using (ctx.OpenReadTransaction())
{
var readCert = ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
PublicKeyPinningHash = certificate.GetPublicKeyPinningHash(),
NotAfter = certificate.NotAfter,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, certificateDefinition,
$"{raftRequestId}/put-new-certificate"));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : true, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null && certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
{
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, modifiedServerCert, $"{raftRequestId}/put-modified-certificate"));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
var detailsPerNode = new DetailsPerNode
{
MaxUtilizedCores = maxUtilizedCores,
NumberOfCores = nodeInfo.NumberOfCores,
InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
UsableMemoryInGb = nodeInfo.UsableMemoryInGb,
BuildInfo = nodeInfo.BuildInfo,
OsInfo = nodeInfo.OsInfo
};
var maxCores = ServerStore.LicenseManager.LicenseStatus.MaxCores;
try
{
await ServerStore.PutNodeLicenseLimitsAsync(nodeTag, detailsPerNode, maxCores, $"{raftRequestId}/put-license-limits");
}
catch
{
// we'll retry this again later
}
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetStringQueryString("url").TrimEnd('/');
var watcher = GetBoolValueQueryString("watcher", false);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.ClusterCertificateHolder.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
ServerStore.EnsureNotPassive();
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
topologyId = clusterTopology.TopologyId;
}
if (nodeInfo.TopologyId != null && topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag
? null : nodeInfo.NodeTag;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because it has no certificate while trying to use HTTPS");
}
var certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate));
if (certificate.NotBefore > DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
if (certificate.NotAfter < DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbrpint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
NoContentStatus();
return;
}
}
RedirectToLeader();
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetStringQueryString("url").TrimEnd('/');
var watcher = GetBoolValueQueryString("watcher", false);
var assignedCores = GetIntValueQueryString("assignedCores", false);
if (assignedCores <= 0)
{
throw new ArgumentException("Assigned cores must be greater than 0!");
}
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.ClusterCertificateHolder.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (assignedCores == null)
{
assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
}
if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
{
throw new ArgumentException("Cannot add node because the assigned cores is larger " +
$"than the available cores on that machine: {nodeInfo.NumberOfCores}");
}
ServerStore.EnsureNotPassive();
if (ServerStore.LicenseManager.CanAddNode(nodeUrl, assignedCores, out var licenseLimit) == false)
{
SetLicenseLimitResponse(licenseLimit);
return;
}
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
topologyId = clusterTopology.TopologyId;
}
if (nodeInfo.TopologyId != null && topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate));
if (certificate.NotBefore > DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
if (certificate.NotAfter < DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbrpint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
using (ctx.OpenReadTransaction())
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var readCert = ServerStore.Cluster.Read(ctx, key);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null)
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
ServerStore.LicenseManager.CalculateLicenseLimits(nodeTag, assignedCores, nodeInfo, forceFetchingNodeInfo: true);
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
public Task GetOngoingTaskInfo()
{
if (ResourceNameValidator.IsValidResourceName(Database.Name, ServerStore.Configuration.Core.DataDirectory.FullPath, out string errorMessage) == false)
{
throw new BadRequestException(errorMessage);
}
var key = GetLongQueryString("key");
var typeStr = GetQueryStringValueAndAssertIfSingleAndNotEmpty("type");
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
using (context.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(context);
var record = ServerStore.Cluster.ReadDatabase(context, Database.Name);
var dbTopology = record?.Topology;
if (Enum.TryParse <OngoingTaskType>(typeStr, true, out var type) == false)
{
throw new ArgumentException($"Unknown task type: {type}", "type");
}
string tag;
switch (type)
{
case OngoingTaskType.Replication:
var watcher = record?.ExternalReplication.Find(x => x.TaskId == key);
if (watcher == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
break;
}
tag = dbTopology?.WhoseTaskIsIt(watcher, ServerStore.IsPassive());
var replicationTaskInfo = new OngoingTaskReplication
{
TaskId = watcher.TaskId,
TaskName = watcher.Name,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
DestinationDatabase = watcher.Database,
TaskState = watcher.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
DestinationUrl = watcher.Url
};
WriteResult(context, replicationTaskInfo);
break;
case OngoingTaskType.Backup:
var backupConfiguration = record?.PeriodicBackups?.Find(x => x.TaskId == key);
if (backupConfiguration == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
break;
}
tag = dbTopology?.WhoseTaskIsIt(backupConfiguration, ServerStore.IsPassive());
var backupDestinations = GetBackupDestinations(backupConfiguration);
var backupStatus = Database.PeriodicBackupRunner.GetBackupStatus(key);
var nextBackup = Database.PeriodicBackupRunner.GetNextBackupDetails(record, backupConfiguration, backupStatus);
var backupTaskInfo = new OngoingTaskBackup
{
TaskId = backupConfiguration.TaskId,
BackupType = backupConfiguration.BackupType,
TaskName = backupConfiguration.Name,
TaskState = backupConfiguration.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
BackupDestinations = backupDestinations,
LastFullBackup = backupStatus.LastFullBackup,
LastIncrementalBackup = backupStatus.LastIncrementalBackup,
NextBackup = nextBackup
};
WriteResult(context, backupTaskInfo);
break;
case OngoingTaskType.SqlEtl:
var sqlEtl = record?.SqlEtls?.Find(x => x.TaskId == key);
if (sqlEtl == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
break;
}
WriteResult(context, new OngoingTaskSqlEtl
{
TaskId = sqlEtl.TaskId,
TaskName = sqlEtl.Name,
Configuration = sqlEtl
});
break;
case OngoingTaskType.RavenEtl:
var ravenEtl = record?.RavenEtls?.Find(x => x.TaskId == key);
if (ravenEtl == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
break;
}
WriteResult(context, new OngoingTaskRavenEtl
{
TaskId = ravenEtl.TaskId,
TaskName = ravenEtl.Name,
Configuration = ravenEtl
});
break;
case OngoingTaskType.Subscription:
var nameKey = GetQueryStringValueAndAssertIfSingleAndNotEmpty("taskName");
var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(record.DatabaseName, nameKey);
var doc = ServerStore.Cluster.Read(context, itemKey);
if (doc == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
break;
}
var subscriptionState = JsonDeserializationClient.SubscriptionState(doc);
tag = dbTopology?.WhoseTaskIsIt(subscriptionState, ServerStore.IsPassive());
var subscriptionStateInfo = new SubscriptionStateWithNodeDetails
{
Query = subscriptionState.Query,
ChangeVectorForNextBatchStartingPoint = subscriptionState.ChangeVectorForNextBatchStartingPoint,
SubscriptionId = subscriptionState.SubscriptionId,
SubscriptionName = subscriptionState.SubscriptionName,
LastTimeServerMadeProgressWithDocuments = subscriptionState.LastTimeServerMadeProgressWithDocuments,
Disabled = subscriptionState.Disabled,
LastClientConnectionTime = subscriptionState.LastClientConnectionTime,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
}
};
// Todo: here we'll need to talk with the running node? TaskConnectionStatus = subscriptionState.Disabled ? OngoingTaskConnectionStatus.NotActive : OngoingTaskConnectionStatus.Active,
WriteResult(context, subscriptionStateInfo.ToJson());
break;
default:
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
break;
}
}
}
return(Task.CompletedTask);
}
private static IEnumerable <OngoingTask> CollectEtlTasks(DatabaseRecord databaseRecord, DatabaseTopology dbTopology, ClusterTopology clusterTopology, ServerStore store)
{
if (dbTopology == null)
{
yield break;
}
if (databaseRecord.RavenEtls != null)
{
foreach (var ravenEtl in databaseRecord.RavenEtls)
{
var tag = dbTopology.WhoseTaskIsIt(ravenEtl, store.IsPassive());
var taskState = OngoingTaskState.Enabled;
if (ravenEtl.Disabled || ravenEtl.Transforms.All(x => x.Disabled))
{
taskState = OngoingTaskState.Disabled;
}
else if (ravenEtl.Transforms.Any(x => x.Disabled))
{
taskState = OngoingTaskState.PartiallyEnabled;
}
if (databaseRecord.RavenConnectionStrings.TryGetValue(ravenEtl.ConnectionStringName, out var connection) == false)
{
throw new InvalidOperationException(
$"Could not find connection string named '{ravenEtl.ConnectionStringName}' in the database record for '{ravenEtl.Name}' ETL");
}
yield return(new OngoingTaskRavenEtl
{
TaskId = ravenEtl.TaskId,
TaskName = ravenEtl.Name,
// TODO arek TaskConnectionStatus =
TaskState = taskState,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
DestinationUrl = connection.Url,
DestinationDatabase = connection.Database
});
}
}
if (databaseRecord.SqlEtls != null)
{
foreach (var sqlEtl in databaseRecord.SqlEtls)
{
var tag = dbTopology.WhoseTaskIsIt(sqlEtl, store.IsPassive());
var taskState = OngoingTaskState.Enabled;
if (sqlEtl.Disabled || sqlEtl.Transforms.All(x => x.Disabled))
{
taskState = OngoingTaskState.Disabled;
}
else if (sqlEtl.Transforms.Any(x => x.Disabled))
{
taskState = OngoingTaskState.PartiallyEnabled;
}
if (databaseRecord.SqlConnectionStrings.TryGetValue(sqlEtl.ConnectionStringName, out var sqlConnection) == false)
{
throw new InvalidOperationException(
$"Could not find connection string named '{sqlEtl.ConnectionStringName}' in the database record for '{sqlEtl.Name}' ETL");
}
var(database, server) =
SqlConnectionStringParser.GetDatabaseAndServerFromConnectionString(sqlEtl.FactoryName, sqlConnection.ConnectionString);
yield return(new OngoingTaskSqlEtl
{
TaskId = sqlEtl.TaskId,
TaskName = sqlEtl.Name,
// TODO arek TaskConnectionStatus =
TaskState = taskState,
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
DestinationServer = server,
DestinationDatabase = database
});
}
}
}