public async Task AddNode()
{
var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
var tag = GetStringQueryString("tag", false);
var watcher = GetBoolValueQueryString("watcher", false);
var raftRequestId = GetRaftRequestIdFromQuery();
var assignedCores = GetIntValueQueryString("assignedCores", false);
if (assignedCores <= 0)
{
throw new ArgumentException("Assigned cores must be greater than 0!");
}
nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
// test connection to remote.
var result = await ServerStore.TestConnectionToRemote(nodeUrl, database : null);
if (result.Success == false)
{
throw new InvalidOperationException(result.Error);
}
// test connection from remote to destination
result = await ServerStore.TestConnectionFromRemote(requestExecutor, ctx, nodeUrl);
if (result.Success == false)
{
throw new InvalidOperationException(result.Error);
}
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (SchemaUpgrader.CurrentVersion.ServerVersion != nodeInfo.ServerSchemaVersion)
{
var nodesVersion = nodeInfo.ServerSchemaVersion == 0 ? "Pre 4.2 version" : nodeInfo.ServerSchemaVersion.ToString();
throw new InvalidOperationException($"Can't add node with mismatched storage schema version.{Environment.NewLine}" +
$"My version is {SchemaUpgrader.CurrentVersion.ServerVersion}, while node's version is {nodesVersion}");
}
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
{
throw new ArgumentException("Cannot add node because the assigned cores is larger " +
$"than the available cores on that machine: {nodeInfo.NumberOfCores}");
}
if (ServerStore.ValidateFixedPort && nodeInfo.HasFixedPort == false)
{
throw new InvalidOperationException($"Failed to add node '{nodeUrl}' to cluster. " +
$"Node '{nodeUrl}' has port '0' in 'Configuration.Core.ServerUrls' setting. " +
"Adding a node with non fixed port is forbidden. Define a fixed port for the node to enable cluster creation.");
}
ServerStore.EnsureNotPassive();
if (assignedCores == null)
{
assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
}
Debug.Assert(assignedCores <= nodeInfo.NumberOfCores);
ServerStore.LicenseManager.AssertCanAddNode(nodeUrl, assignedCores.Value);
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
ClusterTopology clusterTopology;
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
topologyId = clusterTopology.TopologyId;
}
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
if (nodeInfo.ServerId == ServerStore.GetServerId())
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
}
if (nodeInfo.TopologyId != null)
{
if (topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. " +
$"Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
if (nodeInfo.CurrentState != RachisState.Passive)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " +
$"because it's already in the cluster under tag :{nodeInfo.NodeTag} " +
$"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}");
}
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? tag : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);
var now = DateTime.UtcNow;
if (certificate.NotBefore.ToUniversalTime() > now)
{
// Because of time zone and time drift issues, we can't assume that the certificate generation will be
// proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
// current time. Clients may still fail to work with our certificate because of timing issues,
// but the admin needs to setup time sync properly and there isn't much we can do at that point
if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
{
throw new InvalidOperationException(
$"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
}
if (certificate.NotAfter.ToUniversalTime() < now)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbprint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
// if it's the same server certificate as our own, we don't want to add it to the cluster
if (certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
{
using (ctx.OpenReadTransaction())
{
var readCert = ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
PublicKeyPinningHash = certificate.GetPublicKeyPinningHash(),
NotAfter = certificate.NotAfter,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, certificateDefinition,
$"{raftRequestId}/put-new-certificate"));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : true, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null && certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
{
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, modifiedServerCert, $"{raftRequestId}/put-modified-certificate"));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
var nodeDetails = new NodeDetails
{
NodeTag = nodeTag,
AssignedCores = assignedCores.Value,
NumberOfCores = nodeInfo.NumberOfCores,
InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
UsableMemoryInGb = nodeInfo.UsableMemoryInGb,
BuildInfo = nodeInfo.BuildInfo,
OsInfo = nodeInfo.OsInfo
};
await ServerStore.LicenseManager.CalculateLicenseLimits(nodeDetails, forceFetchingNodeInfo : true);
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
public Task Bootstrap()
{
ServerStore.EnsureNotPassive();
return(NoContent());
}
public async Task Put()
{
// one of the first admin action is to create a certificate, so let
// us also use that to indicate that we are the seed node
ServerStore.EnsureNotPassive();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var certificateJson = ctx.ReadForDisk(RequestBodyStream(), "put-certificate"))
{
var certificate = JsonDeserializationServer.CertificateDefinition(certificateJson);
ValidateCertificate(certificate, ServerStore);
if ((certificate.SecurityClearance == SecurityClearance.ClusterAdmin || certificate.SecurityClearance == SecurityClearance.ClusterNode) && IsClusterAdmin() == false)
{
var clientCert = (HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection)?.Certificate;
var clientCertDef = ReadCertificateFromCluster(ctx, Constants.Certificates.Prefix + clientCert?.Thumbprint);
throw new InvalidOperationException($"Cannot save the certificate '{certificate.Name}' with '{certificate.SecurityClearance}' security clearance because the current client certificate being used has a lower clearance: {clientCertDef.SecurityClearance}");
}
if (string.IsNullOrWhiteSpace(certificate.Certificate))
{
throw new ArgumentException($"{nameof(certificate.Certificate)} is a mandatory property when saving an existing certificate");
}
byte[] certBytes;
try
{
certBytes = Convert.FromBase64String(certificate.Certificate);
}
catch (Exception e)
{
throw new ArgumentException($"Unable to parse the {nameof(certificate.Certificate)} property, expected a Base64 value", e);
}
var collection = new X509Certificate2Collection();
collection.Import(certBytes);
var first = true;
var collectionPrimaryKey = string.Empty;
foreach (var x509Certificate in collection)
{
var currentCertificate = new CertificateDefinition
{
Name = certificate.Name,
Permissions = certificate.Permissions,
SecurityClearance = certificate.SecurityClearance,
Password = certificate.Password
};
if (x509Certificate.HasPrivateKey)
{
// avoid storing the private key
currentCertificate.Certificate = Convert.ToBase64String(x509Certificate.Export(X509ContentType.Cert));
}
// In case of a collection, we group all the certificates together and treat them as one unit.
// They all have the same name and permissions but a different thumbprint.
// The first certificate in the collection will be the primary certificate and its thumbprint will be the one shown in a GET request
// The other certificates are secondary certificates and will contain a link to the primary certificate.
currentCertificate.Thumbprint = x509Certificate.Thumbprint;
currentCertificate.NotAfter = x509Certificate.NotAfter;
currentCertificate.Certificate = Convert.ToBase64String(x509Certificate.Export(X509ContentType.Cert));
if (first)
{
var firstKey = Constants.Certificates.Prefix + x509Certificate.Thumbprint;
collectionPrimaryKey = firstKey;
foreach (var cert in collection)
{
if (Constants.Certificates.Prefix + cert.Thumbprint != firstKey)
{
currentCertificate.CollectionSecondaryKeys.Add(Constants.Certificates.Prefix + cert.Thumbprint);
}
}
}
else
{
currentCertificate.CollectionPrimaryKey = collectionPrimaryKey;
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + x509Certificate.Thumbprint, currentCertificate));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
first = false;
}
NoContentStatus();
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
}
}
public async Task Edit()
{
ServerStore.EnsureNotPassive();
var feature = HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection;
var clientCert = feature?.Certificate;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var certificateJson = ctx.ReadForDisk(RequestBodyStream(), "edit-certificate"))
{
var newCertificate = JsonDeserializationServer.CertificateDefinition(certificateJson);
ValidateCertificate(newCertificate, ServerStore);
var key = Constants.Certificates.Prefix + newCertificate.Thumbprint;
CertificateDefinition existingCertificate;
using (ctx.OpenWriteTransaction())
{
var certificate = ServerStore.Cluster.Read(ctx, key);
if (certificate == null)
{
throw new InvalidOperationException($"Cannot edit permissions for certificate with thumbprint '{newCertificate.Thumbprint}'. It doesn't exist in the cluster.");
}
existingCertificate = JsonDeserializationServer.CertificateDefinition(certificate);
if ((existingCertificate.SecurityClearance == SecurityClearance.ClusterAdmin || existingCertificate.SecurityClearance == SecurityClearance.ClusterNode) && IsClusterAdmin() == false)
{
var clientCertDef = ReadCertificateFromCluster(ctx, Constants.Certificates.Prefix + clientCert?.Thumbprint);
throw new InvalidOperationException($"Cannot edit the certificate '{existingCertificate.Name}'. It has '{existingCertificate.SecurityClearance}' security clearance while the current client certificate being used has a lower clearance: {clientCertDef.SecurityClearance}");
}
if ((newCertificate.SecurityClearance == SecurityClearance.ClusterAdmin || newCertificate.SecurityClearance == SecurityClearance.ClusterNode) && IsClusterAdmin() == false)
{
var clientCertDef = ReadCertificateFromCluster(ctx, Constants.Certificates.Prefix + clientCert?.Thumbprint);
throw new InvalidOperationException($"Cannot edit security clearance to '{newCertificate.SecurityClearance}' for certificate '{existingCertificate.Name}'. Only a 'Cluster Admin' can do that and your current client certificate has a lower clearance: {clientCertDef.SecurityClearance}");
}
ServerStore.Cluster.DeleteLocalState(ctx, key);
}
var deleteResult = await ServerStore.SendToLeaderAsync(new DeleteCertificateFromClusterCommand
{
Name = Constants.Certificates.Prefix + existingCertificate.Thumbprint
});
await ServerStore.Cluster.WaitForIndexNotification(deleteResult.Index);
var putResult = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + newCertificate.Thumbprint,
new CertificateDefinition
{
Name = newCertificate.Name,
Certificate = existingCertificate.Certificate,
Permissions = newCertificate.Permissions,
SecurityClearance = newCertificate.SecurityClearance,
Thumbprint = existingCertificate.Thumbprint,
NotAfter = existingCertificate.NotAfter
}));
await ServerStore.Cluster.WaitForIndexNotification(putResult.Index);
NoContentStatus();
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
}
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetStringQueryString("url").TrimEnd('/');
var watcher = GetBoolValueQueryString("watcher", false);
var assignedCores = GetIntValueQueryString("assignedCores", false);
if (assignedCores <= 0)
{
throw new ArgumentException("Assigned cores must be greater than 0!");
}
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
{
throw new ArgumentException("Cannot add node because the assigned cores is larger " +
$"than the available cores on that machine: {nodeInfo.NumberOfCores}");
}
ServerStore.EnsureNotPassive();
if (assignedCores == null)
{
assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
}
Debug.Assert(assignedCores <= nodeInfo.NumberOfCores);
ServerStore.LicenseManager.AssertCanAddNode(nodeUrl, assignedCores.Value);
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
ClusterTopology clusterTopology;
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
topologyId = clusterTopology.TopologyId;
}
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
if (nodeInfo.ServerId == ServerStore.GetServerId())
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
}
if (nodeInfo.TopologyId != null)
{
if (topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. " +
$"Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
if (nodeInfo.CurrentState != RachisState.Passive)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " +
$"because it's already in the cluster under tag :{nodeInfo.NodeTag} " +
$"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}");
}
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate));
var now = DateTime.UtcNow;
if (certificate.NotBefore.ToUniversalTime() > now)
{
// Because of time zone and time drift issues, we can't assume that the certificate generation will be
// proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
// current time. Clients may still fail to work with our certificate because of timing issues,
// but the admin needs to setup time sync properly and there isn't much we can do at that point
if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
{
throw new InvalidOperationException(
$"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
}
if (certificate.NotAfter.ToUniversalTime() < now)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbprint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
using (ctx.OpenReadTransaction())
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var readCert = ServerStore.Cluster.Read(ctx, key);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
NotAfter = certificate.NotAfter,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null)
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
var nodeDetails = new NodeDetails
{
NodeTag = nodeTag,
AssignedCores = assignedCores.Value,
NumberOfCores = nodeInfo.NumberOfCores,
InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
UsableMemoryInGb = nodeInfo.UsableMemoryInGb
};
await ServerStore.LicenseManager.CalculateLicenseLimits(nodeDetails, forceFetchingNodeInfo : true, waitToUpdate : true);
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress)
{
var databaseName = _restoreConfiguration.DatabaseName;
var result = new RestoreResult
{
DataDirectory = _restoreConfiguration.DataDirectory
};
try
{
if (onProgress == null)
{
onProgress = _ => { }
}
;
Stopwatch sw = null;
RestoreSettings restoreSettings = null;
var firstFile = _filesToRestore[0];
var lastFile = _filesToRestore.Last();
var extension = Path.GetExtension(firstFile);
var snapshotRestore = false;
if (extension == Constants.Documents.PeriodicBackup.SnapshotExtension)
{
onProgress.Invoke(result.Progress);
snapshotRestore = true;
sw = Stopwatch.StartNew();
// restore the snapshot
restoreSettings = SnapshotRestore(firstFile,
_restoreConfiguration.DataDirectory,
onProgress,
result);
if (restoreSettings != null && _restoreConfiguration.SkipIndexes)
{
// remove all indexes from the database record
restoreSettings.DatabaseRecord.AutoIndexes = null;
restoreSettings.DatabaseRecord.Indexes = null;
}
// removing the snapshot from the list of files
_filesToRestore.RemoveAt(0);
}
else
{
result.SnapshotRestore.Skipped = true;
result.SnapshotRestore.Processed = true;
onProgress.Invoke(result.Progress);
}
if (restoreSettings == null)
{
restoreSettings = new RestoreSettings
{
DatabaseRecord = new DatabaseRecord(databaseName)
{
// we only have a smuggler restore
// use the encryption key to encrypt the database
Encrypted = _hasEncryptionKey
}
};
DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
}
var databaseRecord = restoreSettings.DatabaseRecord;
if (databaseRecord.Settings == null)
{
databaseRecord.Settings = new Dictionary <string, string>();
}
var runInMemoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.RunInMemory);
databaseRecord.Settings.Remove(runInMemoryConfigurationKey);
if (_serverStore.Configuration.Core.RunInMemory)
{
databaseRecord.Settings[runInMemoryConfigurationKey] = "false";
}
var dataDirectoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.DataDirectory);
databaseRecord.Settings.Remove(dataDirectoryConfigurationKey); // removing because we want to restore to given location, not to serialized in backup one
if (_restoringToDefaultDataDirectory == false)
{
databaseRecord.Settings[dataDirectoryConfigurationKey] = _restoreConfiguration.DataDirectory;
}
if (_hasEncryptionKey)
{
// save the encryption key so we'll be able to access the database
_serverStore.PutSecretKey(_restoreConfiguration.EncryptionKey,
databaseName, overwrite: false);
}
var addToInitLog = new Action <string>(txt => // init log is not save in mem during RestoreBackup
{
var msg = $"[RestoreBackup] {DateTime.UtcNow} :: Database '{databaseName}' : {txt}";
if (Logger.IsInfoEnabled)
{
Logger.Info(msg);
}
});
var configuration = _serverStore
.DatabasesLandlord
.CreateDatabaseConfiguration(databaseName, ignoreDisabledDatabase: true, ignoreBeenDeleted: true, ignoreNotRelevant: true, databaseRecord);
using (var database = new DocumentDatabase(databaseName, configuration, _serverStore, addToInitLog))
{
// smuggler needs an existing document database to operate
var options = InitializeOptions.SkipLoadingDatabaseRecord;
if (snapshotRestore)
{
options |= InitializeOptions.GenerateNewDatabaseId;
}
database.Initialize(options);
if (snapshotRestore)
{
result.SnapshotRestore.Processed = true;
var summary = database.GetDatabaseSummary();
result.Documents.ReadCount += summary.DocumentsCount;
result.Documents.Attachments.ReadCount += summary.AttachmentsCount;
result.Counters.ReadCount += summary.CountersCount;
result.RevisionDocuments.ReadCount += summary.RevisionsCount;
result.Conflicts.ReadCount += summary.ConflictsCount;
result.Indexes.ReadCount += databaseRecord.GetIndexesCount();
result.AddInfo($"Successfully restored {result.SnapshotRestore.ReadCount} " +
$"files during snapshot restore, took: {sw.ElapsedMilliseconds:#,#;;0}ms");
onProgress.Invoke(result.Progress);
}
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
SmugglerRestore(_restoreConfiguration.BackupLocation, database, context, databaseRecord, onProgress, result);
result.DatabaseRecord.Processed = true;
result.Documents.Processed = true;
result.RevisionDocuments.Processed = true;
result.Conflicts.Processed = true;
result.Indexes.Processed = true;
result.Counters.Processed = true;
onProgress.Invoke(result.Progress);
databaseRecord.Topology = new DatabaseTopology();
// restoring to the current node only
databaseRecord.Topology.Members.Add(_nodeTag);
databaseRecord.Disabled = true; // we are currently restoring, shouldn't try to access it
_serverStore.EnsureNotPassive();
DisableOngoingTasksIfNeeded(databaseRecord);
var(index, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, restoreSettings.DatabaseValues, isRestore : true);
await _serverStore.Cluster.WaitForIndexNotification(index);
// restore identities & cmpXchg values
RestoreFromLastFile(onProgress, database, lastFile, context, result);
}
}
// after the db for restore is done, we can safely set the db status to active
databaseRecord = _serverStore.LoadDatabaseRecord(databaseName, out _);
databaseRecord.Disabled = false;
var(updateIndex, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null);
await _serverStore.Cluster.WaitForIndexNotification(updateIndex);
if (databaseRecord.Topology.RelevantFor(_serverStore.NodeTag))
{
// we need to wait for the database record change to be propagated properly
var db = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
await db.RachisLogIndexNotifications.WaitForIndexNotification(updateIndex, _operationCancelToken.Token);
}
return(result);
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations("Failed to restore database", e);
}
var alert = AlertRaised.Create(
_restoreConfiguration.DatabaseName,
"Failed to restore database",
$"Could not restore database named {_restoreConfiguration.DatabaseName}",
AlertType.RestoreError,
NotificationSeverity.Error,
details: new ExceptionDetails(e));
_serverStore.NotificationCenter.Add(alert);
if (_serverStore.LoadDatabaseRecord(_restoreConfiguration.DatabaseName, out var _) == null)
{
// delete any files that we already created during the restore
IOExtensions.DeleteDirectory(_restoreConfiguration.DataDirectory);
}
else
{
var deleteResult = await _serverStore.DeleteDatabaseAsync(_restoreConfiguration.DatabaseName, true, new[] { _serverStore.NodeTag });
await _serverStore.Cluster.WaitForIndexNotification(deleteResult.Index);
}
result.AddError($"Error occurred during restore of database {databaseName}. Exception: {e.Message}");
onProgress.Invoke(result.Progress);
throw;
}
finally
{
_operationCancelToken.Dispose();
}
}
public async Task SetupUnsecured()
{
AssertOnlyInSetupMode();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var setupInfoJson = context.ReadForMemory(RequestBodyStream(), "setup-unsecured"))
{
// Making sure we don't have leftovers from previous setup
try
{
using (var tx = context.OpenWriteTransaction())
{
ServerStore.Engine.DeleteTopology(context);
tx.Commit();
}
}
catch (Exception)
{
// ignored
}
var setupInfo = JsonDeserializationServer.UnsecuredSetupInfo(setupInfoJson);
BlittableJsonReaderObject settingsJson;
using (var fs = new FileStream(ServerStore.Configuration.ConfigPath, FileMode.Open, FileAccess.Read))
{
settingsJson = context.ReadForMemory(fs, "settings-json");
}
settingsJson.Modifications = new DynamicJsonValue(settingsJson)
{
[RavenConfiguration.GetKey(x => x.Licensing.EulaAccepted)] = true,
[RavenConfiguration.GetKey(x => x.Core.SetupMode)] = nameof(SetupMode.Unsecured),
[RavenConfiguration.GetKey(x => x.Security.UnsecuredAccessAllowed)] = nameof(UnsecuredAccessAddressRange.PublicNetwork)
};
if (setupInfo.Port == 0)
{
setupInfo.Port = 8080;
}
settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.Port)));
if (setupInfo.TcpPort == 0)
{
setupInfo.TcpPort = 38888;
}
settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.TcpServerUrls)] = string.Join(";", setupInfo.Addresses.Select(ip => IpAddressToUrl(ip, setupInfo.TcpPort, "tcp")));
if (setupInfo.EnableExperimentalFeatures)
{
settingsJson.Modifications[RavenConfiguration.GetKey(x => x.Core.FeaturesAvailability)] = FeaturesAvailability.Experimental;
}
if (!string.IsNullOrEmpty(setupInfo.LocalNodeTag))
{
ServerStore.EnsureNotPassive(nodeTag: setupInfo.LocalNodeTag);
}
if (setupInfo.Environment != StudioConfiguration.StudioEnvironment.None)
{
var res = await ServerStore.PutValueInClusterAsync(new PutServerWideStudioConfigurationCommand(new ServerWideStudioConfiguration
{
Disabled = false,
Environment = setupInfo.Environment
}));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
var modifiedJsonObj = context.ReadObject(settingsJson, "modified-settings-json");
var indentedJson = SetupManager.IndentJsonString(modifiedJsonObj.ToString());
SetupManager.WriteSettingsJsonLocally(ServerStore.Configuration.ConfigPath, indentedJson);
}
NoContentStatus();
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetStringQueryString("url").TrimEnd('/');
var watcher = GetBoolValueQueryString("watcher", false);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.ClusterCertificateHolder.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
ServerStore.EnsureNotPassive();
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
topologyId = clusterTopology.TopologyId;
}
if (nodeInfo.TopologyId != null && topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag
? null : nodeInfo.NodeTag;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because it has no certificate while trying to use HTTPS");
}
var certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate));
if (certificate.NotBefore > DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
if (certificate.NotAfter < DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbrpint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
NoContentStatus();
return;
}
}
RedirectToLeader();
}
protected RestoreBackupTaskBase(ServerStore serverStore,
RestoreBackupConfigurationBase restoreFromConfiguration,
string nodeTag,
OperationCancelToken operationCancelToken)
{
_serverStore = serverStore;
RestoreFromConfiguration = restoreFromConfiguration;
_nodeTag = nodeTag;
_operationCancelToken = operationCancelToken;
var dataDirectoryThatWillBeUsed = string.IsNullOrWhiteSpace(RestoreFromConfiguration.DataDirectory) ?
_serverStore.Configuration.Core.DataDirectory.FullPath :
new PathSetting(RestoreFromConfiguration.DataDirectory, _serverStore.Configuration.Core.DataDirectory.FullPath).FullPath;
if (ResourceNameValidator.IsValidResourceName(RestoreFromConfiguration.DatabaseName, dataDirectoryThatWillBeUsed, out string errorMessage) == false)
{
throw new InvalidOperationException(errorMessage);
}
_serverStore.EnsureNotPassive();
ClusterTopology clusterTopology;
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
if (_serverStore.Cluster.DatabaseExists(context, RestoreFromConfiguration.DatabaseName))
{
throw new ArgumentException($"Cannot restore data to an existing database named {RestoreFromConfiguration.DatabaseName}");
}
clusterTopology = _serverStore.GetClusterTopology(context);
}
_hasEncryptionKey = string.IsNullOrWhiteSpace(RestoreFromConfiguration.EncryptionKey) == false;
if (_hasEncryptionKey)
{
var key = Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey);
if (key.Length != 256 / 8)
{
throw new InvalidOperationException($"The size of the key must be 256 bits, but was {key.Length * 8} bits.");
}
if (AdminDatabasesHandler.NotUsingHttps(clusterTopology.GetUrlFromTag(_serverStore.NodeTag)))
{
throw new InvalidOperationException("Cannot restore an encrypted database to a node which doesn't support SSL!");
}
}
var backupEncryptionSettings = RestoreFromConfiguration.BackupEncryptionSettings;
if (backupEncryptionSettings != null)
{
if (backupEncryptionSettings.EncryptionMode == EncryptionMode.UseProvidedKey &&
backupEncryptionSettings.Key == null)
{
throw new InvalidOperationException($"{nameof(BackupEncryptionSettings.EncryptionMode)} is set to {nameof(EncryptionMode.UseProvidedKey)} but an encryption key wasn't provided");
}
if (backupEncryptionSettings.EncryptionMode != EncryptionMode.UseProvidedKey &&
backupEncryptionSettings.Key != null)
{
throw new InvalidOperationException($"{nameof(BackupEncryptionSettings.EncryptionMode)} is set to {backupEncryptionSettings.EncryptionMode} but an encryption key was provided");
}
}
var hasRestoreDataDirectory = string.IsNullOrWhiteSpace(RestoreFromConfiguration.DataDirectory) == false;
if (hasRestoreDataDirectory &&
HasFilesOrDirectories(dataDirectoryThatWillBeUsed))
{
throw new ArgumentException("New data directory must be empty of any files or folders, " +
$"path: {dataDirectoryThatWillBeUsed}");
}
if (hasRestoreDataDirectory == false)
{
RestoreFromConfiguration.DataDirectory = GetDataDirectory();
}
_restoringToDefaultDataDirectory = IsDefaultDataDirectory(RestoreFromConfiguration.DataDirectory, RestoreFromConfiguration.DatabaseName);
}
public async Task ReplaceClusterCert()
{
var replaceImmediately = GetBoolValueQueryString("replaceImmediately", required: false) ?? false;
ServerStore.EnsureNotPassive();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var certificateJson = ctx.ReadForDisk(RequestBodyStream(), "replace-cluster-cert"))
{
try
{
var certificate = JsonDeserializationServer.CertificateDefinition(certificateJson);
if (string.IsNullOrWhiteSpace(certificate.Name))
{
certificate.Name = "Cluster-Wide Certificate";
}
if (string.IsNullOrWhiteSpace(certificate.Certificate))
{
throw new ArgumentException($"{nameof(certificate.Certificate)} is a required field in the certificate definition.");
}
if (IsClusterAdmin() == false)
{
throw new InvalidOperationException("Cannot replace the server certificate. Only a ClusterAdmin can do this.");
}
byte[] certBytes;
try
{
certBytes = Convert.FromBase64String(certificate.Certificate);
}
catch (Exception e)
{
throw new ArgumentException($"Unable to parse the {nameof(certificate.Certificate)} property, expected a Base64 value", e);
}
X509Certificate2 newCertificate;
try
{
newCertificate = new X509Certificate2(certBytes, certificate.Password, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.PersistKeySet);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to load the new certificate.", e);
}
var timeoutTask = TimeoutManager.WaitFor(TimeSpan.FromSeconds(60), ServerStore.ServerShutdown);
var replicationTask = Server.StartCertificateReplicationAsync(newCertificate, certificate.Name, replaceImmediately);
await Task.WhenAny(replicationTask, timeoutTask);
if (replicationTask.IsCompleted == false)
{
throw new TimeoutException("Timeout when trying to replace the server certificate.");
}
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to replace the server certificate.", e);
}
}
NoContentStatus();
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
}
public async Task Put()
{
// one of the first admin action is to create a certificate, so let
// us also use that to indicate that we are the seed node
ServerStore.EnsureNotPassive();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var certificateJson = ctx.ReadForDisk(RequestBodyStream(), "put-certificate"))
{
var certificate = JsonDeserializationServer.CertificateDefinition(certificateJson);
ValidateCertificate(certificate, ServerStore);
if (certificate.SecurityClearance == SecurityClearance.ClusterAdmin && IsClusterAdmin() == false)
{
var clientCert = (HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection)?.Certificate;
throw new InvalidOperationException($"Cannot save the certificate '{certificate.Name}' with 'Cluster Admin' security clearance because the current client certificate being used has a lower clearance: {clientCert}");
}
if (string.IsNullOrWhiteSpace(certificate.Certificate))
{
throw new ArgumentException($"{nameof(certificate.Certificate)} is a mandatory property when saving an existing certificate");
}
byte[] certBytes;
try
{
certBytes = Convert.FromBase64String(certificate.Certificate);
}
catch (Exception e)
{
throw new ArgumentException($"Unable to parse the {nameof(certificate.Certificate)} property, expected a Base64 value", e);
}
var x509Certificate = new X509Certificate2(certBytes);
if (PlatformDetails.RunningOnPosix)
{
// Implementation of SslStream AuthenticateAsServer is different in Linux. See RavenDB-8524
// A list of allowed CAs is sent from the server to the client. The handshake will fail if the client's CA is not in that list. This list is taken from the root and certificate authority stores of the OS.
// In this workaround we make sure that the CA (who signed the server cert, which in turn signed the client cert) is registered in one of the OS stores.
using (var currentUserStore = new X509Store(StoreName.Root, StoreLocation.CurrentUser))
{
currentUserStore.Open(OpenFlags.ReadOnly);
var userCerts = currentUserStore.Certificates.Find(X509FindType.FindByIssuerDistinguishedName, x509Certificate.IssuerName, true);
if (userCerts.Contains(x509Certificate) == false)
{
throw new InvalidOperationException($"Cannot save the client certificate '{certificate.Name}'. " +
$"First, you must register the issuer certificate '{x509Certificate.IssuerName}' in the trusted root store, on the server machine." +
"This step is required because you are using a self-signed certificate or one with unknown issuer.");
}
}
}
if (x509Certificate.HasPrivateKey)
{
// avoid storing the private key
certificate.Certificate = Convert.ToBase64String(x509Certificate.Export(X509ContentType.Cert));
}
certificate.Thumbprint = x509Certificate.Thumbprint;
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + x509Certificate.Thumbprint, certificate));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
NoContentStatus();
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
}
}
