private static void ValidatePermissions(CertificateDefinition certificate, ServerStore serverStore)
{
if (certificate.Permissions == null)
{
throw new ArgumentException($"{nameof(certificate.Permissions)} is a required field in the certificate definition");
}
foreach (var kvp in certificate.Permissions)
{
if (string.IsNullOrWhiteSpace(kvp.Key))
{
throw new ArgumentException("Error in permissions in the certificate definition, database name is empty");
}
if (ResourceNameValidator.IsValidResourceName(kvp.Key, serverStore.Configuration.Core.DataDirectory.FullPath, out var errorMessage) == false)
{
throw new ArgumentException("Error in permissions in the certificate definition:" + errorMessage);
}
if (kvp.Value != DatabaseAccess.ReadWrite && kvp.Value != DatabaseAccess.Admin)
{
throw new ArgumentException($"Error in permissions in the certificate definition, invalid access {kvp.Value} for database {kvp.Key}");
}
}
}
public ClientCertificate()
{
using (var store = new DocumentStore())
{
{
#region cert_1_4
// With user role set to Cluster Administator or Operator the user of this certificate
// is going to have access to all databases
CreateClientCertificateOperation operation =
new CreateClientCertificateOperation(
"admin", null, SecurityClearance.Operator);
CertificateRawData certificateRawData =
store.Maintenance.Server.Send(operation);
byte[] cert = certificateRawData.RawData;
#endregion
}
{
#region cert_1_5
// when security clearance is ValidUser, you need to specify per database permissions
CreateClientCertificateOperation operation =
new CreateClientCertificateOperation(
"user1", new Dictionary <string, DatabaseAccess>
{
{ "Northwind", DatabaseAccess.Admin }
}, SecurityClearance.ValidUser, "myPassword");
CertificateRawData certificateRawData =
store.Maintenance.Server.Send(operation);
byte[] cert = certificateRawData.RawData;
#endregion
}
{
#region get_cert_2
string thumbprint = "a909502dd82ae41433e6f83886b00d4277a32a7b";
CertificateDefinition definition =
store.Maintenance.Server.Send(new GetCertificateOperation(thumbprint));
#endregion
}
{
#region get_certs_2
CertificateDefinition[] definitions =
store.Maintenance.Server.Send(new GetCertificatesOperation(0, 20));
#endregion
}
{
#region cert_put_2
X509Certificate2 certificate = new X509Certificate2("c:\\path_to_pfx_file");
store.Maintenance.Server.Send(
new PutClientCertificateOperation(
"cert1", certificate, null, SecurityClearance.ClusterAdmin));
#endregion
}
}
}
public static void ValidateCertificateDefinition(CertificateDefinition certificate, ServerStore serverStore)
{
if (string.IsNullOrWhiteSpace(certificate.Name))
{
throw new ArgumentException($"{nameof(certificate.Name)} is a required field in the certificate definition");
}
ValidatePermissions(certificate, serverStore);
}
public static async Task <byte[]> GenerateCertificateInternal(CertificateDefinition certificate, ServerStore serverStore)
{
ValidateCertificateDefinition(certificate, serverStore);
if (serverStore.Server.Certificate?.Certificate == null)
{
var keys = new[]
{
RavenConfiguration.GetKey(x => x.Security.CertificatePath),
RavenConfiguration.GetKey(x => x.Security.CertificateExec)
};
throw new InvalidOperationException($"Cannot generate the client certificate '{certificate.Name}' because the server certificate is not loaded. " +
$"You can supply a server certificate by using the following configuration keys: {string.Join(", ", keys)}" +
"For a more detailed explanation please read about authentication and certificates in the RavenDB documentation.");
}
// this creates a client certificate which is signed by the current server certificate
var selfSignedCertificate = CertificateUtils.CreateSelfSignedClientCertificate(certificate.Name, serverStore.Server.Certificate, out var clientCertBytes);
var newCertDef = new CertificateDefinition
{
Name = certificate.Name,
// this does not include the private key, that is only for the client
Certificate = Convert.ToBase64String(selfSignedCertificate.Export(X509ContentType.Cert)),
Permissions = certificate.Permissions,
SecurityClearance = certificate.SecurityClearance,
Thumbprint = selfSignedCertificate.Thumbprint,
NotAfter = selfSignedCertificate.NotAfter
};
var res = await serverStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + selfSignedCertificate.Thumbprint, newCertDef));
await serverStore.Cluster.WaitForIndexNotification(res.Index);
var ms = new MemoryStream();
using (var archive = new ZipArchive(ms, ZipArchiveMode.Create, true))
{
var certBytes = selfSignedCertificate.Export(X509ContentType.Pfx, certificate.Password);
var entry = archive.CreateEntry(certificate.Name + ".pfx");
// Structure of the external attributes field: https://unix.stackexchange.com/questions/14705/the-zip-formats-external-file-attribute/14727#14727
// The permissions go into the most significant 16 bits of an int
entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;
using (var s = entry.Open())
s.Write(certBytes, 0, certBytes.Length);
WriteCertificateAsPem(certificate.Name, clientCertBytes, certificate.Password, archive);
}
return(ms.ToArray());
}
public async Task RemoteWatch()
{
var thumbprint = GetStringQueryString("thumbprint", required: false);
CertificateDefinition clientConnectedCertificate = null;
var canAccessDatabase = GetDatabaseAccessValidationFunc();
var currentCertificate = GetCurrentCertificate();
if (string.IsNullOrEmpty(thumbprint) == false && currentCertificate != null && thumbprint != currentCertificate.Thumbprint)
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
using (ctx.OpenReadTransaction())
{
var certByThumbprint = ServerStore.Cluster.GetCertificateByThumbprint(ctx, thumbprint) ?? ServerStore.Cluster.GetLocalStateByThumbprint(ctx, thumbprint);
if (certByThumbprint != null)
{
clientConnectedCertificate = JsonDeserializationServer.CertificateDefinition(certByThumbprint);
}
}
if (clientConnectedCertificate != null)
{
// we're already connected as ClusterAdmin, here we're just limiting the access to databases based on the thumbprint of the originally connected certificated
// so we'll send notifications only about relevant databases
var authenticationStatus = new RavenServer.AuthenticateConnection();
authenticationStatus.SetBasedOnCertificateDefinition(clientConnectedCertificate);
canAccessDatabase = GetDatabaseAccessValidationFunc(authenticationStatus);
}
}
}
using (var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync())
{
try
{
await SendNotifications(canAccessDatabase, webSocket);
}
catch (OperationCanceledException)
{
// ignored
}
catch (Exception ex)
{
await HandleException(ex, webSocket);
}
}
}
public Task WhoAmI()
{
var feature = HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection;
var clientCert = feature?.Certificate;
if (clientCert == null)
{
return(NoContent());
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
BlittableJsonReaderObject certificate;
using (ctx.OpenReadTransaction())
certificate = ServerStore.Cluster.Read(ctx, Constants.Certificates.Prefix + clientCert.Thumbprint);
if (certificate == null && clientCert.Equals(Server.Certificate.Certificate))
{
var certKey = Constants.Certificates.Prefix + clientCert.Thumbprint;
using (ctx.OpenReadTransaction())
certificate = ServerStore.Cluster.Read(ctx, certKey) ??
ServerStore.Cluster.GetLocalState(ctx, certKey);
if (certificate == null && Server.Certificate.Certificate != null)
{
// Since we didn't go through EnsureNotPassive(), the server certificate is not registered yet so we'll add it to the local state.
var serverCertDef = new CertificateDefinition
{
Name = "Server Certificate",
Certificate = Convert.ToBase64String(Server.Certificate.Certificate.Export(X509ContentType.Cert)),
Permissions = new Dictionary <string, DatabaseAccess>(),
SecurityClearance = SecurityClearance.ClusterNode,
Thumbprint = Server.Certificate.Certificate.Thumbprint,
NotAfter = Server.Certificate.Certificate.NotAfter
};
certificate = ctx.ReadObject(serverCertDef.ToJson(), "Server/Certificate/Definition");
using (var tx = ctx.OpenWriteTransaction())
{
ServerStore.Cluster.PutLocalState(ctx, Constants.Certificates.Prefix + Server.Certificate.Certificate.Thumbprint, certificate);
tx.Commit();
}
}
}
using (var writer = new BlittableJsonTextWriter(ctx, ResponseBodyStream()))
{
writer.WriteObject(certificate);
}
}
return(Task.CompletedTask);
}
private async Task Annotate(V1Secret secret, CertificateDefinition certificate)
{
var secretAnnotations =
new Dictionary <string, string>(secret.Metadata.Annotations ?? new Dictionary <string, string>());
var original = secretAnnotations.ToDictionary(s => s.Key, s => s.Value);
var certificateAnnotations =
new Dictionary <string, string>(certificate.Metadata.Annotations ?? new Dictionary <string, string>());
if (certificateAnnotations.TryGetValue(Annotations.CertManagerCertificate.SecretReflectionAllowed,
out var reflectionAllowed))
{
secretAnnotations[Annotations.Reflection.Allowed] = reflectionAllowed;
}
else
{
secretAnnotations.Remove(Annotations.Reflection.Allowed);
}
if (certificateAnnotations.TryGetValue(Annotations.CertManagerCertificate.SecretReflectionAllowedNamespaces,
out var allowedNamespaces))
{
secretAnnotations[Annotations.Reflection.AllowedNamespaces] = allowedNamespaces;
}
else
{
secretAnnotations.Remove(Annotations.Reflection.AllowedNamespaces);
}
if (secretAnnotations.Count == original.Count &&
secretAnnotations.Keys.All(s => original.ContainsKey(s)) &&
secretAnnotations.All(s => original[s.Key] == s.Value))
{
_logger.LogDebug("Secret {secretNs}/{secretName} matches certificate {certNs}/{certName} reflection annotations",
secret.Metadata.NamespaceProperty, secret.Metadata.Name,
certificate.Metadata.NamespaceProperty, certificate.Metadata.Name);
return;
}
_logger.LogInformation(
"Patching {secretNs}/{secretName} to match certificate {certNs}/{certName} reflection annotations",
secret.Metadata.NamespaceProperty, secret.Metadata.Name,
certificate.Metadata.NamespaceProperty, certificate.Metadata.Name);
var patch = new JsonPatchDocument <V1Secret>();
patch.Replace(e => e.Metadata.Annotations, secretAnnotations);
await _apiClient.PatchNamespacedSecretWithHttpMessagesAsync(new V1Patch(patch),
secret.Metadata.Name, secret.Metadata.NamespaceProperty);
}
private async Task OnSecretsEvent(WatchEventType eventType, V1Secret item)
{
if (eventType != WatchEventType.Added && eventType != WatchEventType.Modified)
{
return;
}
var metadata = item.Metadata;
if (metadata.Labels == null)
{
return;
}
if (metadata.Labels.TryGetValue(CertManagerConstants.CertificateNameLabel, out var certificateName))
{
_logger.LogDebug("Secret {secretNs}/{secretName} belongs to certificate {certNs}/{certName}",
metadata.NamespaceProperty, metadata.Name,
metadata.NamespaceProperty, certificateName);
CertificateDefinition cert = null;
try
{
var certificate = await _apiClient.GetNamespacedCustomObjectAsync(CertManagerConstants.CrdGroup,
_currentVersion, metadata.NamespaceProperty, CertManagerConstants.CertificatePlural,
certificateName);
cert = ((JObject)certificate).ToObject <CertificateDefinition>();
}
catch (HttpOperationException exception) when(exception.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogDebug("Could not find certificate {certNs}/{certName}",
metadata.NamespaceProperty, certificateName);
}
if (cert != null)
{
await Annotate(item, cert);
}
}
}
private async Task OnCertificatesEvent(WatchEventType eventType, CertificateDefinition item)
{
if (eventType != WatchEventType.Added && eventType != WatchEventType.Modified)
{
return;
}
if (item.Metadata == null)
{
return;
}
if (item.Spec == null)
{
return;
}
_logger.LogDebug("Certificate {certNs}/{certName} has secret {secretNs}/{secretName}",
item.Metadata.NamespaceProperty, item.Metadata.Name,
item.Metadata.NamespaceProperty, item.Spec.SecretName);
V1Secret secret = null;
try
{
secret = await _apiClient.ReadNamespacedSecretAsync(item.Spec.SecretName,
item.Metadata.NamespaceProperty);
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogDebug("Could not find matching secret {secretNs}/{secretName}",
item.Metadata.NamespaceProperty, item.Spec.SecretName);
}
if (secret != null)
{
await Annotate(secret, item);
}
}
public Task GetCertificates()
{
var thumbprint = GetStringQueryString("thumbprint", required: false);
var showSecondary = GetBoolValueQueryString("secondary", required: false) ?? false;
var start = GetStart();
var pageSize = GetPageSize();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
var certificates = new List <(string ItemName, BlittableJsonReaderObject Value)>();
try
{
if (string.IsNullOrEmpty(thumbprint))
{
if (ServerStore.CurrentRachisState == RachisState.Passive)
{
List <string> localCertKeys;
using (context.OpenReadTransaction())
localCertKeys = ServerStore.Cluster.GetCertificateKeysFromLocalState(context).ToList();
var serverCertKey = Constants.Certificates.Prefix + Server.Certificate.Certificate?.Thumbprint;
if (Server.Certificate.Certificate != null && localCertKeys.Contains(serverCertKey) == false)
{
// Since we didn't go through EnsureNotPassive(), the server certificate is not registered yet so we'll add it to the local state.
var serverCertDef = new CertificateDefinition
{
Name = "Server Certificate",
Certificate = Convert.ToBase64String(Server.Certificate.Certificate.Export(X509ContentType.Cert)),
Permissions = new Dictionary <string, DatabaseAccess>(),
SecurityClearance = SecurityClearance.ClusterNode,
Thumbprint = Server.Certificate.Certificate.Thumbprint,
NotAfter = Server.Certificate.Certificate.NotAfter
};
var serverCert = context.ReadObject(serverCertDef.ToJson(), "Server/Certificate/Definition");
using (var tx = context.OpenWriteTransaction())
{
ServerStore.Cluster.PutLocalState(context, Constants.Certificates.Prefix + Server.Certificate.Certificate.Thumbprint, serverCert);
tx.Commit();
}
certificates.Add((serverCertKey, serverCert));
}
foreach (var localCertKey in localCertKeys)
{
BlittableJsonReaderObject localCertificate;
using (context.OpenReadTransaction())
localCertificate = ServerStore.Cluster.GetLocalState(context, localCertKey);
if (localCertificate == null)
{
continue;
}
var def = JsonDeserializationServer.CertificateDefinition(localCertificate);
if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey))
{
certificates.Add((localCertKey, localCertificate));
}
else
{
localCertificate.Dispose();
}
}
}
else
{
using (context.OpenReadTransaction())
{
foreach (var item in ServerStore.Cluster.ItemsStartingWith(context, Constants.Certificates.Prefix, start, pageSize))
{
var def = JsonDeserializationServer.CertificateDefinition(item.Value);
if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey))
{
certificates.Add(item);
}
else
{
item.Value.Dispose();
}
}
}
}
}
else
{
using (context.OpenReadTransaction())
{
var key = Constants.Certificates.Prefix + thumbprint;
var certificate = ServerStore.CurrentRachisState == RachisState.Passive
? ServerStore.Cluster.GetLocalState(context, key)
: ServerStore.Cluster.Read(context, key);
if (certificate == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
var definition = JsonDeserializationServer.CertificateDefinition(certificate);
if (string.IsNullOrEmpty(definition.CollectionPrimaryKey) == false)
{
certificate = ServerStore.Cluster.Read(context, definition.CollectionPrimaryKey);
if (certificate == null)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return(Task.CompletedTask);
}
}
certificates.Add((key, certificate));
}
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WriteArray(context, "Results", certificates.ToArray(), (w, c, cert) =>
{
c.Write(w, cert.Value);
});
writer.WriteComma();
writer.WritePropertyName("LoadedServerCert");
writer.WriteString(Server.Certificate.Certificate?.Thumbprint);
writer.WriteEndObject();
}
}
finally
{
foreach (var cert in certificates)
{
cert.Value?.Dispose();
}
}
}
return(Task.CompletedTask);
}
public static async Task PutCertificateCollectionInCluster(CertificateDefinition certDef, byte[] certBytes, string password, ServerStore serverStore, TransactionOperationContext ctx)
{
var collection = new X509Certificate2Collection();
if (string.IsNullOrEmpty(password))
{
collection.Import(certBytes);
}
else
{
collection.Import(certBytes, password, X509KeyStorageFlags.PersistKeySet | X509KeyStorageFlags.Exportable);
}
var first = true;
var collectionPrimaryKey = string.Empty;
foreach (var x509Certificate in collection)
{
if (serverStore.Server.Certificate.Certificate?.Thumbprint != null && serverStore.Server.Certificate.Certificate.Thumbprint.Equals(x509Certificate.Thumbprint))
{
throw new InvalidOperationException($"You are trying to import the same server certificate ({x509Certificate.Thumbprint}) as the one which is already loaded. This is not supported.");
}
}
foreach (var x509Certificate in collection)
{
var currentCertDef = new CertificateDefinition
{
Name = certDef.Name,
Permissions = certDef.Permissions,
SecurityClearance = certDef.SecurityClearance,
Password = certDef.Password,
};
if (x509Certificate.HasPrivateKey)
{
// avoid storing the private key
currentCertDef.Certificate = Convert.ToBase64String(x509Certificate.Export(X509ContentType.Cert));
}
// In case of a collection, we group all the certificates together and treat them as one unit.
// They all have the same name and permissions but a different thumbprint.
// The first certificate in the collection will be the primary certificate and its thumbprint will be the one shown in a GET request
// The other certificates are secondary certificates and will contain a link to the primary certificate.
currentCertDef.Thumbprint = x509Certificate.Thumbprint;
currentCertDef.NotAfter = x509Certificate.NotAfter;
currentCertDef.Certificate = Convert.ToBase64String(x509Certificate.Export(X509ContentType.Cert));
if (first)
{
var firstKey = Constants.Certificates.Prefix + x509Certificate.Thumbprint;
collectionPrimaryKey = firstKey;
foreach (var cert in collection)
{
if (Constants.Certificates.Prefix + cert.Thumbprint != firstKey)
{
currentCertDef.CollectionSecondaryKeys.Add(Constants.Certificates.Prefix + cert.Thumbprint);
}
}
}
else
{
currentCertDef.CollectionPrimaryKey = collectionPrimaryKey;
}
var certKey = Constants.Certificates.Prefix + currentCertDef.Thumbprint;
if (serverStore.CurrentRachisState == RachisState.Passive)
{
using (var certificate = ctx.ReadObject(currentCertDef.ToJson(), "Client/Certificate/Definition"))
using (var tx = ctx.OpenWriteTransaction())
{
serverStore.Cluster.PutLocalState(ctx, certKey, certificate);
tx.Commit();
}
}
else
{
var putResult = await serverStore.PutValueInClusterAsync(new PutCertificateCommand(certKey, currentCertDef));
await serverStore.Cluster.WaitForIndexNotification(putResult.Index);
}
first = false;
}
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
var watcher = GetBoolValueQueryString("watcher", false);
var assignedCores = GetIntValueQueryString("assignedCores", false);
if (assignedCores <= 0)
{
throw new ArgumentException("Assigned cores must be greater than 0!");
}
nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
{
throw new ArgumentException("Cannot add node because the assigned cores is larger " +
$"than the available cores on that machine: {nodeInfo.NumberOfCores}");
}
ServerStore.EnsureNotPassive();
if (assignedCores == null)
{
assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
}
Debug.Assert(assignedCores <= nodeInfo.NumberOfCores);
ServerStore.LicenseManager.AssertCanAddNode(nodeUrl, assignedCores.Value);
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
ClusterTopology clusterTopology;
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
topologyId = clusterTopology.TopologyId;
}
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
if (nodeInfo.ServerId == ServerStore.GetServerId())
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
}
if (nodeInfo.TopologyId != null)
{
if (topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. " +
$"Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
if (nodeInfo.CurrentState != RachisState.Passive)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " +
$"because it's already in the cluster under tag :{nodeInfo.NodeTag} " +
$"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}");
}
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);
var now = DateTime.UtcNow;
if (certificate.NotBefore.ToUniversalTime() > now)
{
// Because of time zone and time drift issues, we can't assume that the certificate generation will be
// proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
// current time. Clients may still fail to work with our certificate because of timing issues,
// but the admin needs to setup time sync properly and there isn't much we can do at that point
if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
{
throw new InvalidOperationException(
$"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
}
if (certificate.NotAfter.ToUniversalTime() < now)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbprint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
using (ctx.OpenReadTransaction())
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var readCert = ServerStore.Cluster.Read(ctx, key);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
NotAfter = certificate.NotAfter,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null)
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
var nodeDetails = new NodeDetails
{
NodeTag = nodeTag,
AssignedCores = assignedCores.Value,
NumberOfCores = nodeInfo.NumberOfCores,
InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
UsableMemoryInGb = nodeInfo.UsableMemoryInGb,
BuildInfo = nodeInfo.BuildInfo
};
await ServerStore.LicenseManager.CalculateLicenseLimits(nodeDetails, forceFetchingNodeInfo : true, waitToUpdate : true);
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
private async Task <byte[]> GenerateCertificateInternal(TransactionOperationContext ctx, CertificateDefinition certificate)
{
ValidateCertificate(certificate, ServerStore);
if (certificate.SecurityClearance == SecurityClearance.ClusterAdmin && IsClusterAdmin() == false)
{
var clientCert = (HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection)?.Certificate;
throw new InvalidOperationException($"Cannot generate the certificate '{certificate.Name}' with 'Cluster Admin' security clearance because the current client certificate being used has a lower clearance: {clientCert}");
}
if (Server.ClusterCertificateHolder?.Certificate == null)
{
throw new InvalidOperationException($"Cannot generate the client certificate '{certificate.Name}' becuase the server certificate is not loaded. " +
$"You can supply a server certificate by using the following configuration keys: " +
$"'{RavenConfiguration.GetKey(x => x.Security.CertificatePath)}'/'{RavenConfiguration.GetKey(x => x.Security.CertificateExec)}'/" +
$"'{RavenConfiguration.GetKey(x => x.Security.ClusterCertificatePath)}'/'{RavenConfiguration.GetKey(x => x.Security.ClusterCertificateExec)}'. " +
$"For a more detailed explanation please read about authentication and certificates in the RavenDB documentation.");
}
if (PlatformDetails.RunningOnPosix)
{
// For the client certificate to work properly, we need that the issuer (our server certificate) will be registered in the trusted root store.
// In Linux, when using SslStream AuthenticateAsServer, the server sends the list of allowed CAs to the client. So the client's CA must be in the list. See RavenDB-8524
using (var machineRootStore = new X509Store(StoreName.Root, StoreLocation.LocalMachine, OpenFlags.ReadOnly))
using (var machineCaStore = new X509Store(StoreName.CertificateAuthority, StoreLocation.LocalMachine, OpenFlags.ReadOnly))
using (var userRootStore = new X509Store(StoreName.Root, StoreLocation.CurrentUser, OpenFlags.ReadOnly))
using (var userCaStore = new X509Store(StoreName.CertificateAuthority, StoreLocation.CurrentUser, OpenFlags.ReadOnly))
{
// workaround for lack of cert store inheritance RavenDB-8904
if (machineCaStore.Certificates.Contains(Server.ClusterCertificateHolder.Certificate) == false &&
machineRootStore.Certificates.Contains(Server.ClusterCertificateHolder.Certificate) == false &&
userCaStore.Certificates.Contains(Server.ClusterCertificateHolder.Certificate) == false &&
userRootStore.Certificates.Contains(Server.ClusterCertificateHolder.Certificate) == false)
{
throw new InvalidOperationException($"Cannot generate the client certificate '{certificate.Name}'. " +
$"First, you must register the server certificate '{Server.ClusterCertificateHolder.Certificate.FriendlyName}' in the trusted root store, on the server machine." +
$"The server certificate is located in one of the following locations: {ServerStore.Configuration.Security.CertificatePath ?? " "} / {ServerStore.Configuration.Security.ClusterCertificatePath ?? " "} / {ServerStore.Configuration.Security.CertificateExec ?? " "} / {ServerStore.Configuration.Security.ClusterCertificateExec ?? " "}." +
"This step is required because you are using a self-signed server certificate.");
}
}
}
// this creates a client certificate which is signed by the current server certificate
var selfSignedCertificate = CertificateUtils.CreateSelfSignedClientCertificate(certificate.Name, Server.ClusterCertificateHolder);
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + selfSignedCertificate.Thumbprint,
new CertificateDefinition
{
Name = certificate.Name,
// this does not include the private key, that is only for the client
Certificate = Convert.ToBase64String(selfSignedCertificate.Export(X509ContentType.Cert)),
Permissions = certificate.Permissions,
SecurityClearance = certificate.SecurityClearance,
Thumbprint = selfSignedCertificate.Thumbprint
}));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
return(selfSignedCertificate.Export(X509ContentType.Pfx, certificate.Password));
}
private async Task <byte[]> GenerateCertificateInternal(TransactionOperationContext ctx, CertificateDefinition certificate)
{
ValidateCertificate(certificate, ServerStore);
if (certificate.SecurityClearance == SecurityClearance.ClusterAdmin && IsClusterAdmin() == false)
{
var clientCert = (HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection)?.Certificate;
throw new InvalidOperationException($"Cannot generate the certificate '{certificate.Name}' with 'Cluster Admin' security clearance because the current client certificate being used has a lower clearance: {clientCert}");
}
if (Server.ClusterCertificateHolder?.Certificate == null)
{
var keys = new[]
{
RavenConfiguration.GetKey(x => x.Security.CertificatePath),
RavenConfiguration.GetKey(x => x.Security.CertificateExec),
RavenConfiguration.GetKey(x => x.Security.ClusterCertificatePath),
RavenConfiguration.GetKey(x => x.Security.ClusterCertificateExec)
};
throw new InvalidOperationException($"Cannot generate the client certificate '{certificate.Name}' becuase the server certificate is not loaded. " +
$"You can supply a server certificate by using the following configuration keys: {keys}" +
"For a more detailed explanation please read about authentication and certificates in the RavenDB documentation.");
}
if (PlatformDetails.RunningOnPosix)
{
// Implementation of SslStream AuthenticateAsServer is different in Linux. See RavenDB-8524
// A list of allowed CAs is sent from the server to the client. The handshake will fail if the client's CA is not in that list. This list is taken from the root and certificate authority stores of the OS.
// In this workaround we make sure that the CA (who signed the server cert, which in turn signed the client cert) is registered in one of the OS stores.
var chain = new X509Chain
{
ChainPolicy =
{
RevocationMode = X509RevocationMode.NoCheck,
RevocationFlag = X509RevocationFlag.ExcludeRoot,
VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority,
VerificationTime = DateTime.UtcNow,
UrlRetrievalTimeout = new TimeSpan(0, 0, 0)
}
};
if (chain.Build(Server.ClusterCertificateHolder.Certificate) == false)
{
throw new InvalidOperationException($"Cannot generate the client certificate '{certificate.Name}'. The server certificate chain is broken, admin assistance required.");
}
var rootCert = GetRootCertificate(chain);
if (rootCert == null)
{
throw new InvalidOperationException($"Cannot generate the client certificate '{certificate.Name}'. The server certificate chain is broken, admin assistance required.");
}
using (var machineRootStore = new X509Store(StoreName.Root, StoreLocation.LocalMachine, OpenFlags.ReadOnly))
using (var machineCaStore = new X509Store(StoreName.CertificateAuthority, StoreLocation.LocalMachine, OpenFlags.ReadOnly))
using (var userRootStore = new X509Store(StoreName.Root, StoreLocation.CurrentUser, OpenFlags.ReadOnly))
using (var userCaStore = new X509Store(StoreName.CertificateAuthority, StoreLocation.CurrentUser, OpenFlags.ReadOnly))
{
// workaround for lack of cert store inheritance RavenDB-8904
if (machineCaStore.Certificates.Contains(rootCert) == false &&
machineRootStore.Certificates.Contains(rootCert) == false &&
userCaStore.Certificates.Contains(rootCert) == false &&
userRootStore.Certificates.Contains(rootCert) == false)
{
var path = new[]
{
ServerStore.Configuration.Security.CertificatePath,
ServerStore.Configuration.Security.ClusterCertificatePath,
ServerStore.Configuration.Security.CertificateExec,
ServerStore.Configuration.Security.ClusterCertificateExec
}.FirstOrDefault(File.Exists) ?? "no path defined";
throw new InvalidOperationException($"Cannot generate the client certificate '{certificate.Name}'. " +
$"First, you must register the CA of the server certificate '{Server.ClusterCertificateHolder.Certificate.SubjectName.Name}' in the trusted root store, on the server machine." +
$"The server certificate is located in: '{path}'" +
"This step is required because you are using a self-signed server certificate.");
}
}
}
// this creates a client certificate which is signed by the current server certificate
var selfSignedCertificate = CertificateUtils.CreateSelfSignedClientCertificate(certificate.Name, Server.ClusterCertificateHolder);
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + selfSignedCertificate.Thumbprint,
new CertificateDefinition
{
Name = certificate.Name,
// this does not include the private key, that is only for the client
Certificate = Convert.ToBase64String(selfSignedCertificate.Export(X509ContentType.Cert)),
Permissions = certificate.Permissions,
SecurityClearance = certificate.SecurityClearance,
Thumbprint = selfSignedCertificate.Thumbprint
}));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
return(selfSignedCertificate.Export(X509ContentType.Pfx, certificate.Password));
}
public Task WhoAmI()
{
var clientCert = GetCurrentCertificate();
if (clientCert == null)
{
return(NoContent());
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
var certKey = Constants.Certificates.Prefix + clientCert.Thumbprint;
BlittableJsonReaderObject certificate;
using (ctx.OpenReadTransaction())
certificate = ServerStore.Cluster.Read(ctx, certKey) ??
ServerStore.Cluster.GetLocalState(ctx, certKey);
if (certificate == null)
{
// The client certificate is not registered in the ServerStore.
// Let's check if the client is using the server certificate or one of the well known admin certs.
var wellKnown = ServerStore.Configuration.Security.WellKnownAdminCertificates;
if (clientCert.Equals(Server.Certificate.Certificate))
{
if (Server.Certificate.Certificate != null)
{
var serverCertDef = new CertificateDefinition
{
Name = "Server Certificate",
Certificate = Convert.ToBase64String(Server.Certificate.Certificate.Export(X509ContentType.Cert)),
Permissions = new Dictionary <string, DatabaseAccess>(),
SecurityClearance = SecurityClearance.ClusterNode,
Thumbprint = Server.Certificate.Certificate.Thumbprint,
NotAfter = Server.Certificate.Certificate.NotAfter
};
certificate = ctx.ReadObject(serverCertDef.ToJson(), "Server/Certificate/Definition");
}
}
else if (wellKnown != null && wellKnown.Contains(clientCert.Thumbprint, StringComparer.OrdinalIgnoreCase))
{
var serverCertDef = new CertificateDefinition
{
Name = "Well Known Admin Certificate",
Permissions = new Dictionary <string, DatabaseAccess>(),
SecurityClearance = SecurityClearance.ClusterAdmin,
Thumbprint = clientCert.Thumbprint,
};
certificate = ctx.ReadObject(serverCertDef.ToJson(), "WellKnown/Certificate/Definition");
}
}
using (var writer = new BlittableJsonTextWriter(ctx, ResponseBodyStream()))
{
writer.WriteObject(certificate);
}
}
return(Task.CompletedTask);
}
private async Task <byte[]> GenerateCertificateInternal(TransactionOperationContext ctx, CertificateDefinition certificate)
{
ValidateCertificate(certificate, ServerStore);
if (certificate.SecurityClearance == SecurityClearance.ClusterAdmin && IsClusterAdmin() == false)
{
var clientCert = (HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection)?.Certificate;
var clientCertDef = ReadCertificateFromCluster(ctx, Constants.Certificates.Prefix + clientCert?.Thumbprint);
throw new InvalidOperationException($"Cannot generate the certificate '{certificate.Name}' with 'Cluster Admin' security clearance because the current client certificate being used has a lower clearance: {clientCertDef.SecurityClearance}");
}
if (Server.ClusterCertificateHolder?.Certificate == null)
{
var keys = new[]
{
RavenConfiguration.GetKey(x => x.Security.CertificatePath),
RavenConfiguration.GetKey(x => x.Security.CertificateExec),
RavenConfiguration.GetKey(x => x.Security.ClusterCertificatePath),
RavenConfiguration.GetKey(x => x.Security.ClusterCertificateExec)
};
throw new InvalidOperationException($"Cannot generate the client certificate '{certificate.Name}' becuase the server certificate is not loaded. " +
$"You can supply a server certificate by using the following configuration keys: {string.Join(", ", keys)}" +
"For a more detailed explanation please read about authentication and certificates in the RavenDB documentation.");
}
if (PlatformDetails.RunningOnPosix)
{
ValidateCaExistsInOsStores(certificate);
}
// this creates a client certificate which is signed by the current server certificate
var selfSignedCertificate = CertificateUtils.CreateSelfSignedClientCertificate(certificate.Name, Server.ClusterCertificateHolder);
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + selfSignedCertificate.Thumbprint,
new CertificateDefinition
{
Name = certificate.Name,
// this does not include the private key, that is only for the client
Certificate = Convert.ToBase64String(selfSignedCertificate.Export(X509ContentType.Cert)),
Permissions = certificate.Permissions,
SecurityClearance = certificate.SecurityClearance,
Thumbprint = selfSignedCertificate.Thumbprint
}));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
return(selfSignedCertificate.Export(X509ContentType.Pfx, certificate.Password));
}
private async Task <byte[]> GenerateCertificateInternal(TransactionOperationContext ctx, CertificateDefinition certificate)
{
ValidateCertificate(certificate, ServerStore);
if (certificate.SecurityClearance == SecurityClearance.ClusterAdmin && IsClusterAdmin() == false)
{
var clientCert = (HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection)?.Certificate;
var clientCertDef = ReadCertificateFromCluster(ctx, Constants.Certificates.Prefix + clientCert?.Thumbprint);
throw new InvalidOperationException($"Cannot generate the certificate '{certificate.Name}' with 'Cluster Admin' security clearance because the current client certificate being used has a lower clearance: {clientCertDef.SecurityClearance}");
}
if (Server.Certificate?.Certificate == null)
{
var keys = new[]
{
RavenConfiguration.GetKey(x => x.Security.CertificatePath),
RavenConfiguration.GetKey(x => x.Security.CertificateExec)
};
throw new InvalidOperationException($"Cannot generate the client certificate '{certificate.Name}' becuase the server certificate is not loaded. " +
$"You can supply a server certificate by using the following configuration keys: {string.Join(", ", keys)}" +
"For a more detailed explanation please read about authentication and certificates in the RavenDB documentation.");
}
// this creates a client certificate which is signed by the current server certificate
var selfSignedCertificate = CertificateUtils.CreateSelfSignedClientCertificate(certificate.Name, Server.Certificate, out var clientCertBytes);
var newCertDef = new CertificateDefinition
{
Name = certificate.Name,
// this does not include the private key, that is only for the client
Certificate = Convert.ToBase64String(selfSignedCertificate.Export(X509ContentType.Cert)),
Permissions = certificate.Permissions,
SecurityClearance = certificate.SecurityClearance,
Thumbprint = selfSignedCertificate.Thumbprint,
NotAfter = selfSignedCertificate.NotAfter
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + selfSignedCertificate.Thumbprint, newCertDef));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
var ms = new MemoryStream();
using (var archive = new ZipArchive(ms, ZipArchiveMode.Create, true))
{
var certBytes = selfSignedCertificate.Export(X509ContentType.Pfx, certificate.Password);
var entry = archive.CreateEntry(certificate.Name + ".pfx");
using (var s = entry.Open())
s.Write(certBytes, 0, certBytes.Length);
entry = archive.CreateEntry(certificate.Name + ".pem");
using (var s = entry.Open())
{
WriteCertificateAsPem(clientCertBytes, certificate.Password, s);
}
}
return(ms.ToArray());
}
public async Task Put()
{
// one of the first admin action is to create a certificate, so let
// us also use that to indicate that we are the seed node
ServerStore.EnsureNotPassive();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var certificateJson = ctx.ReadForDisk(RequestBodyStream(), "put-certificate"))
{
var certificate = JsonDeserializationServer.CertificateDefinition(certificateJson);
ValidateCertificate(certificate, ServerStore);
if ((certificate.SecurityClearance == SecurityClearance.ClusterAdmin || certificate.SecurityClearance == SecurityClearance.ClusterNode) && IsClusterAdmin() == false)
{
var clientCert = (HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection)?.Certificate;
var clientCertDef = ReadCertificateFromCluster(ctx, Constants.Certificates.Prefix + clientCert?.Thumbprint);
throw new InvalidOperationException($"Cannot save the certificate '{certificate.Name}' with '{certificate.SecurityClearance}' security clearance because the current client certificate being used has a lower clearance: {clientCertDef.SecurityClearance}");
}
if (string.IsNullOrWhiteSpace(certificate.Certificate))
{
throw new ArgumentException($"{nameof(certificate.Certificate)} is a mandatory property when saving an existing certificate");
}
byte[] certBytes;
try
{
certBytes = Convert.FromBase64String(certificate.Certificate);
}
catch (Exception e)
{
throw new ArgumentException($"Unable to parse the {nameof(certificate.Certificate)} property, expected a Base64 value", e);
}
var collection = new X509Certificate2Collection();
collection.Import(certBytes);
var first = true;
var collectionPrimaryKey = string.Empty;
foreach (var x509Certificate in collection)
{
var currentCertificate = new CertificateDefinition
{
Name = certificate.Name,
Permissions = certificate.Permissions,
SecurityClearance = certificate.SecurityClearance,
Password = certificate.Password
};
if (x509Certificate.HasPrivateKey)
{
// avoid storing the private key
currentCertificate.Certificate = Convert.ToBase64String(x509Certificate.Export(X509ContentType.Cert));
}
// In case of a collection, we group all the certificates together and treat them as one unit.
// They all have the same name and permissions but a different thumbprint.
// The first certificate in the collection will be the primary certificate and its thumbprint will be the one shown in a GET request
// The other certificates are secondary certificates and will contain a link to the primary certificate.
currentCertificate.Thumbprint = x509Certificate.Thumbprint;
currentCertificate.NotAfter = x509Certificate.NotAfter;
currentCertificate.Certificate = Convert.ToBase64String(x509Certificate.Export(X509ContentType.Cert));
if (first)
{
var firstKey = Constants.Certificates.Prefix + x509Certificate.Thumbprint;
collectionPrimaryKey = firstKey;
foreach (var cert in collection)
{
if (Constants.Certificates.Prefix + cert.Thumbprint != firstKey)
{
currentCertificate.CollectionSecondaryKeys.Add(Constants.Certificates.Prefix + cert.Thumbprint);
}
}
}
else
{
currentCertificate.CollectionPrimaryKey = collectionPrimaryKey;
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + x509Certificate.Thumbprint, currentCertificate));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
first = false;
}
NoContentStatus();
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
}
}
public PutCertificateWithSamePinningHashCommand(string name, CertificateDefinition value, string uniqueRequestId) : base(name, value, uniqueRequestId)
{
}
public async Task AddNode()
{
var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
var tag = GetStringQueryString("tag", false);
var watcher = GetBoolValueQueryString("watcher", false);
var raftRequestId = GetRaftRequestIdFromQuery();
var maxUtilizedCores = GetIntValueQueryString("maxUtilizedCores", false);
if (maxUtilizedCores != null && maxUtilizedCores <= 0)
{
throw new ArgumentException("Max utilized cores cores must be greater than 0");
}
nodeUrl = nodeUrl.Trim();
if (Uri.IsWellFormedUriString(nodeUrl, UriKind.Absolute) == false)
{
throw new InvalidOperationException($"Given node URL '{nodeUrl}' is not in a correct format.");
}
nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
tag = tag?.Trim();
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
// test connection to remote.
var result = await ServerStore.TestConnectionToRemote(nodeUrl, database : null);
if (result.Success == false)
{
throw new InvalidOperationException(result.Error);
}
// test connection from remote to destination
result = await ServerStore.TestConnectionFromRemote(requestExecutor, ctx, nodeUrl);
if (result.Success == false)
{
throw new InvalidOperationException(result.Error);
}
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (SchemaUpgrader.CurrentVersion.ServerVersion != nodeInfo.ServerSchemaVersion)
{
var nodesVersion = nodeInfo.ServerSchemaVersion == 0 ? "Pre 4.2 version" : nodeInfo.ServerSchemaVersion.ToString();
throw new InvalidOperationException($"Can't add node with mismatched storage schema version.{Environment.NewLine}" +
$"My version is {SchemaUpgrader.CurrentVersion.ServerVersion}, while node's version is {nodesVersion}");
}
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (ServerStore.ValidateFixedPort && nodeInfo.HasFixedPort == false)
{
throw new InvalidOperationException($"Failed to add node '{nodeUrl}' to cluster. " +
$"Node '{nodeUrl}' has port '0' in 'Configuration.Core.ServerUrls' setting. " +
"Adding a node with non fixed port is forbidden. Define a fixed port for the node to enable cluster creation.");
}
await ServerStore.EnsureNotPassiveAsync();
ServerStore.LicenseManager.AssertCanAddNode();
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
var clusterTopology = ServerStore.GetClusterTopology();
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
if (nodeInfo.ServerId == ServerStore.GetServerId())
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
}
if (nodeInfo.TopologyId != null)
{
AssertCanAddNodeWithTopologyId(clusterTopology, nodeInfo, nodeUrl);
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? tag : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);
var now = DateTime.UtcNow;
if (certificate.NotBefore.ToUniversalTime() > now)
{
// Because of time zone and time drift issues, we can't assume that the certificate generation will be
// proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
// current time. Clients may still fail to work with our certificate because of timing issues,
// but the admin needs to setup time sync properly and there isn't much we can do at that point
if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
{
throw new InvalidOperationException(
$"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
}
if (certificate.NotAfter.ToUniversalTime() < now)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbprint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
// if it's the same server certificate as our own, we don't want to add it to the cluster
if (certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
{
using (ctx.OpenReadTransaction())
{
var readCert = ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
PublicKeyPinningHash = certificate.GetPublicKeyPinningHash(),
NotAfter = certificate.NotAfter,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, certificateDefinition,
$"{raftRequestId}/put-new-certificate"));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : true, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null && certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
{
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, modifiedServerCert, $"{raftRequestId}/put-modified-certificate"));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
var detailsPerNode = new DetailsPerNode
{
MaxUtilizedCores = maxUtilizedCores,
NumberOfCores = nodeInfo.NumberOfCores,
InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
UsableMemoryInGb = nodeInfo.UsableMemoryInGb,
BuildInfo = nodeInfo.BuildInfo,
OsInfo = nodeInfo.OsInfo
};
var maxCores = ServerStore.LicenseManager.LicenseStatus.MaxCores;
try
{
await ServerStore.PutNodeLicenseLimitsAsync(nodeTag, detailsPerNode, maxCores, $"{raftRequestId}/put-license-limits");
}
catch
{
// we'll retry this again later
}
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetStringQueryString("url").TrimEnd('/');
var watcher = GetBoolValueQueryString("watcher", false);
var assignedCores = GetIntValueQueryString("assignedCores", false);
if (assignedCores <= 0)
{
throw new ArgumentException("Assigned cores must be greater than 0!");
}
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.ClusterCertificateHolder.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (assignedCores == null)
{
assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
}
if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
{
throw new ArgumentException("Cannot add node because the assigned cores is larger " +
$"than the available cores on that machine: {nodeInfo.NumberOfCores}");
}
ServerStore.EnsureNotPassive();
if (ServerStore.LicenseManager.CanAddNode(nodeUrl, assignedCores, out var licenseLimit) == false)
{
SetLicenseLimitResponse(licenseLimit);
return;
}
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
topologyId = clusterTopology.TopologyId;
}
if (nodeInfo.TopologyId != null && topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate));
if (certificate.NotBefore > DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
if (certificate.NotAfter < DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbrpint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
using (ctx.OpenReadTransaction())
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var readCert = ServerStore.Cluster.Read(ctx, key);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null)
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
ServerStore.LicenseManager.CalculateLicenseLimits(nodeTag, assignedCores, nodeInfo, forceFetchingNodeInfo: true);
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetStringQueryString("url").TrimEnd('/');
var watcher = GetBoolValueQueryString("watcher", false);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.ClusterCertificateHolder.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
ServerStore.EnsureNotPassive();
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
topologyId = clusterTopology.TopologyId;
}
if (nodeInfo.TopologyId != null && topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag
? null : nodeInfo.NodeTag;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because it has no certificate while trying to use HTTPS");
}
var certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate));
if (certificate.NotBefore > DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
if (certificate.NotAfter < DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbrpint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
NoContentStatus();
return;
}
}
RedirectToLeader();
}
private void ValidateCaExistsInOsStores(CertificateDefinition certificate)
{
// Implementation of SslStream AuthenticateAsServer is different in Linux. See RavenDB-8524
// A list of allowed CAs is sent from the server to the client. The handshake will fail if the client's CA is not in that list. This list is taken from the root and certificate authority stores of the OS.
// In this workaround we make sure that the CA (who signed the certificate) is registered in one of the OS stores.
var x509Certificate2 = new X509Certificate2(Convert.FromBase64String(certificate.Certificate));
var chain = new X509Chain
{
ChainPolicy =
{
RevocationMode = X509RevocationMode.NoCheck,
RevocationFlag = X509RevocationFlag.ExcludeRoot,
VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority,
VerificationTime = DateTime.UtcNow,
UrlRetrievalTimeout = new TimeSpan(0, 0, 0)
}
};
if (chain.Build(x509Certificate2) == false)
{
var status = new StringBuilder();
if (chain.ChainStatus.Length != 0)
{
status.Append("Chain Status:\r\n");
foreach (var chainStatus in chain.ChainStatus)
{
status.Append(chainStatus.Status + " : " + chainStatus.StatusInformation + "\r\n");
}
}
throw new InvalidOperationException($"The certificate chain for {certificate.Name} is broken, admin assistance required. {status}");
}
var rootCert = GetRootCertificate(chain);
if (rootCert == null)
{
throw new InvalidOperationException($"The certificate chain for {certificate.Name} is broken. Reason: partial chain, cannot extract CA from chain. Admin assistance required.");
}
using (var machineRootStore = new X509Store(StoreName.Root, StoreLocation.LocalMachine, OpenFlags.ReadOnly))
using (var machineCaStore = new X509Store(StoreName.CertificateAuthority, StoreLocation.LocalMachine, OpenFlags.ReadOnly))
using (var userRootStore = new X509Store(StoreName.Root, StoreLocation.CurrentUser, OpenFlags.ReadOnly))
using (var userCaStore = new X509Store(StoreName.CertificateAuthority, StoreLocation.CurrentUser, OpenFlags.ReadOnly))
{
// workaround for lack of cert store inheritance RavenDB-8904
if (machineCaStore.Certificates.Contains(rootCert) == false &&
machineRootStore.Certificates.Contains(rootCert) == false &&
userCaStore.Certificates.Contains(rootCert) == false &&
userRootStore.Certificates.Contains(rootCert) == false)
{
var path = new[]
{
ServerStore.Configuration.Security.CertificatePath,
ServerStore.Configuration.Security.ClusterCertificatePath,
ServerStore.Configuration.Security.CertificateExec,
ServerStore.Configuration.Security.ClusterCertificateExec
}.FirstOrDefault(File.Exists) ?? "no path defined";
throw new InvalidOperationException($"Cannot save the client certificate '{certificate.Name}'. " +
$"First, you must register the CA of the certificate '{rootCert.SubjectName.Name}' in the trusted root store, on the server machine." +
$"The server certificate is located in: '{path}'" +
"This step is required because you are trying to save a certificate which was signed by an unknown or self-signed certificate authority.");
}
}
}
