private static void GenerateSelfSignedCertFileName()
{
lock (typeof(TestBase))
{
if (_selfSignedCertFileName != null)
{
return;
}
var log = new StringBuilder();
byte[] certBytes;
try
{
certBytes = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "RavenTestsServer", log);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to generate the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
try
{
new X509Certificate2(certBytes, (string)null, X509KeyStorageFlags.MachineKeySet);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to load the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
if (certBytes.Length == 0)
{
throw new CryptographicException($"Test certificate length is 0 bytes. Machine: '{Environment.MachineName}', Log: {log}");
}
string tempFileName = null;
try
{
tempFileName = Path.GetTempFileName();
File.WriteAllBytes(tempFileName, certBytes);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to write the test certificate to a temp file." +
$"tempFileName = {tempFileName}" +
$"certBytes.Length = {certBytes.Length}" +
$"MachineName = {Environment.MachineName}.", e);
}
_selfSignedCertFileName = tempFileName;
GlobalPathsToDelete.Add(_selfSignedCertFileName);
}
}
public void PublicKeyPinningHashShouldNotBeEqual()
{
// Different private key
var c1 = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "first");
var c2 = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "second");
var c1Cert = new X509Certificate2(c1);
var c2Cert = new X509Certificate2(c2);
var h1 = c1Cert.GetPublicKeyPinningHash();
var h2 = c2Cert.GetPublicKeyPinningHash();
Assert.NotEqual(h1, h2);
}
protected TestCertificatesHolder GenerateAndSaveSelfSignedCertificate(bool createNew = false)
{
var selfSignedCertificatePaths = _selfSignedCertificates;
if (selfSignedCertificatePaths != null && createNew == false)
{
return(ReturnCertificatesHolder(selfSignedCertificatePaths));
}
lock (typeof(TestBase))
{
selfSignedCertificatePaths = _selfSignedCertificates;
if (selfSignedCertificatePaths == null || createNew)
{
_selfSignedCertificates = selfSignedCertificatePaths = Generate();
}
return(ReturnCertificatesHolder(selfSignedCertificatePaths));
}
TestCertificatesHolder ReturnCertificatesHolder(TestCertificatesHolder certificates)
{
return(new TestCertificatesHolder(certificates, GetTempFileName));
}
TestCertificatesHolder Generate()
{
var log = new StringBuilder();
byte[] certBytes;
try
{
certBytes = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "RavenTestsServer", log);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to generate the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
X509Certificate2 serverCertificate;
try
{
serverCertificate = new X509Certificate2(certBytes, (string)null, X509KeyStorageFlags.MachineKeySet);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to load the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
if (certBytes.Length == 0)
{
throw new CryptographicException($"Test certificate length is 0 bytes. Machine: '{Environment.MachineName}', Log: {log}");
}
string serverCertificatePath = null;
try
{
serverCertificatePath = Path.GetTempFileName();
File.WriteAllBytes(serverCertificatePath, certBytes);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to write the test certificate to a temp file." +
$"tempFileName = {serverCertificatePath}" +
$"certBytes.Length = {certBytes.Length}" +
$"MachineName = {Environment.MachineName}.", e);
}
GlobalPathsToDelete.Add(serverCertificatePath);
SecretProtection.ValidatePrivateKey(serverCertificatePath, null, certBytes, out var pk);
var clientCertificate1Path = GenerateClientCertificate(1, serverCertificate, pk);
var clientCertificate2Path = GenerateClientCertificate(2, serverCertificate, pk);
var clientCertificate3Path = GenerateClientCertificate(3, serverCertificate, pk);
return(new TestCertificatesHolder(serverCertificatePath, clientCertificate1Path, clientCertificate2Path, clientCertificate3Path));
}
string GenerateClientCertificate(int index, X509Certificate2 serverCertificate, Org.BouncyCastle.Pkcs.AsymmetricKeyEntry pk)
{
CertificateUtils.CreateSelfSignedClientCertificate(
$"{Environment.MachineName}_CC_{index}",
new RavenServer.CertificateHolder
{
Certificate = serverCertificate,
PrivateKey = pk
},
out var certBytes, DateTime.UtcNow.Date.AddYears(5));
string clientCertificatePath = null;
try
{
clientCertificatePath = Path.GetTempFileName();
File.WriteAllBytes(clientCertificatePath, certBytes);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to write the test certificate to a temp file." +
$"tempFileName = {clientCertificatePath}" +
$"certBytes.Length = {certBytes.Length}" +
$"MachineName = {Environment.MachineName}.", e);
}
GlobalPathsToDelete.Add(clientCertificatePath);
return(clientCertificatePath);
}
}
public async Task CanReplaceClusterCert()
{
var clusterSize = 3;
var databaseName = GetDatabaseName();
var leader = await CreateRaftClusterAndGetLeader(clusterSize, false, useSsl : true);
X509Certificate2 adminCertificate = null;
adminCertificate = AskServerForClientCertificate(_selfSignedCertFileName, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin, server: leader);
DatabasePutResult databaseResult;
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Certificate = adminCertificate,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
}
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers.Where(s => databaseResult.NodesAddedTo.Any(n => n == s.WebUrl)))
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
using (var store = new DocumentStore()
{
Urls = new[] { databaseResult.NodesAddedTo[0] },
Database = databaseName,
Certificate = adminCertificate,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Karmelush" }, "users/1");
await session.SaveChangesAsync();
}
var certBytes = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "RavenTestsServerReplacementCert");
var newServerCert = new X509Certificate2(certBytes, (string)null, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.PersistKeySet | X509KeyStorageFlags.MachineKeySet);
var mre = new ManualResetEventSlim();
leader.ServerCertificateChanged += (sender, args) => mre.Set();
var requestExecutor = store.GetRequestExecutor();
using (requestExecutor.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var command = new ReplaceClusterCertificateOperation(certBytes, false)
.GetCommand(store.Conventions, context);
requestExecutor.Execute(command, context);
}
Assert.True(mre.Wait(Debugger.IsAttached ? TimeSpan.FromMinutes(10) : TimeSpan.FromMinutes(2)), "Waited too long");
Assert.True(leader.Certificate.Certificate.Thumbprint.Equals(newServerCert.Thumbprint), "New cert is identical");
using (var session = store.OpenSession())
{
var user1 = session.Load <User>("users/1");
Assert.NotNull(user1);
Assert.Equal("Karmelush", user1.Name);
}
}
}
public async Task WillNotTrustNewClientCertIfPublicKeyPinningHashIsDifferent()
{
// Setting up two clusters with external replication cluster1 --> cluster2
var clusterSize = 1;
var databaseName = GetDatabaseName();
// We generate the first certificate before calling CreateRaftClusterAndGetLeader so that the two clusters will have separate certificates
var cluster1CertBytes = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "RavenTests");
var cluster1CertFileName = GetTempFileName();
File.WriteAllBytes(cluster1CertFileName, cluster1CertBytes);
var leader1 = await CreateRaftClusterAndGetLeader(clusterSize, false, useSsl : true, serverCertPath : cluster1CertFileName);
var cluster1Cert = new X509Certificate2(cluster1CertFileName);
var adminCertificate1 = AskServerForClientCertificate(cluster1CertFileName, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin, server: leader1);
await CreateDatabaseInCluster(databaseName, clusterSize, leader1.WebUrl, adminCertificate1);
// Cluster 2 gets a normal test certificate
var leader2 = await CreateRaftClusterAndGetLeader(clusterSize, false, useSsl : true);
var cluster2Cert = new X509Certificate2(_selfSignedCertFileName);
var adminCertificate2 = AskServerForClientCertificate(_selfSignedCertFileName, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin, server: leader2);
await CreateDatabaseInCluster(databaseName, clusterSize, leader2.WebUrl, adminCertificate2);
// This will register cluster 1's cert as a user cert in cluster 2
AskCluster2ToTrustCluster1(cluster1Cert, cluster2Cert, new Dictionary <string, DatabaseAccess>
{
[databaseName] = DatabaseAccess.ReadWrite
}, SecurityClearance.ValidUser, leader2);
// First we'll make sure external replication works between the two clusters
using (var store1 = new DocumentStore
{
Urls = new[] { leader1.WebUrl },
Database = databaseName,
Certificate = adminCertificate1
}.Initialize())
using (var store2 = new DocumentStore
{
Urls = new[] { leader2.WebUrl },
Database = databaseName,
Certificate = adminCertificate2
}.Initialize())
{
var externalList = await SetupReplicationAsync((DocumentStore)store1, (DocumentStore)store2);
using (var session = store1.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Karmelush" }, "users/1");
await session.SaveChangesAsync();
}
var replicated1 = WaitForDocumentToReplicate <User>(store2, "users/1", 10000);
Assert.NotNull(replicated1);
Assert.Equal("Karmelush", replicated1.Name);
// Let's replace the certificate in cluster 1 (new cert has different private key) and make sure cluster 2 WILL NOT trusts cluster 1.
var certBytes = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "ReplacementCertDifferentKey");
var cluster1ReplacementCert = new X509Certificate2(certBytes);
var mre = new ManualResetEventSlim();
leader1.ServerCertificateChanged += (sender, args) => mre.Set();
var requestExecutor = store1.GetRequestExecutor();
using (requestExecutor.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var command = new ReplaceClusterCertificateOperation(certBytes, false)
.GetCommand(store1.Conventions, context);
requestExecutor.Execute(command, context);
}
Assert.True(mre.Wait(Debugger.IsAttached ? TimeSpan.FromMinutes(10) : TimeSpan.FromMinutes(2)), "Waited too long");
Assert.NotNull(leader1.Certificate.Certificate.Thumbprint);
Assert.True(leader1.Certificate.Certificate.Thumbprint.Equals(cluster1ReplacementCert.Thumbprint), "New cert is identical");
// Disable external replication
var external = new ExternalReplication(store1.Database, $"ConnectionString-{store2.Identifier}")
{
TaskId = externalList.First().TaskId,
Disabled = true
};
var responsibleNode = externalList[0].ResponsibleNode;
var clusterNodes1 = Servers.Where(s => s.ServerStore.GetClusterTopology().TryGetNodeTagByUrl(leader1.WebUrl).HasUrl);
var node = clusterNodes1.Single(n => n.ServerStore.NodeTag == responsibleNode);
var db1 = await node.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store1.Database);
Assert.NotNull(db1);
var replicationConnection = db1.ReplicationLoader.OutgoingHandlers.Single(x => x.Destination is ExternalReplication);
var res = await store1.Maintenance.SendAsync(new UpdateExternalReplicationOperation(external));
Assert.Equal(externalList.First().TaskId, res.TaskId);
// Make sure the command is processed
await db1.ServerStore.Cluster.WaitForIndexNotification(res.RaftCommandIndex);
var connectionDropped = await WaitForValueAsync(() => replicationConnection.IsConnectionDisposed, true);
Assert.True(connectionDropped);
// Enable external replication
external.Disabled = false;
res = await store1.Maintenance.SendAsync(new UpdateExternalReplicationOperation(external));
await db1.ServerStore.Cluster.WaitForIndexNotification(res.RaftCommandIndex);
Assert.Equal(externalList.First().TaskId, res.TaskId);
using (var session = store1.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Avivush" }, "users/2").ConfigureAwait(false);
await session.SaveChangesAsync();
}
//WaitForUserToContinueTheTest(store1, clientCert: adminCertificate1);
// WaitForUserToContinueTheTest(store2, clientCert: adminCertificate2);
var replicated = WaitForDocumentToReplicate <User>(store2, "users/2", 10000);
Assert.Null(replicated);
// RavenDB-13010
Assert.True(WaitForValue(() =>
{
var repStats = store1.Maintenance.Send(new GetReplicationPerformanceStatisticsOperation());
return(repStats.Outgoing.SelectMany(x => x.Performance).Count(x => x.Errors.Count > 0) > 0);
}, true));
}
}
public TestCertificatesHolder GenerateAndSaveSelfSignedCertificate(bool createNew = false, [CallerMemberName] string caller = null)
{
if (createNew)
{
return(ReturnCertificatesHolder(Generate(caller, Interlocked.Increment(ref Counter))));
}
var selfSignedCertificates = SelfSignedCertificates;
if (selfSignedCertificates != null)
{
return(ReturnCertificatesHolder(selfSignedCertificates));
}
lock (typeof(TestBase))
{
selfSignedCertificates = SelfSignedCertificates;
if (selfSignedCertificates == null)
{
SelfSignedCertificates = selfSignedCertificates = Generate(caller);
}
return(ReturnCertificatesHolder(selfSignedCertificates));
}
TestCertificatesHolder ReturnCertificatesHolder(TestCertificatesHolder certificates)
{
return(new TestCertificatesHolder(certificates, _parent.GetTempFileName));
}
TestCertificatesHolder Generate(string caller, int gen = 0)
{
var log = new StringBuilder();
byte[] certBytes;
string serverCertificatePath = null;
serverCertificatePath = Path.Combine(Path.GetTempPath(), $"Server-{gen}-{RavenVersionAttribute.Instance.Build}-{DateTime.Today:yyyy-MM-dd}.pfx");
if (File.Exists(serverCertificatePath) == false)
{
try
{
certBytes = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "RavenTestsServer", log);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to generate the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
if (certBytes.Length == 0)
{
throw new CryptographicException($"Test certificate length is 0 bytes. Machine: '{Environment.MachineName}', Log: {log}");
}
try
{
File.WriteAllBytes(serverCertificatePath, certBytes);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to write the test certificate to a temp file." +
$"tempFileName = {serverCertificatePath}" +
$"certBytes.Length = {certBytes.Length}" +
$"MachineName = {Environment.MachineName}.", e);
}
}
else
{
certBytes = File.ReadAllBytes(serverCertificatePath);
}
X509Certificate2 serverCertificate;
try
{
serverCertificate = new X509Certificate2(certBytes, (string)null, X509KeyStorageFlags.MachineKeySet);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to load the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
SecretProtection.ValidatePrivateKey(serverCertificatePath, null, certBytes, out var pk);
SecretProtection.ValidateKeyUsages(serverCertificatePath, serverCertificate, validateKeyUsages: true);
var clientCertificate1Path = GenerateClientCertificate(1, serverCertificate, pk);
var clientCertificate2Path = GenerateClientCertificate(2, serverCertificate, pk);
var clientCertificate3Path = GenerateClientCertificate(3, serverCertificate, pk);
return(new TestCertificatesHolder(serverCertificatePath, clientCertificate1Path, clientCertificate2Path, clientCertificate3Path));
}
string GenerateClientCertificate(int index, X509Certificate2 serverCertificate, Org.BouncyCastle.Pkcs.AsymmetricKeyEntry pk)
{
string name = $"{Environment.MachineName}_CC_{RavenVersionAttribute.Instance.Build}_{index}_{DateTime.Today:yyyy-MM-dd}";
string clientCertificatePath = Path.Combine(Path.GetTempPath(), name + ".pfx");
if (File.Exists(clientCertificatePath) == false)
{
CertificateUtils.CreateSelfSignedClientCertificate(
name,
new RavenServer.CertificateHolder
{
Certificate = serverCertificate,
PrivateKey = pk
},
out var certBytes, DateTime.UtcNow.Date.AddYears(5));
try
{
File.WriteAllBytes(clientCertificatePath, certBytes);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to write the test certificate to a temp file." +
$"tempFileName = {clientCertificatePath}" +
$"certBytes.Length = {certBytes.Length}" +
$"MachineName = {Environment.MachineName}.", e);
}
}
return(clientCertificatePath);
}
}
