public string SetupEncryptedDatabase(out TestCertificatesHolder certificates, out byte[] masterKey, [CallerMemberName] string caller = null)
{
certificates = _parent.Certificates.SetupServerAuthentication();
var dbName = _parent.GetDatabaseName(caller);
_parent.Certificates.RegisterClientCertificate(certificates, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin);
string base64Key = CreateMasterKey(out masterKey);
EnsureServerMasterKeyIsSetup(_parent.Server);
_parent.Server.ServerStore.PutSecretKey(base64Key, dbName, true);
return(dbName);
}
public void EncryptedCluster(List <RavenServer> nodes, TestCertificatesHolder certificates, out string databaseName)
{
databaseName = _parent.GetDatabaseName();
foreach (var node in nodes)
{
_parent.Certificates.RegisterClientCertificate(certificates, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin, node);
var base64Key = CreateMasterKey(out _);
EnsureServerMasterKeyIsSetup(node);
Assert.True(node.ServerStore.EnsureNotPassiveAsync().Wait(TimeSpan.FromSeconds(30))); // activate license so we can insert the secret key
Assert.True(node.ServerStore.LicenseManager.TryActivateLicenseAsync(_parent.Server.ThrowOnLicenseActivationFailure).Wait(TimeSpan.FromSeconds(30))); // activate license so we can insert the secret key
node.ServerStore.PutSecretKey(base64Key, databaseName, overwrite: true);
}
}
public string EncryptedServer(out TestCertificatesHolder certificates, out string databaseName)
{
certificates = _parent.Certificates.SetupServerAuthentication();
databaseName = _parent.GetDatabaseName();
_parent.Certificates.RegisterClientCertificate(certificates, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin);
var buffer = new byte[32];
using (var rand = RandomNumberGenerator.Create())
{
rand.GetBytes(buffer);
}
var base64Key = Convert.ToBase64String(buffer);
var canUseProtect = PlatformDetails.RunningOnPosix == false;
if (canUseProtect)
{
// sometimes when using `dotnet xunit` we get platform not supported from ProtectedData
try
{
#pragma warning disable CA1416 // Validate platform compatibility
ProtectedData.Protect(Encoding.UTF8.GetBytes("Is supported?"), null, DataProtectionScope.CurrentUser);
#pragma warning restore CA1416 // Validate platform compatibility
}
catch (PlatformNotSupportedException)
{
canUseProtect = false;
}
}
if (canUseProtect == false) // fall back to a file
{
_parent.Server.ServerStore.Configuration.Security.MasterKeyPath = _parent.GetTempFileName();
}
Assert.True(_parent.Server.ServerStore.EnsureNotPassiveAsync().Wait(TimeSpan.FromSeconds(30))); // activate license so we can insert the secret key
Assert.True(_parent.Server.ServerStore.LicenseManager.TryActivateLicenseAsync(_parent.Server.ThrowOnLicenseActivationFailure).Wait(TimeSpan.FromSeconds(30))); // activate license so we can insert the secret key
_parent.Server.ServerStore.PutSecretKey(base64Key, databaseName, overwrite: true);
return(Convert.ToBase64String(buffer));
}
protected Dictionary<string, string> GetServerSettingsForPort(bool useSsl, out string serverUrl, out TestCertificatesHolder certificates)
{
var customSettings = new Dictionary<string, string>();
if (useSsl)
{
serverUrl = UseFiddlerUrl("https://127.0.0.1:0");
certificates = SetupServerAuthentication(customSettings, serverUrl);
}
else
{
serverUrl = UseFiddlerUrl("http://127.0.0.1:0");
customSettings[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = serverUrl;
certificates = null;
}
return customSettings;
}
protected async Task<(List<RavenServer> Nodes, RavenServer Leader, TestCertificatesHolder Certificates)> CreateRaftClusterInternalAsync(
int numberOfNodes,
bool? shouldRunInMemory = null,
int? leaderIndex = null,
bool useSsl = false,
IDictionary<string, string> customSettings = null,
List<IDictionary<string, string>> customSettingsList = null,
bool watcherCluster = false,
[CallerMemberName] string caller = null)
{
string[] allowedNodeTags = { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" };
var actualLeaderIndex = leaderIndex;
leaderIndex ??= _random.Next(0, numberOfNodes);
RavenServer leader = null;
var serversToPorts = new Dictionary<RavenServer, string>();
var clusterNodes = new List<RavenServer>(); // we need this in case we create more than 1 cluster in the same test
_electionTimeoutInMs = Math.Max(300, numberOfNodes * 80);
if (customSettingsList != null && customSettingsList.Count != numberOfNodes)
{
throw new InvalidOperationException("The number of custom settings must equal the number of nodes.");
}
TestCertificatesHolder certificates = null;
for (var i = 0; i < numberOfNodes; i++)
{
if (customSettingsList == null)
{
customSettings ??= new Dictionary<string, string>(DefaultClusterSettings)
{
[RavenConfiguration.GetKey(x => x.Cluster.ElectionTimeout)] = _electionTimeoutInMs.ToString(),
};
}
else
{
customSettings = customSettingsList[i];
}
string serverUrl;
if (useSsl)
{
serverUrl = UseFiddlerUrl("https://127.0.0.1:0");
certificates = SetupServerAuthentication(customSettings, serverUrl);
}
else
{
serverUrl = UseFiddlerUrl("http://127.0.0.1:0");
customSettings[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = serverUrl;
}
var co = new ServerCreationOptions
{
CustomSettings = customSettings,
RunInMemory = shouldRunInMemory,
RegisterForDisposal = false,
NodeTag = allowedNodeTags[i]
};
var server = GetNewServer(co, caller);
var port = Convert.ToInt32(server.ServerStore.GetNodeHttpServerUrl().Split(':')[2]);
var prefix = useSsl ? "https" : "http";
serverUrl = UseFiddlerUrl($"{prefix}://127.0.0.1:{port}");
Servers.Add(server);
clusterNodes.Add(server);
serversToPorts.Add(server, serverUrl);
if (i == leaderIndex)
{
await server.ServerStore.EnsureNotPassiveAsync(null, nodeTag: co.NodeTag);
leader = server;
}
}
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)))
{
for (var i = 0; i < numberOfNodes; i++)
{
if (i == leaderIndex)
{
continue;
}
var follower = clusterNodes[i];
// ReSharper disable once PossibleNullReferenceException
leader = await ActionWithLeader(l =>
l.ServerStore.AddNodeToClusterAsync(serversToPorts[follower], nodeTag: allowedNodeTags[i], asWatcher: watcherCluster, token: cts.Token), clusterNodes);
if (watcherCluster)
{
await follower.ServerStore.WaitForTopology(Leader.TopologyModification.NonVoter, cts.Token);
}
else
{
await follower.ServerStore.WaitForTopology(Leader.TopologyModification.Voter, cts.Token);
}
}
}
await WaitForClusterTopologyOnAllNodes(clusterNodes);
// ReSharper disable once PossibleNullReferenceException
var condition = await leader.ServerStore.WaitForState(RachisState.Leader, CancellationToken.None).WaitAsync(numberOfNodes * _electionTimeoutInMs * 5);
var states = "The leader has changed while waiting for cluster to become stable. All nodes status: ";
if (condition == false)
{
InvalidOperationException e = null;
if (actualLeaderIndex == null)
{
// leader changed, try get the new leader if no leader index was selected
try
{
leader = await ActionWithLeader(_ => Task.CompletedTask, clusterNodes);
return (clusterNodes, leader, certificates);
}
catch (InvalidOperationException ex)
{
e = ex;
}
}
states += GetLastStatesFromAllServersOrderedByTime();
if (e != null)
states += $"{Environment.NewLine}{e}";
}
Assert.True(condition, states);
return (clusterNodes, leader, certificates);
}
public TestCertificatesHolder GenerateAndSaveSelfSignedCertificate(bool createNew = false, [CallerMemberName] string caller = null)
{
if (createNew)
{
return(ReturnCertificatesHolder(Generate(caller, Interlocked.Increment(ref Counter))));
}
var selfSignedCertificates = SelfSignedCertificates;
if (selfSignedCertificates != null)
{
return(ReturnCertificatesHolder(selfSignedCertificates));
}
lock (typeof(TestBase))
{
selfSignedCertificates = SelfSignedCertificates;
if (selfSignedCertificates == null)
{
SelfSignedCertificates = selfSignedCertificates = Generate(caller);
}
return(ReturnCertificatesHolder(selfSignedCertificates));
}
TestCertificatesHolder ReturnCertificatesHolder(TestCertificatesHolder certificates)
{
return(new TestCertificatesHolder(certificates, _parent.GetTempFileName));
}
TestCertificatesHolder Generate(string caller, int gen = 0)
{
var log = new StringBuilder();
byte[] certBytes;
string serverCertificatePath = null;
serverCertificatePath = Path.Combine(Path.GetTempPath(), $"Server-{gen}-{RavenVersionAttribute.Instance.Build}-{DateTime.Today:yyyy-MM-dd}.pfx");
if (File.Exists(serverCertificatePath) == false)
{
try
{
certBytes = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "RavenTestsServer", log);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to generate the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
if (certBytes.Length == 0)
{
throw new CryptographicException($"Test certificate length is 0 bytes. Machine: '{Environment.MachineName}', Log: {log}");
}
try
{
File.WriteAllBytes(serverCertificatePath, certBytes);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to write the test certificate to a temp file." +
$"tempFileName = {serverCertificatePath}" +
$"certBytes.Length = {certBytes.Length}" +
$"MachineName = {Environment.MachineName}.", e);
}
}
else
{
certBytes = File.ReadAllBytes(serverCertificatePath);
}
X509Certificate2 serverCertificate;
try
{
serverCertificate = new X509Certificate2(certBytes, (string)null, X509KeyStorageFlags.MachineKeySet);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to load the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
SecretProtection.ValidatePrivateKey(serverCertificatePath, null, certBytes, out var pk);
SecretProtection.ValidateKeyUsages(serverCertificatePath, serverCertificate, validateKeyUsages: true);
var clientCertificate1Path = GenerateClientCertificate(1, serverCertificate, pk);
var clientCertificate2Path = GenerateClientCertificate(2, serverCertificate, pk);
var clientCertificate3Path = GenerateClientCertificate(3, serverCertificate, pk);
return(new TestCertificatesHolder(serverCertificatePath, clientCertificate1Path, clientCertificate2Path, clientCertificate3Path));
}
string GenerateClientCertificate(int index, X509Certificate2 serverCertificate, Org.BouncyCastle.Pkcs.AsymmetricKeyEntry pk)
{
string name = $"{Environment.MachineName}_CC_{RavenVersionAttribute.Instance.Build}_{index}_{DateTime.Today:yyyy-MM-dd}";
string clientCertificatePath = Path.Combine(Path.GetTempPath(), name + ".pfx");
if (File.Exists(clientCertificatePath) == false)
{
CertificateUtils.CreateSelfSignedClientCertificate(
name,
new RavenServer.CertificateHolder
{
Certificate = serverCertificate,
PrivateKey = pk
},
out var certBytes, DateTime.UtcNow.Date.AddYears(5));
try
{
File.WriteAllBytes(clientCertificatePath, certBytes);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to write the test certificate to a temp file." +
$"tempFileName = {clientCertificatePath}" +
$"certBytes.Length = {certBytes.Length}" +
$"MachineName = {Environment.MachineName}.", e);
}
}
return(clientCertificatePath);
}
}
public TestCertificatesHolder SetupServerAuthentication(IDictionary <string, string> customSettings = null, string serverUrl = null, TestCertificatesHolder certificates = null, [CallerMemberName] string caller = null)
{
if (customSettings == null)
{
customSettings = new ConcurrentDictionary <string, string>();
}
if (certificates == null)
{
certificates = GenerateAndSaveSelfSignedCertificate(caller: caller);
}
if (customSettings.TryGetValue(RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec), out var _) == false)
{
customSettings[RavenConfiguration.GetKey(x => x.Security.CertificatePath)] = certificates.ServerCertificatePath;
}
customSettings[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = serverUrl ?? "https://" + Environment.MachineName + ":0";
_parent.DoNotReuseServer(customSettings);
return(certificates);
}
public X509Certificate2 RegisterClientCertificate(TestCertificatesHolder certificates, Dictionary <string, DatabaseAccess> permissions, SecurityClearance clearance = SecurityClearance.ValidUser, RavenServer server = null)
{
return(RegisterClientCertificate(certificates.ServerCertificate.Value, certificates.ClientCertificate1.Value, permissions, clearance, server));
}
