public async Task ReAddWatcherNode()
{
var cluster = await CreateRaftCluster(2, watcherCluster : true);
var leader = cluster.Leader;
var watcher = cluster.Nodes.Single(x => x != leader);
await leader.ServerStore.RemoveFromClusterAsync(watcher.ServerStore.NodeTag);
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(leader.WebUrl, null))
using (requestExecutor.ContextPool.AllocateOperationContext(out var ctx))
{
await requestExecutor.ExecuteAsync(new AddClusterNodeCommand(watcher.WebUrl, watcher.ServerStore.NodeTag), ctx);
}
}
public static async Task <TcpConnectionInfo> GetTcpInfoAsync(string url, string databaseName, string tag, X509Certificate2 certificate)
{
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(url, certificate))
using (requestExecutor.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var getTcpInfoCommand = new GetTcpInfoCommand(tag, databaseName);
await requestExecutor.ExecuteAsync(getTcpInfoCommand, context);
var tcpConnectionInfo = getTcpInfoCommand.Result;
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase) && tcpConnectionInfo.Certificate == null)
{
throw new InvalidOperationException("Getting TCP info over HTTPS but the server didn't return the expected certificate to use over TCP, invalid response, aborting");
}
return(tcpConnectionInfo);
}
}
public async Task CanSnapshotCompareExchangeTombstones()
{
var leader = await CreateRaftClusterAndGetLeader(1);
using (var store = GetDocumentStore(options: new Options
{
Server = leader
}))
{
using (var session = store.OpenAsyncSession(new SessionOptions
{
TransactionMode = TransactionMode.ClusterWide
}))
{
session.Advanced.ClusterTransaction.CreateCompareExchangeValue("foo", "bar");
await session.SaveChangesAsync();
var result = await session.Advanced.ClusterTransaction.GetCompareExchangeValueAsync <string>("foo");
session.Advanced.ClusterTransaction.DeleteCompareExchangeValue(result);
await session.SaveChangesAsync();
}
var server2 = GetNewServer();
var server2Url = server2.ServerStore.GetNodeHttpServerUrl();
Servers.Add(server2);
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(leader.WebUrl, null))
using (requestExecutor.ContextPool.AllocateOperationContext(out var ctx))
{
await requestExecutor.ExecuteAsync(new AddClusterNodeCommand(server2Url, watcher : true), ctx);
var addDatabaseNode = new AddDatabaseNodeOperation(store.Database);
await store.Maintenance.Server.SendAsync(addDatabaseNode);
}
using (server2.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (ctx.OpenReadTransaction())
{
Assert.True(server2.ServerStore.Cluster.HasCompareExchangeTombstones(ctx, store.Database));
}
}
}
public async Task RemoveEntryFromLog()
{
var index = GetLongQueryString("index");
var first = GetBoolValueQueryString("first", false) ?? true;
var nodeList = new List <string>();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var removed = ServerStore.Engine.RemoveEntryFromRaftLog(index);
if (removed)
{
nodeList.Add(ServerStore.NodeTag);
}
if (first)
{
foreach (var node in ServerStore.GetClusterTopology(context).AllNodes)
{
if (node.Value == Server.WebUrl)
{
continue;
}
var cmd = new RemoveEntryFromRaftLogCommand(index);
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(node.Value, Server.Certificate.Certificate))
{
await requestExecutor.ExecuteAsync(cmd, context);
nodeList.AddRange(cmd.Result);
}
}
}
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WriteArray("Nodes", nodeList);
writer.WriteEndObject();
}
}
}
public async Task FailOnAddingNodeThatHasPortZero()
{
var leader = await CreateRaftClusterAndGetLeader(1);
leader.ServerStore.Configuration.Core.ServerUrls = new[] { leader.WebUrl };
leader.ServerStore.ValidateFixedPort = true;
var server2 = GetNewServer();
var server2Url = server2.ServerStore.GetNodeHttpServerUrl();
Servers.Add(server2);
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(leader.WebUrl, null))
using (requestExecutor.ContextPool.AllocateOperationContext(out var ctx))
{
var ex = await Assert.ThrowsAsync<RavenException>(async () =>
await requestExecutor.ExecuteAsync(new AddClusterNodeCommand(server2Url), ctx));
Assert.Contains($"Node '{server2Url}' has port '0' in 'Configuration.Core.ServerUrls' setting. " +
"Adding a node with non fixed port is forbidden. Define a fixed port for the node to enable cluster creation.", ex.Message);
}
}
public async Task CanSnapshotManyCompareExchangeWithExpirationToManyNodes()
{
var count = 3 * 1024;
var nodesCount = 7;
using var leader = GetNewServer();
using (var store = GetDocumentStore(new Options
{
Server = leader
}))
{
var now = DateTime.UtcNow;
var expiry = now.AddMinutes(2);
var compareExchanges = new Dictionary <string, User>();
await AddCompareExchangesWithExpire(count, compareExchanges, store, expiry);
for (int i = 0; i < nodesCount; i++)
{
var follower = GetNewServer();
ServersForDisposal.Add(follower);
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(leader.WebUrl, null))
using (requestExecutor.ContextPool.AllocateOperationContext(out var ctx))
{
await requestExecutor.ExecuteAsync(new AddClusterNodeCommand(follower.WebUrl, watcher : true), ctx);
}
await follower.ServerStore.Engine.WaitForTopology(Leader.TopologyModification.NonVoter);
}
leader.ServerStore.Observer.Time.UtcDateTime = () => now.AddMinutes(3);
var val = await WaitForValueAsync(async() =>
{
var stats = await store.Maintenance.SendAsync(new GetDetailedStatisticsOperation());
return(stats.CountOfCompareExchange);
}, 0);
Assert.Equal(0, val);
}
}
private static async Task <string[]> GetClusterNodeUrlsAsync(string leadersUrl, IDocumentStore store)
{
string[] urls;
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(leadersUrl, store.Certificate))
{
try
{
await requestExecutor.UpdateTopologyAsync(new ServerNode
{ Url = leadersUrl }, 15000, true);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
urls = requestExecutor.Topology.Nodes.Select(x => x.Url).ToArray();
}
return(urls);
}
public async Task MoveOnToNextTcpAddressOnGuidFailInCluster()
{
//Leader asks node for tcpInfo and node answers with Leader's url. Leader will try to connect to Node and end up connecting to itself,
//guid check will fail, and failover will happen.
var cluster = await CreateRaftClusterWithSsl(1, watcherCluster : true);
var serverB = CreateSecuredServer(cluster.Leader.ServerStore.GetNodeTcpServerUrl(), uniqueCerts: false);
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(cluster.Leader.WebUrl, cluster.Leader.Certificate.Certificate))
using (requestExecutor.ContextPool.AllocateOperationContext(out var ctx))
{
string database = GetDatabaseName();
await requestExecutor.ExecuteAsync(new AddClusterNodeCommand(serverB.WebUrl, serverB.ServerStore.NodeTag), ctx);
using (var leaderStore =
new DocumentStore {
Urls = new[] { cluster.Leader.WebUrl }, Certificate = cluster.Leader.Certificate.Certificate, Database = database
}.Initialize())
{
var res = leaderStore.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord(database)));
}
using (var storeB = new DocumentStore {
Urls = new[] { serverB.WebUrl }, Certificate = serverB.Certificate.Certificate, Database = database
}.Initialize())
{
var name = await WaitForValueAsync(() =>
{
var record = storeB.Maintenance.Server.Send(new GetDatabaseRecordOperation(database));
return(record.DatabaseName);
}, database);
Assert.Equal(database, name);
}
}
}
private async Task WriteDebugInfoPackageForNodeAsync(
JsonOperationContext context,
ZipArchive archive,
string tag,
string url,
IEnumerable <string> databaseNames,
X509Certificate2 certificate,
bool stacktraces)
{
//note : theoretically GetDebugInfoFromNodeAsync() can throw, error handling is done at the level of WriteDebugInfoPackageForNodeAsync() calls
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(url, certificate))
{
var timeout = TimeSpan.FromMinutes(1);
if (ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan > timeout)
{
timeout = ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan;
}
requestExecutor.DefaultTimeout = timeout;
using (var responseStream = await GetDebugInfoFromNodeAsync(
context,
requestExecutor,
databaseNames ?? EmptyStringArray,
stacktraces))
{
var entry = archive.CreateEntry($"Node - [{tag}].zip");
entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;
using (var entryStream = entry.Open())
{
await responseStream.CopyToAsync(entryStream);
await entryStream.FlushAsync();
}
}
}
}
public async Task AddNodeToClusterWithoutError()
{
var(_, leader) = await CreateRaftCluster(1);
var server2 = GetNewServer();
var server2Url = server2.ServerStore.GetNodeHttpServerUrl();
Servers.Add(server2);
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(leader.WebUrl, null))
using (requestExecutor.ContextPool.AllocateOperationContext(out var ctx))
{
await requestExecutor.ExecuteAsync(new AddClusterNodeCommand(server2Url, watcher : true), ctx);
await server2.ServerStore.Engine.WaitForTopology(Leader.TopologyModification.NonVoter);
}
using (server2.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
var logs = ctx.ReadObject(server2.ServerStore.Engine.InMemoryDebug.ToJson(), "watcher-logs").ToString();
Assert.False(logs.Contains("Exception"), logs);
}
}
protected async Task <List <ProcessNode> > CreateCluster(string[] peers, IDictionary <string, string> customSettings = null, X509Certificate2 certificate = null)
{
var processes = new List <ProcessNode>();
foreach (var peer in peers)
{
processes.Add(await GetServerAsync(peer));
}
var chosenOne = processes[0];
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(chosenOne.Url, certificate))
using (requestExecutor.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
foreach (var processNode in processes)
{
if (processNode == chosenOne)
{
continue;
}
var addCommand = new AddClusterNodeCommand(processNode.Url);
await requestExecutor.ExecuteAsync(addCommand, context);
}
var clusterCreated = await WaitForValueAsync(async() =>
{
var clusterTopology = new GetClusterTopologyCommand();
await requestExecutor.ExecuteAsync(clusterTopology, context);
return(clusterTopology.Result.Topology.Members.Count);
}, peers.Length);
Assert.True(clusterCreated == peers.Length, "Failed to create initial cluster");
}
return(processes);
}
public ServerOperationExecutor ForNode(string nodeTag)
{
if (string.IsNullOrWhiteSpace(nodeTag))
{
throw new ArgumentException("Value cannot be null or whitespace.", nameof(nodeTag));
}
if (string.Equals(_nodeTag, nodeTag, StringComparison.OrdinalIgnoreCase))
{
return(this);
}
if (_store.Conventions.DisableTopologyUpdates)
{
throw new InvalidOperationException($"Cannot switch server operation executor, because {nameof(Conventions)}.{nameof(_store.Conventions.DisableTopologyUpdates)} is set to 'true'.");
}
return(_cache.GetOrAdd(nodeTag, tag =>
{
var requestExecutor = _initialRequestExecutor ?? _requestExecutor;
var topology = GetTopology(requestExecutor);
var node = topology
.Nodes
.Find(x => string.Equals(x.ClusterTag, tag, StringComparison.OrdinalIgnoreCase));
if (node == null)
{
throw new InvalidOperationException($"Could not find node '{tag}' in the topology. Available nodes: [{string.Join(", ", topology.Nodes.Select(x => x.ClusterTag))}]");
}
var clusterExecutor = ClusterRequestExecutor.CreateForSingleNode(node.Url, _store.Certificate, _store.Conventions);
return new ServerOperationExecutor(_store, clusterExecutor, requestExecutor, _cache, node.ClusterTag);
}));
}
private ServerOperationExecutor(DocumentStoreBase store, ClusterRequestExecutor requestExecutor, ClusterRequestExecutor initialRequestExecutor, ConcurrentDictionary <string, ServerOperationExecutor> cache, string nodeTag)
{
if (store == null)
{
throw new ArgumentNullException(nameof(store));
}
if (requestExecutor == null)
{
throw new ArgumentNullException(nameof(requestExecutor));
}
_store = store;
_requestExecutor = requestExecutor;
_initialRequestExecutor = initialRequestExecutor;
_nodeTag = nodeTag;
_cache = cache;
store.RegisterEvents(_requestExecutor);
if (_nodeTag == null)
{
store.AfterDispose += (sender, args) => Dispose();
}
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
var watcher = GetBoolValueQueryString("watcher", false);
var assignedCores = GetIntValueQueryString("assignedCores", false);
if (assignedCores <= 0)
{
throw new ArgumentException("Assigned cores must be greater than 0!");
}
nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
{
throw new ArgumentException("Cannot add node because the assigned cores is larger " +
$"than the available cores on that machine: {nodeInfo.NumberOfCores}");
}
ServerStore.EnsureNotPassive();
if (assignedCores == null)
{
assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
}
Debug.Assert(assignedCores <= nodeInfo.NumberOfCores);
ServerStore.LicenseManager.AssertCanAddNode(nodeUrl, assignedCores.Value);
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
ClusterTopology clusterTopology;
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
topologyId = clusterTopology.TopologyId;
}
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
if (nodeInfo.ServerId == ServerStore.GetServerId())
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
}
if (nodeInfo.TopologyId != null)
{
if (topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. " +
$"Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
if (nodeInfo.CurrentState != RachisState.Passive)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " +
$"because it's already in the cluster under tag :{nodeInfo.NodeTag} " +
$"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}");
}
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);
var now = DateTime.UtcNow;
if (certificate.NotBefore.ToUniversalTime() > now)
{
// Because of time zone and time drift issues, we can't assume that the certificate generation will be
// proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
// current time. Clients may still fail to work with our certificate because of timing issues,
// but the admin needs to setup time sync properly and there isn't much we can do at that point
if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
{
throw new InvalidOperationException(
$"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
}
if (certificate.NotAfter.ToUniversalTime() < now)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbprint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
using (ctx.OpenReadTransaction())
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var readCert = ServerStore.Cluster.Read(ctx, key);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
NotAfter = certificate.NotAfter,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null)
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
var nodeDetails = new NodeDetails
{
NodeTag = nodeTag,
AssignedCores = assignedCores.Value,
NumberOfCores = nodeInfo.NumberOfCores,
InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
UsableMemoryInGb = nodeInfo.UsableMemoryInGb,
BuildInfo = nodeInfo.BuildInfo
};
await ServerStore.LicenseManager.CalculateLicenseLimits(nodeDetails, forceFetchingNodeInfo : true, waitToUpdate : true);
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
public ServerOperationExecutor(DocumentStoreBase store)
{
_requestExecutor = store.Conventions.DisableTopologyUpdates
? ClusterRequestExecutor.CreateForSingleNode(store.Urls[0], store.Certificate)
: ClusterRequestExecutor.Create(store.Urls, store.Certificate);
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetStringQueryString("url").TrimEnd('/');
var watcher = GetBoolValueQueryString("watcher", false);
var assignedCores = GetIntValueQueryString("assignedCores", false);
if (assignedCores <= 0)
{
throw new ArgumentException("Assigned cores must be greater than 0!");
}
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.ClusterCertificateHolder.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (assignedCores == null)
{
assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
}
if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
{
throw new ArgumentException("Cannot add node because the assigned cores is larger " +
$"than the available cores on that machine: {nodeInfo.NumberOfCores}");
}
ServerStore.EnsureNotPassive();
if (ServerStore.LicenseManager.CanAddNode(nodeUrl, assignedCores, out var licenseLimit) == false)
{
SetLicenseLimitResponse(licenseLimit);
return;
}
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
topologyId = clusterTopology.TopologyId;
}
if (nodeInfo.TopologyId != null && topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate));
if (certificate.NotBefore > DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
if (certificate.NotAfter < DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbrpint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
using (ctx.OpenReadTransaction())
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var readCert = ServerStore.Cluster.Read(ctx, key);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null)
{
var key = Constants.Certificates.Prefix + certificate.Thumbprint;
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
ServerStore.LicenseManager.CalculateLicenseLimits(nodeTag, assignedCores, nodeInfo, forceFetchingNodeInfo: true);
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
public async Task CanSnapshotManyCompareExchangeWithExpiration()
{
DebuggerAttachedTimeout.DisableLongTimespan = true;
var count = 1024;
using var leader = GetNewServer();
using var follower = GetNewServer();
using (var store = GetDocumentStore(new Options
{
Server = leader
}))
{
var expiry = DateTime.Now.AddMinutes(2);
var compareExchanges = new Dictionary <string, User>();
await AddCompareExchangesWithExpire(count, compareExchanges, store, expiry);
await AssertCompareExchanges(compareExchanges, store, expiry);
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(leader.WebUrl, null))
using (requestExecutor.ContextPool.AllocateOperationContext(out var ctx))
{
await requestExecutor.ExecuteAsync(new AddClusterNodeCommand(follower.WebUrl, watcher : true), ctx);
var cmd = new AddDatabaseNodeOperation(store.Database).GetCommand(store.Conventions, ctx);
await requestExecutor.ExecuteAsync(cmd, ctx);
await follower.ServerStore.Cluster.WaitForIndexNotification(cmd.Result.RaftCommandIndex);
}
using (var fStore = GetDocumentStore(new Options
{
Server = follower,
CreateDatabase = false,
ModifyDatabaseName = _ => store.Database,
ModifyDocumentStore = s => s.Conventions = new DocumentConventions {
DisableTopologyUpdates = true
}
}))
{
await AssertCompareExchanges(compareExchanges, store, expiry);
leader.ServerStore.Observer.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(3);
var val = await WaitForValueAsync(async() =>
{
var stats = await store.Maintenance.SendAsync(new GetDetailedStatisticsOperation());
return(stats.CountOfCompareExchange);
}, 0);
Assert.Equal(0, val);
val = await WaitForValueAsync(async() =>
{
var stats = await fStore.Maintenance.SendAsync(new GetDetailedStatisticsOperation());
return(stats.CountOfCompareExchange);
}, 0);
Assert.Equal(0, val);
}
}
}
public async Task AddNode()
{
var nodeUrl = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
var tag = GetStringQueryString("tag", false);
var watcher = GetBoolValueQueryString("watcher", false);
var raftRequestId = GetRaftRequestIdFromQuery();
var maxUtilizedCores = GetIntValueQueryString("maxUtilizedCores", false);
if (maxUtilizedCores != null && maxUtilizedCores <= 0)
{
throw new ArgumentException("Max utilized cores cores must be greater than 0");
}
nodeUrl = nodeUrl.Trim();
if (Uri.IsWellFormedUriString(nodeUrl, UriKind.Absolute) == false)
{
throw new InvalidOperationException($"Given node URL '{nodeUrl}' is not in a correct format.");
}
nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
tag = tag?.Trim();
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
// test connection to remote.
var result = await ServerStore.TestConnectionToRemote(nodeUrl, database : null);
if (result.Success == false)
{
throw new InvalidOperationException(result.Error);
}
// test connection from remote to destination
result = await ServerStore.TestConnectionFromRemote(requestExecutor, ctx, nodeUrl);
if (result.Success == false)
{
throw new InvalidOperationException(result.Error);
}
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (SchemaUpgrader.CurrentVersion.ServerVersion != nodeInfo.ServerSchemaVersion)
{
var nodesVersion = nodeInfo.ServerSchemaVersion == 0 ? "Pre 4.2 version" : nodeInfo.ServerSchemaVersion.ToString();
throw new InvalidOperationException($"Can't add node with mismatched storage schema version.{Environment.NewLine}" +
$"My version is {SchemaUpgrader.CurrentVersion.ServerVersion}, while node's version is {nodesVersion}");
}
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
if (ServerStore.ValidateFixedPort && nodeInfo.HasFixedPort == false)
{
throw new InvalidOperationException($"Failed to add node '{nodeUrl}' to cluster. " +
$"Node '{nodeUrl}' has port '0' in 'Configuration.Core.ServerUrls' setting. " +
"Adding a node with non fixed port is forbidden. Define a fixed port for the node to enable cluster creation.");
}
await ServerStore.EnsureNotPassiveAsync();
ServerStore.LicenseManager.AssertCanAddNode();
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
var clusterTopology = ServerStore.GetClusterTopology();
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
if (nodeInfo.ServerId == ServerStore.GetServerId())
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
}
if (nodeInfo.TopologyId != null)
{
AssertCanAddNodeWithTopologyId(clusterTopology, nodeInfo, nodeUrl);
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? tag : nodeInfo.NodeTag;
CertificateDefinition oldServerCert = null;
X509Certificate2 certificate = null;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
}
certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);
var now = DateTime.UtcNow;
if (certificate.NotBefore.ToUniversalTime() > now)
{
// Because of time zone and time drift issues, we can't assume that the certificate generation will be
// proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
// current time. Clients may still fail to work with our certificate because of timing issues,
// but the admin needs to setup time sync properly and there isn't much we can do at that point
if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
{
throw new InvalidOperationException(
$"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
}
if (certificate.NotAfter.ToUniversalTime() < now)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbprint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
// if it's the same server certificate as our own, we don't want to add it to the cluster
if (certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
{
using (ctx.OpenReadTransaction())
{
var readCert = ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint);
if (readCert != null)
{
oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
}
}
if (oldServerCert == null)
{
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint,
PublicKeyPinningHash = certificate.GetPublicKeyPinningHash(),
NotAfter = certificate.NotAfter,
Name = "Server Certificate for " + nodeUrl,
SecurityClearance = SecurityClearance.ClusterNode
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, certificateDefinition,
$"{raftRequestId}/put-new-certificate"));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
}
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : true, asWatcher : watcher ?? false);
using (ctx.OpenReadTransaction())
{
clusterTopology = ServerStore.GetClusterTopology(ctx);
possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
nodeTag = possibleNode.HasUrl ? possibleNode.NodeTag : null;
if (certificate != null && certificate.Thumbprint != Server.Certificate.Certificate.Thumbprint)
{
var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.GetCertificateByThumbprint(ctx, certificate.Thumbprint));
if (modifiedServerCert == null)
{
throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
}
if (oldServerCert == null)
{
modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
}
else
{
var value = "Node " + nodeTag;
if (modifiedServerCert.Name.Contains(value) == false)
{
modifiedServerCert.Name += ", " + value;
}
}
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(certificate.Thumbprint, modifiedServerCert, $"{raftRequestId}/put-modified-certificate"));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
var detailsPerNode = new DetailsPerNode
{
MaxUtilizedCores = maxUtilizedCores,
NumberOfCores = nodeInfo.NumberOfCores,
InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
UsableMemoryInGb = nodeInfo.UsableMemoryInGb,
BuildInfo = nodeInfo.BuildInfo,
OsInfo = nodeInfo.OsInfo
};
var maxCores = ServerStore.LicenseManager.LicenseStatus.MaxCores;
try
{
await ServerStore.PutNodeLicenseLimitsAsync(nodeTag, detailsPerNode, maxCores, $"{raftRequestId}/put-license-limits");
}
catch
{
// we'll retry this again later
}
}
NoContentStatus();
return;
}
}
RedirectToLeader();
}
protected async Task <(RavenServer Leader, List <ProcessNode> Peers, List <RavenServer> LocalPeers)> CreateMixedCluster(
string[] peers, int localPeers = 0, IDictionary <string, string> customSettings = null, X509Certificate2 certificate = null)
{
var leaderServer = GetNewServer(new ServerCreationOptions {
CustomSettings = customSettings
});
await leaderServer.ServerStore.EnsureNotPassiveAsync(leaderServer.WebUrl);
var nodeAdded = new ManualResetEvent(false);
var expectedMembers = 2;
leaderServer.ServerStore.Engine.TopologyChanged += (sender, clusterTopology) =>
{
var count = expectedMembers;
if (clusterTopology.Promotables.Count == 0 && clusterTopology.Members.Count == count)
{
var result = Interlocked.CompareExchange(ref expectedMembers, count + 1, count);
if (result == count)
{
nodeAdded.Set();
}
}
};
using (leaderServer.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(leaderServer.WebUrl, certificate))
{
var local = new List <RavenServer>();
for (int i = 0; i < localPeers; i++)
{
var peer = GetNewServer(new ServerCreationOptions {
CustomSettings = customSettings
});
var addCommand = new AddClusterNodeCommand(peer.WebUrl);
await requestExecutor.ExecuteAsync(addCommand, context);
Assert.True(nodeAdded.WaitOne(TimeSpan.FromSeconds(30)));
nodeAdded.Reset();
local.Add(peer);
}
var processes = new List <ProcessNode>();
foreach (var peer in peers)
{
processes.Add(await GetServerAsync(peer));
}
foreach (var processNode in processes)
{
var addCommand = new AddClusterNodeCommand(processNode.Url);
await requestExecutor.ExecuteAsync(addCommand, context);
Assert.True(nodeAdded.WaitOne(TimeSpan.FromSeconds(30)));
nodeAdded.Reset();
}
Assert.Equal(peers.Length + localPeers + 1, leaderServer.ServerStore.GetClusterTopology().Members.Count);
return(leaderServer, processes, local);
}
}
protected async Task WaitForExecutionOnRelevantNodes(JsonOperationContext context, string database, ClusterTopology clusterTopology, List <string> members, long index)
{
await ServerStore.Cluster.WaitForIndexNotification(index); // first let see if we commit this in the leader
if (members.Count == 0)
{
throw new InvalidOperationException("Cannot wait for execution when there are no nodes to execute ON.");
}
var executors = new List <ClusterRequestExecutor>();
try
{
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(ServerStore.ServerShutdown))
{
cts.CancelAfter(ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
var waitingTasks = new List <Task <Exception> >();
List <Exception> exceptions = null;
foreach (var member in members)
{
var url = clusterTopology.GetUrlFromTag(member);
var executor = ClusterRequestExecutor.CreateForSingleNode(url, ServerStore.Server.Certificate.Certificate);
executors.Add(executor);
waitingTasks.Add(ExecuteTask(executor, member, cts.Token));
}
while (waitingTasks.Count > 0)
{
var task = await Task.WhenAny(waitingTasks);
waitingTasks.Remove(task);
if (task.Result == null)
{
continue;
}
var exception = task.Result.ExtractSingleInnerException();
if (exceptions == null)
{
exceptions = new List <Exception>();
}
exceptions.Add(exception);
}
if (exceptions != null)
{
var allTimeouts = true;
foreach (var exception in exceptions)
{
if (exception is OperationCanceledException)
{
continue;
}
allTimeouts = false;
}
var aggregateException = new AggregateException(exceptions);
if (allTimeouts)
{
throw new TimeoutException($"Waited too long for the raft command (number {index}) to be executed on any of the relevant nodes to this command.", aggregateException);
}
throw new InvalidDataException($"The database '{database}' was created but is not accessible, because all of the nodes on which this database was supposed to reside on, threw an exception.", aggregateException);
}
}
}
finally
{
foreach (var executor in executors)
{
executor.Dispose();
}
}
async Task <Exception> ExecuteTask(RequestExecutor executor, string nodeTag, CancellationToken token)
{
try
{
await executor.ExecuteAsync(new WaitForRaftIndexCommand(index), context, token : token);
return(null);
}
catch (RavenException re) when(re.InnerException is HttpRequestException)
{
// we want to throw for self-checks
if (nodeTag == ServerStore.NodeTag)
{
return(re);
}
// ignore - we are ok when connection with a node cannot be established (test: AddDatabaseOnDisconnectedNode)
return(null);
}
catch (Exception e)
{
return(e);
}
}
}
public async Task AddNode()
{
SetupCORSHeaders();
var nodeUrl = GetStringQueryString("url").TrimEnd('/');
var watcher = GetBoolValueQueryString("watcher", false);
var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);
if (HttpContext.Request.IsHttps != remoteIsHttps)
{
throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
}
NodeInfo nodeInfo;
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.ClusterCertificateHolder.Certificate))
{
requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;
var infoCmd = new GetNodeInfoCommand();
try
{
await requestExecutor.ExecuteAsync(infoCmd, ctx);
}
catch (AllTopologyNodesDownException e)
{
throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
}
nodeInfo = infoCmd.Result;
if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
{
throw new TopologyMismatchException("You can't add new node to an already existing cluster");
}
}
ServerStore.EnsureNotPassive();
if (ServerStore.IsLeader())
{
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
string topologyId;
using (ctx.OpenReadTransaction())
{
var clusterTopology = ServerStore.GetClusterTopology(ctx);
var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
if (possibleNode.HasUrl)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
}
topologyId = clusterTopology.TopologyId;
}
if (nodeInfo.TopologyId != null && topologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
}
var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag
? null : nodeInfo.NodeTag;
if (remoteIsHttps)
{
if (nodeInfo.Certificate == null)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because it has no certificate while trying to use HTTPS");
}
var certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate));
if (certificate.NotBefore > DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
}
if (certificate.NotAfter < DateTime.UtcNow)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
}
var expected = GetStringQueryString("expectedThumbrpint", required: false);
if (expected != null)
{
if (certificate.Thumbprint != expected)
{
throw new InvalidOperationException($"Cannot add node {nodeTag} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
}
}
var certificateDefinition = new CertificateDefinition
{
Certificate = nodeInfo.Certificate,
Thumbprint = certificate.Thumbprint
};
var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));
await ServerStore.Cluster.WaitForIndexNotification(res.Index);
}
await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);
NoContentStatus();
return;
}
}
RedirectToLeader();
}
private static ClusterRequestExecutor CreateRequestExecutor(DocumentStoreBase store)
{
return(store.Conventions.DisableTopologyUpdates
? ClusterRequestExecutor.CreateForSingleNode(store.Urls[0], store.Certificate, store.Conventions)
: ClusterRequestExecutor.Create(store.Urls, store.Certificate, store.Conventions));
}
public async Task CanSnapshotCompareExchangeWithExpiration()
{
var count = 45;
var(_, leader) = await CreateRaftCluster(1, watcherCluster : true);
using (var store = GetDocumentStore(options: new Options
{
Server = leader
}))
{
var expiry = DateTime.Now.AddMinutes(2);
var compareExchanges = new Dictionary <string, User>();
await CompareExchangeExpirationTest.AddCompareExchangesWithExpire(count, compareExchanges, store, expiry);
await CompareExchangeExpirationTest.AssertCompareExchanges(compareExchanges, store, expiry);
using (leader.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (context.OpenReadTransaction())
{
Assert.Equal(count, CompareExchangeExpirationStorage.GetExpiredValues(context, long.MaxValue).Count());
}
var server2 = GetNewServer();
var server2Url = server2.ServerStore.GetNodeHttpServerUrl();
Servers.Add(server2);
using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(leader.WebUrl, null))
using (requestExecutor.ContextPool.AllocateOperationContext(out var ctx))
{
await requestExecutor.ExecuteAsync(new AddClusterNodeCommand(server2Url, watcher : true), ctx);
var addDatabaseNode = new AddDatabaseNodeOperation(store.Database);
await store.Maintenance.Server.SendAsync(addDatabaseNode);
}
using (server2.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (context.OpenReadTransaction())
{
Assert.Equal(count, CompareExchangeExpirationStorage.GetExpiredValues(context, long.MaxValue).Count());
}
var now = DateTime.UtcNow;
leader.ServerStore.Observer.Time.UtcDateTime = () => now.AddMinutes(3);
var leaderCount = WaitForValue(() =>
{
using (leader.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (context.OpenReadTransaction())
{
return(CompareExchangeExpirationStorage.GetExpiredValues(context, long.MaxValue).Count());
}
}, 0, interval: 333);
Assert.Equal(0, leaderCount);
var server2count = WaitForValue(() =>
{
using (server2.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (context.OpenReadTransaction())
{
return(CompareExchangeExpirationStorage.GetExpiredValues(context, long.MaxValue).Count());
}
}, 0, interval: 333);
Assert.Equal(0, server2count);
}
}
