public async Task <HttpResponseMessage> JoinToCluster()
{
var nodeConnectionInfo = await ReadJsonObjectAsync <NodeConnectionInfo>().ConfigureAwait(false);
if (nodeConnectionInfo == null)
{
return(GetEmptyMessage(HttpStatusCode.BadRequest));
}
if (nodeConnectionInfo.Name == null)
{
nodeConnectionInfo.Name = RaftHelper.GetNodeName(await ClusterManager.Client.GetDatabaseId(nodeConnectionInfo).ConfigureAwait(false));
}
bool forced;
bool.TryParse(GetQueryStringValue("force"), out forced);
var topology = ClusterManager.Engine.CurrentTopology;
CanJoinResult canJoinResult = CanJoinResult.CanJoin;
if (forced == false)
{
canJoinResult = await ClusterManager.Client.SendCanJoinAsync(nodeConnectionInfo).ConfigureAwait(false);
switch (canJoinResult)
{
case CanJoinResult.IsNonEmpty:
return(GetMessageWithString("Can't join node to cluster. Node is not empty", HttpStatusCode.BadRequest));
case CanJoinResult.InAnotherCluster:
return(GetMessageWithString("Can't join node to cluster. Node is in different cluster", HttpStatusCode.BadRequest));
case CanJoinResult.AlreadyJoined:
return(GetEmptyMessage(HttpStatusCode.NotModified));
}
}
else
{
await ClusterManager.Client.SendInitializeNewClusterForAsync(nodeConnectionInfo, topology.TopologyId).ConfigureAwait(false);
}
if (topology.Contains(nodeConnectionInfo.Name))
{
return(GetEmptyMessage(HttpStatusCode.NotModified));
}
//overriding user request since we know that this node can only join as non voter
if (canJoinResult == CanJoinResult.CanJoinAsNonVoter)
{
nodeConnectionInfo.IsNoneVoter = true;
}
await ClusterManager.Client.SendJoinServerAsync(nodeConnectionInfo).ConfigureAwait(false);
return(GetEmptyMessage());
}
public List <DocumentStore> ExtendRaftCluster(int numberOfExtraNodes, Guid topologyId, string activeBundles = null, Action <DocumentStore> configureStore = null, [CallerMemberName] string databaseName = null, bool inMemory = true)
{
if (configureStore == null)
{
configureStore = defaultConfigureStore;
}
var leader = ChooseTheRealLeader(topologyId);
Assert.NotNull(leader);
var nodes = Enumerable.Range(0, numberOfExtraNodes)
.Select(x => GetNewServer(GetPort(), activeBundles: activeBundles, databaseName: databaseName, runInMemory: inMemory,
configureConfig: configuration =>
{
configuration.Cluster.ElectionTimeout *= 10;
configuration.Cluster.HeartbeatTimeout *= 10;
}))
.ToList();
var allNodesFinishedJoining = new ManualResetEventSlim();
leader.Options.ClusterManager.Value.Engine.TopologyChanged += command =>
{
if (command.Requested.AllNodeNames.All(command.Requested.IsVoter))
{
allNodesFinishedJoining.Set();
}
};
for (var i = 0; i < numberOfExtraNodes; i++)
{
var n = nodes[i];
if (n == leader)
{
continue;
}
Assert.True(leader.Options.ClusterManager.Value.Engine.AddToClusterAsync(new NodeConnectionInfo
{
Name = RaftHelper.GetNodeName(n.SystemDatabase.TransactionalStorage.Id),
Uri = RaftHelper.GetNodeUrl(n.SystemDatabase.Configuration.ServerUrl)
}).Wait(10000));
Assert.True(allNodesFinishedJoining.Wait(10000), "Not all nodes finished joining");
allNodesFinishedJoining.Reset();
}
return(nodes
.Select(node => NewRemoteDocumentStore(ravenDbServer: node, activeBundles: activeBundles, configureStore: configureStore, databaseName: databaseName))
.ToList());
}
public static NodeConnectionInfo CreateSelfConnection(DocumentDatabase database)
{
var configuration = database.Configuration;
var nodeName = RaftHelper.GetNodeName(database.TransactionalStorage.Id);
var url = configuration.ServerUrl;
return(new NodeConnectionInfo
{
Name = nodeName,
Uri = RaftHelper.GetNodeUrl(url)
});
}
public async Task <HttpResponseMessage> Leave([FromUri] Guid name)
{
var nodeName = RaftHelper.GetNodeName(name);
if (ClusterManager.Engine.CurrentTopology.Contains(nodeName) == false)
{
return(GetEmptyMessage(HttpStatusCode.NotModified));
}
var node = ClusterManager.Engine.CurrentTopology.GetNodeByName(nodeName);
await ClusterManager.Client.SendLeaveAsync(node).ConfigureAwait(false);
return(GetMessageWithObject(new
{
Removed = name
}));
}
public void CanInstallSnapshot()
{
CreateRaftCluster(3, inMemory: false); // 3 nodes
for (var i = 0; i < 5; i++)
{
var client = servers[0].Options.ClusterManager.Value.Client;
client.SendClusterConfigurationAsync(new ClusterConfiguration {
EnableReplication = false
}).Wait(3000);
}
WaitForClusterToBecomeNonStale(3);
var leader = servers.FirstOrDefault(server => server.Options.ClusterManager.Value.IsLeader());
Assert.NotNull(leader);
var newServer = GetNewServer(GetPort(), runInMemory: false);
var snapshotInstalledMre = new ManualResetEventSlim();
newServer.Options.ClusterManager.Value.Engine.SnapshotInstalled += () => snapshotInstalledMre.Set();
var allNodesFinishedJoining = new ManualResetEventSlim();
leader.Options.ClusterManager.Value.Engine.TopologyChanged += command =>
{
if (command.Requested.AllNodeNames.All(command.Requested.IsVoter))
{
allNodesFinishedJoining.Set();
}
};
Assert.True(leader.Options.ClusterManager.Value.Engine.AddToClusterAsync(new NodeConnectionInfo
{
Name = RaftHelper.GetNodeName(newServer.SystemDatabase.TransactionalStorage.Id),
Uri = RaftHelper.GetNodeUrl(newServer.SystemDatabase.Configuration.ServerUrl)
}).Wait(10000));
Assert.True(allNodesFinishedJoining.Wait(10000));
Assert.True(snapshotInstalledMre.Wait(TimeSpan.FromSeconds(5)));
}
private void HandleTopologyChanges(TopologyChangeCommand command)
{
if (RaftHelper.HasDifferentNodes(command) == false)
{
return;
}
if (command.Previous == null)
{
HandleClusterConfigurationChanges();
return;
}
var removedNodeUrls = command
.Previous
.AllNodes.Select(x => x.Uri.AbsoluteUri)
.Except(command.Requested.AllNodes.Select(x => x.Uri.AbsoluteUri))
.ToList();
HandleClusterConfigurationChanges(removedNodeUrls);
}
public List <DocumentStore> CreateRaftCluster(int numberOfNodes, string activeBundles = null, Action <DocumentStore> configureStore = null, [CallerMemberName] string databaseName = null, bool inMemory = true, bool fiddler = false)
{
if (configureStore == null)
{
configureStore = defaultConfigureStore;
}
var nodes = Enumerable.Range(0, numberOfNodes)
.Select(x => GetNewServer(GetPort(), activeBundles: activeBundles, databaseName: databaseName, runInMemory: inMemory,
configureConfig: configuration =>
{
configuration.Cluster.ElectionTimeout *= 10;
configuration.Cluster.HeartbeatTimeout *= 10;
}))
.ToList();
var allNodesFinishedJoining = new ManualResetEventSlim();
var random = new Random();
var leader = nodes[random.Next(0, numberOfNodes - 1)];
leader.Options.ClusterManager.Value.InitializeTopology(forceCandidateState: true);
Assert.True(leader.Options.ClusterManager.Value.Engine.WaitForLeader(), "Leader was not elected by himself in time");
leader.Options.ClusterManager.Value.Engine.TopologyChanged += command =>
{
if (command.Requested.AllNodeNames.All(command.Requested.IsVoter))
{
allNodesFinishedJoining.Set();
}
};
for (var i = 0; i < numberOfNodes; i++)
{
var n = nodes[i];
if (n == leader)
{
continue;
}
Assert.True(leader.Options.ClusterManager.Value.Engine.AddToClusterAsync(new NodeConnectionInfo
{
Name = RaftHelper.GetNodeName(n.SystemDatabase.TransactionalStorage.Id),
Uri = RaftHelper.GetNodeUrl(n.SystemDatabase.Configuration.ServerUrl)
}).Wait(3000), "Failed to add node to cluster");
}
if (numberOfNodes == 1)
{
allNodesFinishedJoining.Set();
}
Assert.True(allNodesFinishedJoining.Wait(10000 * numberOfNodes), "Not all nodes become voters. " + leader.Options.ClusterManager.Value.Engine.CurrentTopology);
Assert.True(leader.Options.ClusterManager.Value.Engine.WaitForLeader(), "Wait for leader timedout");
WaitForClusterToBecomeNonStale(nodes);
foreach (var node in nodes)
{
var url = node.SystemDatabase.ServerUrl.ForDatabase(databaseName);
var serverHash = ServerHash.GetServerHash(url);
ReplicationInformerLocalCache.ClearClusterNodesInformationLocalCache(serverHash);
ReplicationInformerLocalCache.ClearReplicationInformationFromLocalCache(serverHash);
}
var documentStores = nodes
.Select(node => NewRemoteDocumentStore(ravenDbServer: node, fiddler: fiddler, activeBundles: activeBundles, configureStore: configureStore, databaseName: databaseName))
.ToList();
foreach (var documentStore in documentStores)
{
((ClusterAwareRequestExecuter)((ServerClient)documentStore.DatabaseCommands).RequestExecuter).WaitForLeaderTimeout = TimeSpan.FromSeconds(30);
}
return(documentStores);
}
public HttpResponseMessage TopologyGet()
{
if (Database == null)
{
return(GetEmptyMessage(HttpStatusCode.NotFound));
}
var configurationDocument = Database.ConfigurationRetriever.GetConfigurationDocument <ReplicationDocument <ReplicationDestination.ReplicationDestinationWithConfigurationOrigin> >(Constants.RavenReplicationDestinations);
if (configurationDocument == null)
{
return(GetEmptyMessage(HttpStatusCode.NotFound));
}
var mergedDocument = configurationDocument.MergedDocument;
var isInCluster = ClusterManager.IsActive() && Database.IsClusterDatabase();
var commitIndex = isInCluster ? ClusterManager.Engine.CommitIndex : -1;
var term = isInCluster ? ClusterManager.Engine.PersistentState.CurrentTerm : -1;
var currentTopology = isInCluster ? ClusterManager.Engine.CurrentTopology : null;
var currentLeader = ClusterManager.Engine.CurrentLeader;
var isLeader = currentLeader == ClusterManager.Engine.Options.SelfConnection.Name;
var configurationDocumentWithClusterInformation = new ReplicationDocumentWithClusterInformation
{
ClientConfiguration = mergedDocument.ClientConfiguration,
Id = mergedDocument.Id,
Source = mergedDocument.Source,
ClusterCommitIndex = commitIndex,
Term = term
};
if (isInCluster)
{
configurationDocumentWithClusterInformation.ClusterInformation = new ClusterInformation(true, isLeader);
}
foreach (var destination in mergedDocument.Destinations)
{
var destinationIsLeader = isInCluster && isLeader == false && currentTopology != null && currentLeader != null;
if (destinationIsLeader)
{
var destinationUrl = RaftHelper.GetNormalizedNodeUrl(destination.Url);
var node = currentTopology.AllVotingNodes.FirstOrDefault(x => x.Uri.AbsoluteUri.ToLowerInvariant() == destinationUrl);
if (node != null)
{
destinationIsLeader = node.Name == currentLeader;
}
else
{
destinationIsLeader = false;
}
}
configurationDocumentWithClusterInformation
.Destinations
.Add(ReplicationDestination.ReplicationDestinationWithClusterInformation.Create(destination, isInCluster, destinationIsLeader));
}
return(GetMessageWithObject(configurationDocumentWithClusterInformation));
}
private void HandleClusterReplicationChanges(List <string> removedNodes, bool enableReplication)
{
var currentTopology = clusterManager.Engine.CurrentTopology;
var replicationDocumentJson = systemDatabase.Documents.Get(Constants.Global.ReplicationDestinationsDocumentName);
var replicationDocument = replicationDocumentJson != null
? replicationDocumentJson.DataAsJson.JsonDeserialization <ReplicationDocument>()
: new ReplicationDocument();
var replicationDocumentNormalizedDestinations = replicationDocument
.Destinations
.ToDictionary(x => RaftHelper.GetNormalizedNodeUrl(x.Url), x => x);
var currentTopologyNormalizedDestionations = currentTopology
.AllNodes
.ToDictionary(x => x.Uri.AbsoluteUri.ToLowerInvariant(), x => x);
var urls = replicationDocumentNormalizedDestinations
.Keys
.Union(currentTopologyNormalizedDestionations.Keys)
.ToList();
foreach (var url in urls)
{
ReplicationDestination destination;
replicationDocumentNormalizedDestinations.TryGetValue(url, out destination);
NodeConnectionInfo node;
currentTopologyNormalizedDestionations.TryGetValue(url, out node);
if (destination == null && node == null)
{
continue; // not possible, but...
}
if (destination != null && node == null)
{
if (removedNodes.Contains(url, StringComparer.OrdinalIgnoreCase) == false)
{
continue; // external destination
}
replicationDocument.Destinations.Remove(destination);
continue;
}
if (string.Equals(node.Name, clusterManager.Engine.Options.SelfConnection.Name, StringComparison.OrdinalIgnoreCase))
{
continue; // skipping self
}
if (destination == null)
{
destination = new ReplicationDestination();
replicationDocument.Destinations.Add(destination);
}
destination.ApiKey = node.ApiKey;
destination.Database = null;
destination.Disabled = enableReplication == false;
destination.Domain = node.Domain;
destination.Password = node.Password;
destination.TransitiveReplicationBehavior = TransitiveReplicationOptions.Replicate;
destination.SkipIndexReplication = false;
destination.Url = node.Uri.AbsoluteUri;
destination.Username = node.Username;
}
systemDatabase.Documents.Put(Constants.Global.ReplicationDestinationsDocumentName, null, RavenJObject.FromObject(replicationDocument), new RavenJObject(), null);
}
