private void CanRedefineFailoverBehaviorFromServerSide(FailoverBehavior storeBehavior, FailoverBehavior serverBehavior)
{
var clusterStores = CreateRaftCluster(1, "replication");
using (var store = clusterStores[0])
{
using (var session = store.OpenSession())
{
session.Store(new ReplicationDocument
{
ClientConfiguration = new ReplicationClientConfiguration
{
FailoverBehavior = serverBehavior
}
}, "Raven/Replication/Destinations");
session.SaveChanges();
}
using (var secondStore = new DocumentStore
{
Url = store.Url,
Conventions =
{
FailoverBehavior = storeBehavior
},
DefaultDatabase = store.DefaultDatabase
}.Initialize())
{
Assert.True(SpinWait.SpinUntil(() => secondStore.Conventions.FailoverBehavior == serverBehavior, TimeSpan.FromSeconds(30)));
}
}
}
private void CanRedefineFailoverBehaviorFromServerSide(FailoverBehavior storeBehavior, FailoverBehavior serverBehavior)
{
var clusterStores = CreateRaftCluster(1, "replication");
using (var store = clusterStores[0])
{
using (var session = store.OpenSession())
{
session.Store(new ReplicationDocument
{
ClientConfiguration = new ReplicationClientConfiguration
{
FailoverBehavior = serverBehavior
}
}, "Raven/Replication/Destinations");
session.SaveChanges();
}
using (var secondStore = new DocumentStore
{
Url = store.Url,
Conventions =
{
FailoverBehavior = storeBehavior
},
DefaultDatabase = store.DefaultDatabase
}.Initialize())
{
var client = ((AsyncServerClient)secondStore.AsyncDatabaseCommands);
AsyncHelpers.RunSync(() => client.RequestExecuter.UpdateReplicationInformationIfNeededAsync(client, force: true));
Assert.True(SpinWait.SpinUntil(() => secondStore.Conventions.FailoverBehavior == serverBehavior, TimeSpan.FromSeconds(30)));
}
}
}
private void CanReadFromMultipleServersInternal(int numberOfNodes, FailoverBehavior failoverBehavior)
{
var clusterStores = CreateRaftCluster(numberOfNodes, activeBundles: "Replication", configureStore: store => store.Conventions.FailoverBehavior = failoverBehavior);
SetupClusterConfiguration(clusterStores);
Enumerable.Range(0, numberOfNodes).ForEach(i => clusterStores[0].DatabaseCommands.Put($"keys/{i}", null, new RavenJObject(), new RavenJObject()));
using (ForceNonClusterRequests(clusterStores))
{
clusterStores.ForEach(store => WaitForDocument(store.DatabaseCommands, $"keys/{numberOfNodes - 1}"));
}
//Here i want to fetch the topology after a leader was elected so all stores will have latest topology.
clusterStores.ForEach(store =>
{
var client = ((AsyncServerClient)store.AsyncDatabaseCommands);
AsyncHelpers.RunSync(() => client.RequestExecuter.UpdateReplicationInformationIfNeededAsync(client, force: true));
});
var tasks = new List <ReplicationTask>();
foreach (var server in servers)
{
server.Options.DatabaseLandlord.ForAllDatabases(database => tasks.Add(database.StartupTasks.OfType <ReplicationTask>().First()));
server.Options.ClusterManager.Value.Engine.Dispose();
}
foreach (var task in tasks)
{
task.Pause();
SpinWait.SpinUntil(() => task.IsRunning == false, TimeSpan.FromSeconds(3));
}
servers.ForEach(server => server.Options.RequestManager.ResetNumberOfRequests());
using (ForceNonClusterRequests(clusterStores))
{
for (int i = 0; i < clusterStores.Count; i++)
{
var store = clusterStores[i];
Enumerable.Range(0, numberOfNodes).ForEach(j => store.DatabaseCommands.Get($"keys/{j}"));
}
}
servers.ForEach(server =>
{
Assert.True(server.Options.RequestManager.NumberOfRequests >= numberOfNodes);
});
servers.Clear();
}
private void WithFailoversInternal(int numberOfNodes, FailoverBehavior failoverBehavior)
{
using (WithCustomDatabaseSettings(doc => doc.Settings["Raven/Replication/ReplicationRequestTimeout"] = "4000"))
{
var clusterStores = CreateRaftCluster(numberOfNodes, activeBundles: "Replication", configureStore: s => s.Conventions.FailoverBehavior = failoverBehavior);
foreach (var documentStore in clusterStores)
{
// set lower timeout to reduce test time
documentStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(15);
}
SetupClusterConfiguration(clusterStores);
clusterStores.ForEach(store =>
AsyncHelpers.RunSync(() => ((ServerClient)store.DatabaseCommands).RequestExecuter.UpdateReplicationInformationIfNeededAsync((AsyncServerClient)store.AsyncDatabaseCommands, force: true)));
for (int i = 0; i < clusterStores.Count; i++)
{
var store = clusterStores[i];
store.DatabaseCommands.Put("keys/" + i, null, new RavenJObject(), new RavenJObject());
}
using (ForceNonClusterRequests(clusterStores))
{
for (int i = 0; i < clusterStores.Count; i++)
{
clusterStores.ForEach(store => WaitForDocument(store.DatabaseCommands, "keys/" + i));
}
}
var oldLeader = servers.First(x => x.Options.ClusterManager.Value.IsLeader());
oldLeader.Dispose();
for (int i = 0; i < clusterStores.Count; i++)
{
var store = clusterStores[i];
store.DatabaseCommands.Put("keys/" + (i + clusterStores.Count), null, new RavenJObject(), new RavenJObject());
}
for (int i = 0; i < clusterStores.Count; i++)
{
clusterStores.ForEach(store => WaitForDocument(store.DatabaseCommands, "keys/" + (i + clusterStores.Count)));
}
}
}
public HttpJsonRequest AddReplicationStatusHeaders(string thePrimaryUrl, string currentUrl, IDocumentStoreReplicationInformer replicationInformer, FailoverBehavior failoverBehavior, Action <NameValueCollection, string, string> handleReplicationStatusChanges)
{
if (thePrimaryUrl.Equals(currentUrl, StringComparison.OrdinalIgnoreCase))
{
return(this);
}
if (replicationInformer.GetFailureCount(thePrimaryUrl) <= 0)
{
return(this); // not because of failover, no need to do this.
}
var lastPrimaryCheck = replicationInformer.GetFailureLastCheck(thePrimaryUrl);
headers.Set(Constants.RavenClientPrimaryServerUrl, ToRemoteUrl(thePrimaryUrl));
headers.Set(Constants.RavenClientPrimaryServerLastCheck, lastPrimaryCheck.ToString("s"));
primaryUrl = thePrimaryUrl;
operationUrl = currentUrl;
HandleReplicationStatusChanges = handleReplicationStatusChanges;
return(this);
}
public HttpJsonRequest AddReplicationStatusHeaders(string thePrimaryUrl, string currentUrl, ReplicationInformer replicationInformer, FailoverBehavior failoverBehavior, Action<NameValueCollection, string, string> handleReplicationStatusChanges)
{
if (thePrimaryUrl.Equals(currentUrl, StringComparison.InvariantCultureIgnoreCase))
return this;
if (replicationInformer.GetFailureCount(thePrimaryUrl) <= 0)
return this; // not because of failover, no need to do this.
var lastPrimaryCheck = replicationInformer.GetFailureLastCheck(thePrimaryUrl);
webRequest.Headers.Add(Constants.RavenClientPrimaryServerUrl, ToRemoteUrl(thePrimaryUrl));
webRequest.Headers.Add(Constants.RavenClientPrimaryServerLastCheck, lastPrimaryCheck.ToString("s"));
primaryUrl = thePrimaryUrl;
operationUrl = currentUrl;
HandleReplicationStatusChanges = handleReplicationStatusChanges;
return this;
}
