public void WaitForTopologyStabilization(string s, RavenServer workingServer, int rehabCount, int memberCount)
{
using (var tempStore = new DocumentStore
{
Database = s,
Urls = new[] { workingServer.WebUrl },
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
{
Topology topo;
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var value = WaitForValue(() =>
{
var topologyGetCommand = new GetDatabaseTopologyCommand();
tempStore.GetRequestExecutor().Execute(topologyGetCommand, context);
topo = topologyGetCommand.Result;
int rehab = 0;
int members = 0;
topo.Nodes.ForEach(n =>
{
switch (n.ServerRole)
{
case ServerNode.Role.Rehab:
rehab++;
break;
case ServerNode.Role.Member:
members++;
break;
}
});
return(new Tuple <int, int>(rehab, members));
}, new Tuple <int, int>(rehabCount, memberCount));
}
}
}
public async Task Round_robin_load_balancing_with_failing_node_should_work()
{
var databaseName = GetDatabaseName();
var(nodes, leader) = await CreateRaftCluster(3);
var followers = Servers.Where(x => x.ServerStore.IsLeader() == false).ToArray();
var conventionsForLoadBalancing = new DocumentConventions
{
ReadBalanceBehavior = ReadBalanceBehavior.RoundRobin
};
using (var leaderStore = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower1 = new DocumentStore
{
Urls = new[] { followers[0].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower2 = new DocumentStore
{
Urls = new[] { followers[1].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var context = JsonOperationContext.ShortTermSingleUse())
{
leaderStore.Initialize();
follower1.Initialize();
follower2.Initialize();
var(index, _) = await CreateDatabaseInCluster(databaseName, 3, leader.WebUrl);
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(30));
var leaderRequestExecutor = leaderStore.GetRequestExecutor();
//wait until all nodes in database cluster are members (and not promotables)
//GetDatabaseTopologyCommand -> does not retrieve promotables
var topology = new Topology();
while (topology.Nodes?.Count != 3)
{
var topologyGetCommand = new GetDatabaseTopologyCommand();
await leaderRequestExecutor.ExecuteAsync(topologyGetCommand, context);
topology = topologyGetCommand.Result;
Thread.Sleep(50);
}
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(index);
}
using (var session = leaderStore.OpenSession())
{
session.Store(new User {
Name = "John Dow"
});
session.Store(new User {
Name = "Jack Dow"
});
session.Store(new User {
Name = "Jane Dow"
});
session.Store(new User {
Name = "FooBar"
}, "marker");
session.SaveChanges();
await WaitForDocumentInClusterAsync <User>(nodes,
databaseName,
"marker",
x => true,
leader.ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
}
using (var requestExecutor = RequestExecutor.Create(follower1.Urls, databaseName, null, follower1.Conventions))
{
do //make sure there are three nodes in the topology
{
await Task.Delay(100);
} while (requestExecutor.TopologyNodes == null);
DisposeServerAndWaitForFinishOfDisposal(leader);
var failedRequests = new HashSet <(string, Exception)>();
#pragma warning disable CS0618 // Type or member is obsolete
requestExecutor.FailedRequest += (url, e) => failedRequests.Add((url, e));
#pragma warning restore CS0618 // Type or member is obsolete
using (var tmpContext = JsonOperationContext.ShortTermSingleUse())
{
for (var sessionId = 0; sessionId < 5; sessionId++)
{
requestExecutor.Cache.Clear(); //make sure we do not use request cache
await requestExecutor.ExecuteAsync(new GetStatisticsOperation().GetCommand(DocumentConventions.Default, tmpContext), tmpContext, new SessionInfo(sessionId, false));
}
}
}
}
}
public async Task Fastst_node_should_choose_the_node_without_delay()
{
NoTimeouts();
var databaseName = GetDatabaseName();
var(leader, serversToProxies) = await CreateRaftClusterWithProxiesAsync(3);
var followers = Servers.Where(x => x.ServerStore.IsLeader() == false).ToArray();
var conventionsForLoadBalancing = new DocumentConventions
{
ReadBalanceBehavior = ReadBalanceBehavior.FastestNode
};
//set proxies with delays to all servers except follower2
using (var leaderStore = new DocumentStore
{
Urls = new[] { ReplacePort(leader.WebUrl, serversToProxies[leader].Port) },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
{
leaderStore.Initialize();
var(index, _) = await CreateDatabaseInCluster(databaseName, 3, leader.WebUrl);
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(30));
var leaderRequestExecutor = leaderStore.GetRequestExecutor();
//make sure we have updated topology --> more deterministic test
await leaderRequestExecutor.UpdateTopologyAsync(new RequestExecutor.UpdateTopologyParameters(new ServerNode
{
ClusterTag = leader.ServerStore.NodeTag,
Database = databaseName,
Url = leader.WebUrl
})
{
TimeoutInMs = 5000
});
ApplyProxiesOnRequestExecutor(serversToProxies, leaderRequestExecutor);
//wait until all nodes in database cluster are members (and not promotables)
//GetDatabaseTopologyCommand -> does not retrieve promotables
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var topology = new Topology();
while (topology.Nodes?.Count != 3)
{
var topologyGetCommand = new GetDatabaseTopologyCommand();
await leaderRequestExecutor.ExecuteAsync(topologyGetCommand, context).ConfigureAwait(false);
topology = topologyGetCommand.Result;
Thread.Sleep(50);
}
}
//set delays to all servers except follower2
foreach (var server in Servers)
{
if (server == followers[1])
{
continue;
}
serversToProxies[server].ConnectionDelay = 300;
}
using (var session = leaderStore.OpenSession())
{
session.Store(new User {
Name = "John Dow"
}, "users/1");
session.SaveChanges();
}
while (leaderRequestExecutor.InSpeedTestPhase)
{
using (var session = leaderStore.OpenSession())
{
session.Load <User>("users/1");
}
}
var fastest = leaderRequestExecutor.GetFastestNode().Result.Node;
var follower2Proxy = ReplacePort(followers[1].WebUrl, serversToProxies[followers[1]].Port);
Assert.Equal(follower2Proxy, fastest.Url);
}
}
public async Task Round_robin_load_balancing_should_work()
{
var databaseName = GetDatabaseName();
var(nodes, leader) = await CreateRaftCluster(3);
var followers = Servers.Where(x => x.ServerStore.IsLeader() == false).ToArray();
var conventionsForLoadBalancing = new DocumentConventions
{
ReadBalanceBehavior = ReadBalanceBehavior.RoundRobin
};
using (var leaderStore = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower1 = new DocumentStore
{
Urls = new[] { followers[0].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower2 = new DocumentStore
{
Urls = new[] { followers[1].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var context = JsonOperationContext.ShortTermSingleUse())
{
leaderStore.Initialize();
follower1.Initialize();
follower2.Initialize();
var(index, _) = await CreateDatabaseInCluster(databaseName, 3, leader.WebUrl);
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(30));
var leaderRequestExecutor = leaderStore.GetRequestExecutor();
//make sure we have updated topology --> more deterministic test
await leaderRequestExecutor.UpdateTopologyAsync(new RequestExecutor.UpdateTopologyParameters(new ServerNode
{
ClusterTag = leader.ServerStore.NodeTag,
Database = databaseName,
Url = leader.WebUrl
})
{
TimeoutInMs = 5000,
ForceUpdate = true
});
//wait until all nodes in database cluster are members (and not promotables)
//GetDatabaseTopologyCommand -> does not retrieve promotables
var topology = new Topology();
while (topology.Nodes?.Count != 3)
{
var topologyGetCommand = new GetDatabaseTopologyCommand();
await leaderRequestExecutor.ExecuteAsync(topologyGetCommand, context);
topology = topologyGetCommand.Result;
Thread.Sleep(50);
}
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(index);
}
using (var session = leaderStore.OpenSession())
{
session.Store(new User {
Name = "John Dow"
});
session.Store(new User {
Name = "Jack Dow"
});
session.Store(new User {
Name = "Jane Dow"
});
session.Store(new User {
Name = "FooBar"
}, "marker");
session.SaveChanges();
await WaitForDocumentInClusterAsync <User>(nodes, databaseName, "marker", x => true, leader.ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
}
var usedUrls = new List <string>();
for (var i = 0; i < 3; i++)
{
using (var session = leaderStore.OpenSession())
{
// ReSharper disable once ReturnValueOfPureMethodIsNotUsed
session.Query <User>().Where(u => u.Name.StartsWith("Ja")).ToList();
usedUrls.Add((await session.Advanced.GetCurrentSessionNode()).Url.ToLower());
}
}
foreach (var url in usedUrls)
{
Assert.Single(usedUrls, url);
}
}
}