private async Task TestIfNodeAlive(ServerNode node)
{
var command = new GetTopologyCommand();
string url;
var request = CreateRequest(node, command, out url);
var sp = Stopwatch.StartNew();
try
{
var response = await _httpClient.SendAsync(request).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
node.IsFailed = false;
}
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Tested if node alive but it's down: {url}", e);
}
}
finally
{
sp.Stop();
node.UpdateRequestTime(sp.ElapsedMilliseconds);
}
}
private async Task UpdateTopology()
{
JsonOperationContext context;
using (ContextPool.AllocateOperationContext(out context))
{
var node = _topology.LeaderNode;
var serverHash = ServerHash.GetServerHash(node.Url, node.Database);
if (_firstTimeTryLoadFromTopologyCache)
{
_firstTimeTryLoadFromTopologyCache = false;
var cachedTopology = TopologyLocalCache.TryLoadTopologyFromLocalCache(serverHash, context);
if (cachedTopology != null && cachedTopology.Etag > 0)
{
_topology = cachedTopology;
// we have cached topology, but we need to verify it is up to date, we'll check in
// 1 second, and let the rest of the system start
_updateTopologyTimer.Change(TimeSpan.FromSeconds(1), Timeout.InfiniteTimeSpan);
return;
}
}
var command = new GetTopologyCommand();
try
{
await ExecuteAsync(new ChoosenNode { Node = node }, context, command);
if (_topology.Etag != command.Result.Etag)
{
_topology = command.Result;
TopologyLocalCache.TrySavingTopologyToLocalCache(serverHash, _topology, context);
}
}
catch (Exception ex)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failed to update topology", ex);
}
}
finally
{
_updateTopologyTimer.Change(TimeSpan.FromMinutes(5), Timeout.InfiniteTimeSpan);
}
}
}
public async Task Fastst_node_should_choose_the_node_without_delay()
{
NoTimeouts();
var databaseName = "Fastst_node_should_choose_the_node_without_delay" + Guid.NewGuid();
var(leader, serversToProxies) = await CreateRaftClusterWithProxiesAndGetLeader(3);
var followers = Servers.Where(x => x.ServerStore.IsLeader() == false).ToArray();
var conventionsForLoadBalancing = new DocumentConventions
{
ReadBalanceBehavior = ReadBalanceBehavior.FastestNode
};
//set proxies with delays to all servers except follower2
using (var leaderStore = new DocumentStore
{
Urls = new[] { ReplacePort(leader.WebUrl, serversToProxies[leader].Port) },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
{
leaderStore.Initialize();
var(index, _) = await CreateDatabaseInCluster(databaseName, 3, leader.WebUrl);
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(30));
var leaderRequestExecutor = leaderStore.GetRequestExecutor();
//make sure we have updated topology --> more deterministic test
await leaderRequestExecutor.UpdateTopologyAsync(new ServerNode
{
ClusterTag = leader.ServerStore.NodeTag,
Database = databaseName,
Url = leader.WebUrl
}, 5000);
ApplyProxiesOnRequestExecutor(serversToProxies, leaderRequestExecutor);
//wait until all nodes in database cluster are members (and not promotables)
//GetTopologyCommand -> does not retrieve promotables
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var topology = new Topology();
while (topology.Nodes?.Count != 3)
{
var topologyGetCommand = new GetTopologyCommand();
await leaderRequestExecutor.ExecuteAsync(topologyGetCommand, context).ConfigureAwait(false);
topology = topologyGetCommand.Result;
Thread.Sleep(50);
}
}
//set delays to all servers except follower2
foreach (var server in Servers)
{
if (server == followers[1])
{
continue;
}
serversToProxies[server].ConnectionDelay = 300;
}
using (var session = leaderStore.OpenSession())
{
session.Store(new User {
Name = "John Dow"
}, "users/1");
session.SaveChanges();
}
while (leaderRequestExecutor.InSpeedTestPhase)
{
using (var session = leaderStore.OpenSession())
{
session.Load <User>("users/1");
}
}
var fastest = leaderRequestExecutor.GetFastestNode().Result.Node;
var follower2Proxy = ReplacePort(followers[1].WebUrl, serversToProxies[followers[1]].Port);
Assert.Equal(follower2Proxy, fastest.Url);
}
}
public async Task Round_robin_load_balancing_with_failing_node_should_work()
{
var databaseName = "Round_robin_load_balancing_should_work" + Guid.NewGuid();
var leader = await CreateRaftClusterAndGetLeader(3);
var followers = Servers.Where(x => x.ServerStore.IsLeader() == false).ToArray();
var conventionsForLoadBalancing = new DocumentConventions
{
ReadBalanceBehavior = ReadBalanceBehavior.RoundRobin
};
using (var leaderStore = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower1 = new DocumentStore
{
Urls = new[] { followers[0].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower2 = new DocumentStore
{
Urls = new[] { followers[1].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var context = JsonOperationContext.ShortTermSingleUse())
{
leaderStore.Initialize();
follower1.Initialize();
follower2.Initialize();
var(index, _) = await CreateDatabaseInCluster(databaseName, 3, leader.WebUrl);
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(30));
var leaderRequestExecutor = leaderStore.GetRequestExecutor();
//wait until all nodes in database cluster are members (and not promotables)
//GetTopologyCommand -> does not retrieve promotables
var topology = new Topology();
while (topology.Nodes?.Count != 3)
{
var topologyGetCommand = new GetTopologyCommand();
await leaderRequestExecutor.ExecuteAsync(topologyGetCommand, context);
topology = topologyGetCommand.Result;
Thread.Sleep(50);
}
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(index);
}
using (var session = leaderStore.OpenSession())
{
session.Store(new User {
Name = "John Dow"
});
session.Store(new User {
Name = "Jack Dow"
});
session.Store(new User {
Name = "Jane Dow"
});
session.Store(new User {
Name = "FooBar"
}, "marker");
session.SaveChanges();
await WaitForDocumentInClusterAsync <User>(session as DocumentSession,
"marker", x => true,
leader.ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
}
var requestExecutor = RequestExecutor.Create(follower1.Urls, databaseName, null, follower1.Conventions);
do //make sure there are three nodes in the topology
{
await Task.Delay(100);
} while (requestExecutor.TopologyNodes == null);
DisposeServerAndWaitForFinishOfDisposal(leader);
var failedRequests = new HashSet <(string, Exception)>();
requestExecutor.FailedRequest += (url, e) => failedRequests.Add((url, e));
using (var tmpContext = JsonOperationContext.ShortTermSingleUse())
{
for (var sessionId = 0; sessionId < 5; sessionId++)
{
requestExecutor.Cache.Clear(); //make sure we do not use request cache
await requestExecutor.ExecuteAsync(new GetStatisticsCommand(), tmpContext, CancellationToken.None, new SessionInfo(sessionId, false));
}
}
}
}
public async Task Round_robin_load_balancing_should_work()
{
var databaseName = "Round_robin_load_balancing_should_work" + Guid.NewGuid();
var leader = await CreateRaftClusterAndGetLeader(3);
var followers = Servers.Where(x => x.ServerStore.IsLeader() == false).ToArray();
var conventionsForLoadBalancing = new DocumentConventions
{
ReadBalanceBehavior = ReadBalanceBehavior.RoundRobin
};
using (var leaderStore = new DocumentStore
{
Urls = new [] { leader.WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower1 = new DocumentStore
{
Urls = new[] { followers[0].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower2 = new DocumentStore
{
Urls = new[] { followers[1].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var context = JsonOperationContext.ShortTermSingleUse())
{
leaderStore.Initialize();
follower1.Initialize();
follower2.Initialize();
var(index, _) = await CreateDatabaseInCluster(databaseName, 3, leader.WebUrl);
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(30));
var leaderRequestExecutor = leaderStore.GetRequestExecutor();
//make sure we have updated topology --> more deterministic test
await leaderRequestExecutor.UpdateTopologyAsync(new ServerNode
{
ClusterTag = leader.ServerStore.NodeTag,
Database = databaseName,
Url = leader.WebUrl
}, 5000);
//wait until all nodes in database cluster are members (and not promotables)
//GetTopologyCommand -> does not retrieve promotables
var topology = new Topology();
while (topology.Nodes?.Count != 3)
{
var topologyGetCommand = new GetTopologyCommand();
await leaderRequestExecutor.ExecuteAsync(topologyGetCommand, context);
topology = topologyGetCommand.Result;
Thread.Sleep(50);
}
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(index);
}
using (var session = leaderStore.OpenSession())
{
session.Store(new User {
Name = "John Dow"
});
session.Store(new User {
Name = "Jack Dow"
});
session.Store(new User {
Name = "Jane Dow"
});
session.Store(new User {
Name = "FooBar"
}, "marker");
session.SaveChanges();
await WaitForDocumentInClusterAsync <User>(session as DocumentSession, "marker", x => true, leader.ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
}
var usedUrls = new List <string>();
for (var i = 0; i < 3; i++)
{
using (var session = leaderStore.OpenSession())
{
// ReSharper disable once ReturnValueOfPureMethodIsNotUsed
session.Query <User>().Where(u => u.Name.StartsWith("Ja")).ToList();
usedUrls.Add((await session.Advanced.GetCurrentSessionNode()).Url.ToLower());
}
}
foreach (var url in usedUrls)
{
Assert.Single(usedUrls, url);
}
}
}