public async Task GetSentinelMasterConnectionFailoverTest()
{
var conn = Conn.GetSentinelMasterConnection(ServiceOptions);
var endpoint = conn.currentSentinelMasterEndPoint.ToString();
try
{
Log("Failover attempted initiated");
SentinelServerA.SentinelFailover(ServiceName);
Log(" Success!");
}
catch (RedisServerException ex) when(ex.Message.Contains("NOGOODSLAVE"))
{
// Retry once
Log(" Retry initiated");
await Task.Delay(1000).ForAwait();
SentinelServerA.SentinelFailover(ServiceName);
Log(" Retry complete");
}
await Task.Delay(2000).ForAwait();
// Try and complete ASAP
await UntilCondition(TimeSpan.FromSeconds(10), () => {
var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions);
return(endpoint != checkConn.currentSentinelMasterEndPoint.ToString());
});
// Post-check for validity
var conn1 = Conn.GetSentinelMasterConnection(ServiceOptions);
Assert.NotEqual(endpoint, conn1.currentSentinelMasterEndPoint.ToString());
}
public async Task GetSentinelMasterConnectionWriteReadFailover()
{
var conn = Conn.GetSentinelMasterConnection(new ConfigurationOptions {
ServiceName = ServiceName
});
var s = conn.currentSentinelMasterEndPoint.ToString();
IDatabase db = conn.GetDatabase();
var expected = DateTime.Now.Ticks.ToString();
db.StringSet("beforeFailOverValue", expected);
SentinelServerA.SentinelFailover(ServiceName);
await Task.Delay(2000).ForAwait();
var conn1 = Conn.GetSentinelMasterConnection(new ConfigurationOptions {
ServiceName = ServiceName
});
var s1 = conn1.currentSentinelMasterEndPoint.ToString();
var db1 = conn1.GetDatabase();
var actual = db1.StringGet("beforeFailOverValue");
Assert.NotNull(s);
Assert.NotNull(s1);
Assert.NotEmpty(s);
Assert.NotEmpty(s1);
Assert.NotEqual(s, s1);
Assert.Equal(expected, actual);
}
private void DoFailover()
{
WaitForReady();
// capture current replica
var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName);
Log("Starting failover...");
var sw = Stopwatch.StartNew();
SentinelServerA.SentinelFailover(ServiceName);
// wait until the replica becomes the master
WaitForReady(expectedMaster: replicas[0]);
Log($"Time to failover: {sw.Elapsed}");
}
public async Task GetSentinelMasterConnectionFailoverTest()
{
var conn = Conn.GetSentinelMasterConnection(new ConfigurationOptions {
ServiceName = ServiceName
});
var endpoint = conn.currentSentinelMasterEndPoint.ToString();
SentinelServerA.SentinelFailover(ServiceName);
await Task.Delay(2000).ForAwait();
var conn1 = Conn.GetSentinelMasterConnection(new ConfigurationOptions {
ServiceName = ServiceName
});
var endpoint1 = conn1.currentSentinelMasterEndPoint.ToString();
Assert.NotEqual(endpoint, endpoint1);
}
public async Task GetSentinelMasterConnectionWriteReadFailover()
{
Log("Conn:");
foreach (var server in Conn.GetServerSnapshot().ToArray())
{
Log(" Endpoint: " + server.EndPoint);
}
Log("Conn Slaves:");
foreach (var slaves in SentinelServerA.SentinelSlaves(ServiceName))
{
foreach (var pair in slaves)
{
Log(" {0}: {1}", pair.Key, pair.Value);
}
}
var conn = Conn.GetSentinelMasterConnection(ServiceOptions);
var s = conn.currentSentinelMasterEndPoint.ToString();
Log("Sentinel Master Endpoint: " + s);
foreach (var server in conn.GetServerSnapshot().ToArray())
{
Log(" Server: " + server.EndPoint);
Log(" Master Endpoint: " + server.MasterEndPoint);
Log(" IsSlave: " + server.IsSlave);
Log(" SlaveReadOnly: " + server.SlaveReadOnly);
var info = conn.GetServer(server.EndPoint).Info("Replication");
foreach (var section in info)
{
Log(" Section: " + section.Key);
foreach (var pair in section)
{
Log(" " + pair.Key + ": " + pair.Value);
}
}
}
IDatabase db = conn.GetDatabase();
var expected = DateTime.Now.Ticks.ToString();
Log("Tick Key: " + expected);
var key = Me();
db.KeyDelete(key, CommandFlags.FireAndForget);
db.StringSet(key, expected);
await UntilCondition(TimeSpan.FromSeconds(10),
() => SentinelServerA.SentinelMaster(ServiceName).ToDictionary()["num-slaves"] != "0"
);
Log("Conditions met");
try
{
Log("Failover attempted initiated");
SentinelServerA.SentinelFailover(ServiceName);
Log(" Success!");
}
catch (RedisServerException ex) when(ex.Message.Contains("NOGOODSLAVE"))
{
// Retry once
Log(" Retry initiated");
await Task.Delay(1000).ForAwait();
SentinelServerA.SentinelFailover(ServiceName);
Log(" Retry complete");
}
Log("Delaying for failover conditions...");
await Task.Delay(2000).ForAwait();
Log("Conditons check...");
// Spin until complete (with a timeout) - since this can vary
await UntilCondition(TimeSpan.FromSeconds(20), () =>
{
var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions);
return(s != checkConn.currentSentinelMasterEndPoint.ToString() &&
expected == checkConn.GetDatabase().StringGet(key));
});
Log(" Conditions met.");
var conn1 = Conn.GetSentinelMasterConnection(ServiceOptions);
var s1 = conn1.currentSentinelMasterEndPoint.ToString();
Log("New master endpoint: " + s1);
var actual = conn1.GetDatabase().StringGet(key);
Log("Fetched tick key: " + actual);
Assert.NotNull(s);
Assert.NotNull(s1);
Assert.NotEmpty(s);
Assert.NotEmpty(s1);
Assert.NotEqual(s, s1);
// TODO: Track this down on the test race
//Assert.Equal(expected, actual);
}
public async Task ManagedPrimaryConnectionEndToEndWithFailoverTest()
{
var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},serviceName={ServiceOptions.ServiceName},allowAdmin=true";
var conn = await ConnectionMultiplexer.ConnectAsync(connectionString);
conn.ConfigurationChanged += (s, e) => Log($"Configuration changed: {e.EndPoint}");
var sub = conn.GetSubscriber();
sub.Subscribe("*", (channel, message) => Log($"Sub: {channel}, message:{message}"));
var db = conn.GetDatabase();
await db.PingAsync();
var endpoints = conn.GetEndPoints();
Assert.Equal(2, endpoints.Length);
var servers = endpoints.Select(e => conn.GetServer(e)).ToArray();
Assert.Equal(2, servers.Length);
var primary = servers.FirstOrDefault(s => !s.IsReplica);
Assert.NotNull(primary);
var replica = servers.FirstOrDefault(s => s.IsReplica);
Assert.NotNull(replica);
Assert.NotEqual(primary.EndPoint.ToString(), replica.EndPoint.ToString());
// Set string value on current primary
var expected = DateTime.Now.Ticks.ToString();
Log("Tick Key: " + expected);
var key = Me();
await db.KeyDeleteAsync(key, CommandFlags.FireAndForget);
await db.StringSetAsync(key, expected);
var value = await db.StringGetAsync(key);
Assert.Equal(expected, value);
Log("Waiting for first replication check...");
// force read from replica, replication has some lag
await WaitForReplicationAsync(servers[0]).ForAwait();
value = await db.StringGetAsync(key, CommandFlags.DemandReplica);
Assert.Equal(expected, value);
Log("Waiting for ready pre-failover...");
await WaitForReadyAsync();
// capture current replica
var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName);
Log("Starting failover...");
var sw = Stopwatch.StartNew();
SentinelServerA.SentinelFailover(ServiceName);
// There's no point in doing much for 10 seconds - this is a built-in delay of how Sentinel works.
// The actual completion invoking the replication of the former primary is handled via
// https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L4799-L4808
// ...which is invoked by INFO polls every 10 seconds (https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L81)
// ...which is calling https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L2666
// However, the quicker iteration on INFO during an o_down does not apply here: https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L3089-L3104
// So...we're waiting 10 seconds, no matter what. Might as well just idle to be more stable.
await Task.Delay(TimeSpan.FromSeconds(10));
// wait until the replica becomes the primary
Log("Waiting for ready post-failover...");
await WaitForReadyAsync(expectedPrimary : replicas[0]);
Log($"Time to failover: {sw.Elapsed}");
endpoints = conn.GetEndPoints();
Assert.Equal(2, endpoints.Length);
servers = endpoints.Select(e => conn.GetServer(e)).ToArray();
Assert.Equal(2, servers.Length);
var newPrimary = servers.FirstOrDefault(s => !s.IsReplica);
Assert.NotNull(newPrimary);
Assert.Equal(replica.EndPoint.ToString(), newPrimary.EndPoint.ToString());
var newReplica = servers.FirstOrDefault(s => s.IsReplica);
Assert.NotNull(newReplica);
Assert.Equal(primary.EndPoint.ToString(), newReplica.EndPoint.ToString());
Assert.NotEqual(primary.EndPoint.ToString(), replica.EndPoint.ToString());
value = await db.StringGetAsync(key);
Assert.Equal(expected, value);
Log("Waiting for second replication check...");
// force read from replica, replication has some lag
await WaitForReplicationAsync(newPrimary).ForAwait();
value = await db.StringGetAsync(key, CommandFlags.DemandReplica);
Assert.Equal(expected, value);
}
