public void PingTest()
{
var test = SentinelServerA.Ping();
Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer,
TestConfig.Current.SentinelPortA, test.TotalMilliseconds);
test = SentinelServerB.Ping();
Log("ping to sentinel {0}:{1} took {1} ms", TestConfig.Current.SentinelServer,
TestConfig.Current.SentinelPortB, test.TotalMilliseconds);
test = SentinelServerC.Ping();
Log("ping to sentinel {0}:{1} took {1} ms", TestConfig.Current.SentinelServer,
TestConfig.Current.SentinelPortC, test.TotalMilliseconds);
}
public void SentinelSentinelsTest()
{
var sentinels = SentinelServerA.SentinelSentinels(ServiceName);
var expected = new List <string> {
SentinelServerB.EndPoint.ToString(),
SentinelServerC.EndPoint.ToString()
};
var actual = new List <string>();
foreach (var kv in sentinels)
{
var data = kv.ToDictionary();
actual.Add(data["ip"] + ":" + data["port"]);
}
Assert.All(expected, ep => Assert.NotEqual(ep, SentinelServerA.EndPoint.ToString()));
Assert.True(sentinels.Length == 2);
Assert.All(expected, ep => Assert.Contains(ep, actual, _ipComparer));
sentinels = SentinelServerB.SentinelSentinels(ServiceName);
foreach (var kv in sentinels)
{
var data = kv.ToDictionary();
actual.Add(data["ip"] + ":" + data["port"]);
}
expected = new List <string> {
SentinelServerA.EndPoint.ToString(),
SentinelServerC.EndPoint.ToString()
};
Assert.All(expected, ep => Assert.NotEqual(ep, SentinelServerB.EndPoint.ToString()));
Assert.True(sentinels.Length == 2);
Assert.All(expected, ep => Assert.Contains(ep, actual, _ipComparer));
sentinels = SentinelServerC.SentinelSentinels(ServiceName);
foreach (var kv in sentinels)
{
var data = kv.ToDictionary();
actual.Add(data["ip"] + ":" + data["port"]);
}
expected = new List <string> {
SentinelServerA.EndPoint.ToString(),
SentinelServerB.EndPoint.ToString()
};
Assert.All(expected, ep => Assert.NotEqual(ep, SentinelServerC.EndPoint.ToString()));
Assert.True(sentinels.Length == 2);
Assert.All(expected, ep => Assert.Contains(ep, actual, _ipComparer));
}
public async Task SentinelSlavesAsyncTest()
{
var slaveConfigs = await SentinelServerA.SentinelSlavesAsync(ServiceName).ForAwait();
Assert.True(slaveConfigs.Length > 0);
Assert.True(slaveConfigs[0].ToDictionary().ContainsKey("name"));
Assert.StartsWith("slave", slaveConfigs[0].ToDictionary()["flags"]);
foreach (var config in slaveConfigs)
{
foreach (var kvp in config)
{
Log("{0}:{1}", kvp.Key, kvp.Value);
}
}
}
private void WaitForReady(EndPoint expectedMaster = null, bool waitForReplication = false, TimeSpan?duration = null)
{
duration ??= TimeSpan.FromSeconds(30);
var sw = Stopwatch.StartNew();
// wait until we have 1 master and 1 replica and have verified their roles
var master = SentinelServerA.SentinelGetMasterAddressByName(ServiceName);
if (expectedMaster != null && expectedMaster.ToString() != master.ToString())
{
while (sw.Elapsed < duration.Value)
{
Thread.Sleep(1000);
try
{
master = SentinelServerA.SentinelGetMasterAddressByName(ServiceName);
if (expectedMaster.ToString() == master.ToString())
{
break;
}
}
catch (Exception)
{
// ignore
}
}
}
if (expectedMaster != null && expectedMaster.ToString() != master.ToString())
{
throw new RedisException($"Master was expected to be {expectedMaster}");
}
Log($"Master is {master}");
var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName);
var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions);
WaitForRole(checkConn.GetServer(master), "master", duration.Value.Subtract(sw.Elapsed));
if (replicas.Length > 0)
{
WaitForRole(checkConn.GetServer(replicas[0]), "slave", duration.Value.Subtract(sw.Elapsed));
}
if (waitForReplication)
{
WaitForReplication(checkConn.GetServer(master), duration.Value.Subtract(sw.Elapsed));
}
}
public async Task SentinelMastersAsyncTest()
{
var masterConfigs = await SentinelServerA.SentinelMastersAsync().ForAwait();
Assert.Single(masterConfigs);
Assert.True(masterConfigs[0].ToDictionary().ContainsKey("name"));
Assert.Equal(ServiceName, masterConfigs[0].ToDictionary()["name"]);
Assert.StartsWith("master", masterConfigs[0].ToDictionary()["flags"]);
foreach (var config in masterConfigs)
{
foreach (var kvp in config)
{
Log("{0}:{1}", kvp.Key, kvp.Value);
}
}
}
private void DoFailover()
{
WaitForReady();
// capture current replica
var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName);
Log("Starting failover...");
var sw = Stopwatch.StartNew();
SentinelServerA.SentinelFailover(ServiceName);
// wait until the replica becomes the master
WaitForReady(expectedMaster: replicas[0]);
Log($"Time to failover: {sw.Elapsed}");
}
public void SentinelReplicasTest()
{
var replicaConfigs = SentinelServerA.SentinelReplicas(ServiceName);
Assert.True(replicaConfigs.Length > 0);
Assert.True(replicaConfigs[0].ToDictionary().ContainsKey("name"));
Assert.StartsWith("slave", replicaConfigs[0].ToDictionary()["flags"]);
foreach (var config in replicaConfigs)
{
foreach (var kvp in config)
{
Log("{0}:{1}", kvp.Key, kvp.Value);
}
}
}
public void SentinelMastersTest()
{
var masterConfigs = SentinelServerA.SentinelMasters();
Assert.Single(masterConfigs);
Assert.True(masterConfigs[0].ToDictionary().ContainsKey("name"), "replicaConfigs contains 'name'");
Assert.Equal(ServiceName, masterConfigs[0].ToDictionary()["name"]);
Assert.StartsWith("master", masterConfigs[0].ToDictionary()["flags"]);
foreach (var config in masterConfigs)
{
foreach (var kvp in config)
{
Log("{0}:{1}", kvp.Key, kvp.Value);
}
}
}
public void SentinelSlavesTest()
{
var slaveConfigs = SentinelServerA.SentinelSlaves(ServiceName);
Assert.True(slaveConfigs.Length > 0);
Assert.True(slaveConfigs[0].ToDictionary().ContainsKey("name"));
Assert.Equal("slave", slaveConfigs[0].ToDictionary()["flags"]);
foreach (var config in slaveConfigs)
{
foreach (var kvp in config)
{
Log("{0}:{1}", kvp.Key, kvp.Value);
}
}
}
public async Task SentinelGetSentinelAddressesTest()
{
var addresses = await SentinelServerA.SentinelGetSentinelAddressesAsync(ServiceName).ForAwait();
Assert.Contains(SentinelServerB.EndPoint, addresses);
Assert.Contains(SentinelServerC.EndPoint, addresses);
addresses = await SentinelServerB.SentinelGetSentinelAddressesAsync(ServiceName).ForAwait();
Assert.Contains(SentinelServerA.EndPoint, addresses);
Assert.Contains(SentinelServerC.EndPoint, addresses);
addresses = await SentinelServerC.SentinelGetSentinelAddressesAsync(ServiceName).ForAwait();
Assert.Contains(SentinelServerA.EndPoint, addresses);
Assert.Contains(SentinelServerB.EndPoint, addresses);
}
public async Task GetSentinelMasterConnectionFailoverTest()
{
var conn = Conn.GetSentinelMasterConnection(new ConfigurationOptions {
ServiceName = ServiceName
});
var endpoint = conn.currentSentinelMasterEndPoint.ToString();
SentinelServerA.SentinelFailover(ServiceName);
await Task.Delay(2000).ForAwait();
var conn1 = Conn.GetSentinelMasterConnection(new ConfigurationOptions {
ServiceName = ServiceName
});
var endpoint1 = conn1.currentSentinelMasterEndPoint.ToString();
Assert.NotEqual(endpoint, endpoint1);
}
public async Task SentinelReplicasAsyncTest()
{
// Give previous test run a moment to reset when multi-framework failover is in play.
await UntilCondition(TimeSpan.FromSeconds(5), () => SentinelServerA.SentinelReplicas(ServiceName).Length > 0);
var replicaConfigs = await SentinelServerA.SentinelReplicasAsync(ServiceName).ForAwait();
Assert.True(replicaConfigs.Length > 0, "Has replicaConfigs");
Assert.True(replicaConfigs[0].ToDictionary().ContainsKey("name"), "replicaConfigs contains 'name'");
Assert.StartsWith("slave", replicaConfigs[0].ToDictionary()["flags"]);
foreach (var config in replicaConfigs)
{
foreach (var kvp in config)
{
Log("{0}:{1}", kvp.Key, kvp.Value);
}
}
}
public async Task ReadOnlyConnectionReplicasTest()
{
var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName);
var config = new ConfigurationOptions();
foreach (var replica in replicas)
{
config.EndPoints.Add(replica);
}
var readonlyConn = await ConnectionMultiplexer.ConnectAsync(config);
await UntilCondition(TimeSpan.FromSeconds(2), () => readonlyConn.IsConnected);
Assert.True(readonlyConn.IsConnected);
var db = readonlyConn.GetDatabase();
var s = db.StringGet("test");
Assert.True(s.IsNullOrEmpty);
//var ex = Assert.Throws<RedisConnectionException>(() => db.StringSet("test", "try write to read only instance"));
//Assert.StartsWith("No connection is available to service this operation", ex.Message);
}
public async Task GetSentinelMasterConnectionWriteReadFailover()
{
Log("Conn:");
foreach (var server in Conn.GetServerSnapshot().ToArray())
{
Log(" Endpoint: " + server.EndPoint);
}
Log("Conn Slaves:");
foreach (var slaves in SentinelServerA.SentinelSlaves(ServiceName))
{
foreach (var pair in slaves)
{
Log(" {0}: {1}", pair.Key, pair.Value);
}
}
var conn = Conn.GetSentinelMasterConnection(ServiceOptions);
var s = conn.currentSentinelMasterEndPoint.ToString();
Log("Sentinel Master Endpoint: " + s);
foreach (var server in conn.GetServerSnapshot().ToArray())
{
Log(" Server: " + server.EndPoint);
Log(" Master Endpoint: " + server.MasterEndPoint);
Log(" IsSlave: " + server.IsSlave);
Log(" SlaveReadOnly: " + server.SlaveReadOnly);
var info = conn.GetServer(server.EndPoint).Info("Replication");
foreach (var section in info)
{
Log(" Section: " + section.Key);
foreach (var pair in section)
{
Log(" " + pair.Key + ": " + pair.Value);
}
}
}
IDatabase db = conn.GetDatabase();
var expected = DateTime.Now.Ticks.ToString();
Log("Tick Key: " + expected);
var key = Me();
db.KeyDelete(key, CommandFlags.FireAndForget);
db.StringSet(key, expected);
await UntilCondition(TimeSpan.FromSeconds(10),
() => SentinelServerA.SentinelMaster(ServiceName).ToDictionary()["num-slaves"] != "0"
);
Log("Conditions met");
try
{
Log("Failover attempted initiated");
SentinelServerA.SentinelFailover(ServiceName);
Log(" Success!");
}
catch (RedisServerException ex) when(ex.Message.Contains("NOGOODSLAVE"))
{
// Retry once
Log(" Retry initiated");
await Task.Delay(1000).ForAwait();
SentinelServerA.SentinelFailover(ServiceName);
Log(" Retry complete");
}
Log("Delaying for failover conditions...");
await Task.Delay(2000).ForAwait();
Log("Conditons check...");
// Spin until complete (with a timeout) - since this can vary
await UntilCondition(TimeSpan.FromSeconds(20), () =>
{
var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions);
return(s != checkConn.currentSentinelMasterEndPoint.ToString() &&
expected == checkConn.GetDatabase().StringGet(key));
});
Log(" Conditions met.");
var conn1 = Conn.GetSentinelMasterConnection(ServiceOptions);
var s1 = conn1.currentSentinelMasterEndPoint.ToString();
Log("New master endpoint: " + s1);
var actual = conn1.GetDatabase().StringGet(key);
Log("Fetched tick key: " + actual);
Assert.NotNull(s);
Assert.NotNull(s1);
Assert.NotEmpty(s);
Assert.NotEmpty(s1);
Assert.NotEqual(s, s1);
// TODO: Track this down on the test race
//Assert.Equal(expected, actual);
}
public async Task ManagedPrimaryConnectionEndToEndWithFailoverTest()
{
var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},serviceName={ServiceOptions.ServiceName},allowAdmin=true";
var conn = await ConnectionMultiplexer.ConnectAsync(connectionString);
conn.ConfigurationChanged += (s, e) => Log($"Configuration changed: {e.EndPoint}");
var sub = conn.GetSubscriber();
sub.Subscribe("*", (channel, message) => Log($"Sub: {channel}, message:{message}"));
var db = conn.GetDatabase();
await db.PingAsync();
var endpoints = conn.GetEndPoints();
Assert.Equal(2, endpoints.Length);
var servers = endpoints.Select(e => conn.GetServer(e)).ToArray();
Assert.Equal(2, servers.Length);
var primary = servers.FirstOrDefault(s => !s.IsReplica);
Assert.NotNull(primary);
var replica = servers.FirstOrDefault(s => s.IsReplica);
Assert.NotNull(replica);
Assert.NotEqual(primary.EndPoint.ToString(), replica.EndPoint.ToString());
// Set string value on current primary
var expected = DateTime.Now.Ticks.ToString();
Log("Tick Key: " + expected);
var key = Me();
await db.KeyDeleteAsync(key, CommandFlags.FireAndForget);
await db.StringSetAsync(key, expected);
var value = await db.StringGetAsync(key);
Assert.Equal(expected, value);
Log("Waiting for first replication check...");
// force read from replica, replication has some lag
await WaitForReplicationAsync(servers[0]).ForAwait();
value = await db.StringGetAsync(key, CommandFlags.DemandReplica);
Assert.Equal(expected, value);
Log("Waiting for ready pre-failover...");
await WaitForReadyAsync();
// capture current replica
var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName);
Log("Starting failover...");
var sw = Stopwatch.StartNew();
SentinelServerA.SentinelFailover(ServiceName);
// There's no point in doing much for 10 seconds - this is a built-in delay of how Sentinel works.
// The actual completion invoking the replication of the former primary is handled via
// https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L4799-L4808
// ...which is invoked by INFO polls every 10 seconds (https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L81)
// ...which is calling https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L2666
// However, the quicker iteration on INFO during an o_down does not apply here: https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L3089-L3104
// So...we're waiting 10 seconds, no matter what. Might as well just idle to be more stable.
await Task.Delay(TimeSpan.FromSeconds(10));
// wait until the replica becomes the primary
Log("Waiting for ready post-failover...");
await WaitForReadyAsync(expectedPrimary : replicas[0]);
Log($"Time to failover: {sw.Elapsed}");
endpoints = conn.GetEndPoints();
Assert.Equal(2, endpoints.Length);
servers = endpoints.Select(e => conn.GetServer(e)).ToArray();
Assert.Equal(2, servers.Length);
var newPrimary = servers.FirstOrDefault(s => !s.IsReplica);
Assert.NotNull(newPrimary);
Assert.Equal(replica.EndPoint.ToString(), newPrimary.EndPoint.ToString());
var newReplica = servers.FirstOrDefault(s => s.IsReplica);
Assert.NotNull(newReplica);
Assert.Equal(primary.EndPoint.ToString(), newReplica.EndPoint.ToString());
Assert.NotEqual(primary.EndPoint.ToString(), replica.EndPoint.ToString());
value = await db.StringGetAsync(key);
Assert.Equal(expected, value);
Log("Waiting for second replication check...");
// force read from replica, replication has some lag
await WaitForReplicationAsync(newPrimary).ForAwait();
value = await db.StringGetAsync(key, CommandFlags.DemandReplica);
Assert.Equal(expected, value);
}
