public void PingTest() { var test = SentinelServerA.Ping(); Log("ping to sentinel {0}:{1} took {2} ms", TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortA, test.TotalMilliseconds); test = SentinelServerB.Ping(); Log("ping to sentinel {0}:{1} took {1} ms", TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortB, test.TotalMilliseconds); test = SentinelServerC.Ping(); Log("ping to sentinel {0}:{1} took {1} ms", TestConfig.Current.SentinelServer, TestConfig.Current.SentinelPortC, test.TotalMilliseconds); }
public void SentinelSentinelsTest() { var sentinels = SentinelServerA.SentinelSentinels(ServiceName); var expected = new List <string> { SentinelServerB.EndPoint.ToString(), SentinelServerC.EndPoint.ToString() }; var actual = new List <string>(); foreach (var kv in sentinels) { var data = kv.ToDictionary(); actual.Add(data["ip"] + ":" + data["port"]); } Assert.All(expected, ep => Assert.NotEqual(ep, SentinelServerA.EndPoint.ToString())); Assert.True(sentinels.Length == 2); Assert.All(expected, ep => Assert.Contains(ep, actual, _ipComparer)); sentinels = SentinelServerB.SentinelSentinels(ServiceName); foreach (var kv in sentinels) { var data = kv.ToDictionary(); actual.Add(data["ip"] + ":" + data["port"]); } expected = new List <string> { SentinelServerA.EndPoint.ToString(), SentinelServerC.EndPoint.ToString() }; Assert.All(expected, ep => Assert.NotEqual(ep, SentinelServerB.EndPoint.ToString())); Assert.True(sentinels.Length == 2); Assert.All(expected, ep => Assert.Contains(ep, actual, _ipComparer)); sentinels = SentinelServerC.SentinelSentinels(ServiceName); foreach (var kv in sentinels) { var data = kv.ToDictionary(); actual.Add(data["ip"] + ":" + data["port"]); } expected = new List <string> { SentinelServerA.EndPoint.ToString(), SentinelServerB.EndPoint.ToString() }; Assert.All(expected, ep => Assert.NotEqual(ep, SentinelServerC.EndPoint.ToString())); Assert.True(sentinels.Length == 2); Assert.All(expected, ep => Assert.Contains(ep, actual, _ipComparer)); }
public async Task SentinelSlavesAsyncTest() { var slaveConfigs = await SentinelServerA.SentinelSlavesAsync(ServiceName).ForAwait(); Assert.True(slaveConfigs.Length > 0); Assert.True(slaveConfigs[0].ToDictionary().ContainsKey("name")); Assert.StartsWith("slave", slaveConfigs[0].ToDictionary()["flags"]); foreach (var config in slaveConfigs) { foreach (var kvp in config) { Log("{0}:{1}", kvp.Key, kvp.Value); } } }
private void WaitForReady(EndPoint expectedMaster = null, bool waitForReplication = false, TimeSpan?duration = null) { duration ??= TimeSpan.FromSeconds(30); var sw = Stopwatch.StartNew(); // wait until we have 1 master and 1 replica and have verified their roles var master = SentinelServerA.SentinelGetMasterAddressByName(ServiceName); if (expectedMaster != null && expectedMaster.ToString() != master.ToString()) { while (sw.Elapsed < duration.Value) { Thread.Sleep(1000); try { master = SentinelServerA.SentinelGetMasterAddressByName(ServiceName); if (expectedMaster.ToString() == master.ToString()) { break; } } catch (Exception) { // ignore } } } if (expectedMaster != null && expectedMaster.ToString() != master.ToString()) { throw new RedisException($"Master was expected to be {expectedMaster}"); } Log($"Master is {master}"); var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName); var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions); WaitForRole(checkConn.GetServer(master), "master", duration.Value.Subtract(sw.Elapsed)); if (replicas.Length > 0) { WaitForRole(checkConn.GetServer(replicas[0]), "slave", duration.Value.Subtract(sw.Elapsed)); } if (waitForReplication) { WaitForReplication(checkConn.GetServer(master), duration.Value.Subtract(sw.Elapsed)); } }
public async Task SentinelMastersAsyncTest() { var masterConfigs = await SentinelServerA.SentinelMastersAsync().ForAwait(); Assert.Single(masterConfigs); Assert.True(masterConfigs[0].ToDictionary().ContainsKey("name")); Assert.Equal(ServiceName, masterConfigs[0].ToDictionary()["name"]); Assert.StartsWith("master", masterConfigs[0].ToDictionary()["flags"]); foreach (var config in masterConfigs) { foreach (var kvp in config) { Log("{0}:{1}", kvp.Key, kvp.Value); } } }
private void DoFailover() { WaitForReady(); // capture current replica var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName); Log("Starting failover..."); var sw = Stopwatch.StartNew(); SentinelServerA.SentinelFailover(ServiceName); // wait until the replica becomes the master WaitForReady(expectedMaster: replicas[0]); Log($"Time to failover: {sw.Elapsed}"); }
public void SentinelReplicasTest() { var replicaConfigs = SentinelServerA.SentinelReplicas(ServiceName); Assert.True(replicaConfigs.Length > 0); Assert.True(replicaConfigs[0].ToDictionary().ContainsKey("name")); Assert.StartsWith("slave", replicaConfigs[0].ToDictionary()["flags"]); foreach (var config in replicaConfigs) { foreach (var kvp in config) { Log("{0}:{1}", kvp.Key, kvp.Value); } } }
public void SentinelMastersTest() { var masterConfigs = SentinelServerA.SentinelMasters(); Assert.Single(masterConfigs); Assert.True(masterConfigs[0].ToDictionary().ContainsKey("name"), "replicaConfigs contains 'name'"); Assert.Equal(ServiceName, masterConfigs[0].ToDictionary()["name"]); Assert.StartsWith("master", masterConfigs[0].ToDictionary()["flags"]); foreach (var config in masterConfigs) { foreach (var kvp in config) { Log("{0}:{1}", kvp.Key, kvp.Value); } } }
public void SentinelSlavesTest() { var slaveConfigs = SentinelServerA.SentinelSlaves(ServiceName); Assert.True(slaveConfigs.Length > 0); Assert.True(slaveConfigs[0].ToDictionary().ContainsKey("name")); Assert.Equal("slave", slaveConfigs[0].ToDictionary()["flags"]); foreach (var config in slaveConfigs) { foreach (var kvp in config) { Log("{0}:{1}", kvp.Key, kvp.Value); } } }
public async Task SentinelGetSentinelAddressesTest() { var addresses = await SentinelServerA.SentinelGetSentinelAddressesAsync(ServiceName).ForAwait(); Assert.Contains(SentinelServerB.EndPoint, addresses); Assert.Contains(SentinelServerC.EndPoint, addresses); addresses = await SentinelServerB.SentinelGetSentinelAddressesAsync(ServiceName).ForAwait(); Assert.Contains(SentinelServerA.EndPoint, addresses); Assert.Contains(SentinelServerC.EndPoint, addresses); addresses = await SentinelServerC.SentinelGetSentinelAddressesAsync(ServiceName).ForAwait(); Assert.Contains(SentinelServerA.EndPoint, addresses); Assert.Contains(SentinelServerB.EndPoint, addresses); }
public async Task GetSentinelMasterConnectionFailoverTest() { var conn = Conn.GetSentinelMasterConnection(new ConfigurationOptions { ServiceName = ServiceName }); var endpoint = conn.currentSentinelMasterEndPoint.ToString(); SentinelServerA.SentinelFailover(ServiceName); await Task.Delay(2000).ForAwait(); var conn1 = Conn.GetSentinelMasterConnection(new ConfigurationOptions { ServiceName = ServiceName }); var endpoint1 = conn1.currentSentinelMasterEndPoint.ToString(); Assert.NotEqual(endpoint, endpoint1); }
public async Task SentinelReplicasAsyncTest() { // Give previous test run a moment to reset when multi-framework failover is in play. await UntilCondition(TimeSpan.FromSeconds(5), () => SentinelServerA.SentinelReplicas(ServiceName).Length > 0); var replicaConfigs = await SentinelServerA.SentinelReplicasAsync(ServiceName).ForAwait(); Assert.True(replicaConfigs.Length > 0, "Has replicaConfigs"); Assert.True(replicaConfigs[0].ToDictionary().ContainsKey("name"), "replicaConfigs contains 'name'"); Assert.StartsWith("slave", replicaConfigs[0].ToDictionary()["flags"]); foreach (var config in replicaConfigs) { foreach (var kvp in config) { Log("{0}:{1}", kvp.Key, kvp.Value); } } }
public async Task ReadOnlyConnectionReplicasTest() { var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName); var config = new ConfigurationOptions(); foreach (var replica in replicas) { config.EndPoints.Add(replica); } var readonlyConn = await ConnectionMultiplexer.ConnectAsync(config); await UntilCondition(TimeSpan.FromSeconds(2), () => readonlyConn.IsConnected); Assert.True(readonlyConn.IsConnected); var db = readonlyConn.GetDatabase(); var s = db.StringGet("test"); Assert.True(s.IsNullOrEmpty); //var ex = Assert.Throws<RedisConnectionException>(() => db.StringSet("test", "try write to read only instance")); //Assert.StartsWith("No connection is available to service this operation", ex.Message); }
public async Task GetSentinelMasterConnectionWriteReadFailover() { Log("Conn:"); foreach (var server in Conn.GetServerSnapshot().ToArray()) { Log(" Endpoint: " + server.EndPoint); } Log("Conn Slaves:"); foreach (var slaves in SentinelServerA.SentinelSlaves(ServiceName)) { foreach (var pair in slaves) { Log(" {0}: {1}", pair.Key, pair.Value); } } var conn = Conn.GetSentinelMasterConnection(ServiceOptions); var s = conn.currentSentinelMasterEndPoint.ToString(); Log("Sentinel Master Endpoint: " + s); foreach (var server in conn.GetServerSnapshot().ToArray()) { Log(" Server: " + server.EndPoint); Log(" Master Endpoint: " + server.MasterEndPoint); Log(" IsSlave: " + server.IsSlave); Log(" SlaveReadOnly: " + server.SlaveReadOnly); var info = conn.GetServer(server.EndPoint).Info("Replication"); foreach (var section in info) { Log(" Section: " + section.Key); foreach (var pair in section) { Log(" " + pair.Key + ": " + pair.Value); } } } IDatabase db = conn.GetDatabase(); var expected = DateTime.Now.Ticks.ToString(); Log("Tick Key: " + expected); var key = Me(); db.KeyDelete(key, CommandFlags.FireAndForget); db.StringSet(key, expected); await UntilCondition(TimeSpan.FromSeconds(10), () => SentinelServerA.SentinelMaster(ServiceName).ToDictionary()["num-slaves"] != "0" ); Log("Conditions met"); try { Log("Failover attempted initiated"); SentinelServerA.SentinelFailover(ServiceName); Log(" Success!"); } catch (RedisServerException ex) when(ex.Message.Contains("NOGOODSLAVE")) { // Retry once Log(" Retry initiated"); await Task.Delay(1000).ForAwait(); SentinelServerA.SentinelFailover(ServiceName); Log(" Retry complete"); } Log("Delaying for failover conditions..."); await Task.Delay(2000).ForAwait(); Log("Conditons check..."); // Spin until complete (with a timeout) - since this can vary await UntilCondition(TimeSpan.FromSeconds(20), () => { var checkConn = Conn.GetSentinelMasterConnection(ServiceOptions); return(s != checkConn.currentSentinelMasterEndPoint.ToString() && expected == checkConn.GetDatabase().StringGet(key)); }); Log(" Conditions met."); var conn1 = Conn.GetSentinelMasterConnection(ServiceOptions); var s1 = conn1.currentSentinelMasterEndPoint.ToString(); Log("New master endpoint: " + s1); var actual = conn1.GetDatabase().StringGet(key); Log("Fetched tick key: " + actual); Assert.NotNull(s); Assert.NotNull(s1); Assert.NotEmpty(s); Assert.NotEmpty(s1); Assert.NotEqual(s, s1); // TODO: Track this down on the test race //Assert.Equal(expected, actual); }
public async Task ManagedPrimaryConnectionEndToEndWithFailoverTest() { var connectionString = $"{TestConfig.Current.SentinelServer}:{TestConfig.Current.SentinelPortA},serviceName={ServiceOptions.ServiceName},allowAdmin=true"; var conn = await ConnectionMultiplexer.ConnectAsync(connectionString); conn.ConfigurationChanged += (s, e) => Log($"Configuration changed: {e.EndPoint}"); var sub = conn.GetSubscriber(); sub.Subscribe("*", (channel, message) => Log($"Sub: {channel}, message:{message}")); var db = conn.GetDatabase(); await db.PingAsync(); var endpoints = conn.GetEndPoints(); Assert.Equal(2, endpoints.Length); var servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); Assert.Equal(2, servers.Length); var primary = servers.FirstOrDefault(s => !s.IsReplica); Assert.NotNull(primary); var replica = servers.FirstOrDefault(s => s.IsReplica); Assert.NotNull(replica); Assert.NotEqual(primary.EndPoint.ToString(), replica.EndPoint.ToString()); // Set string value on current primary var expected = DateTime.Now.Ticks.ToString(); Log("Tick Key: " + expected); var key = Me(); await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); await db.StringSetAsync(key, expected); var value = await db.StringGetAsync(key); Assert.Equal(expected, value); Log("Waiting for first replication check..."); // force read from replica, replication has some lag await WaitForReplicationAsync(servers[0]).ForAwait(); value = await db.StringGetAsync(key, CommandFlags.DemandReplica); Assert.Equal(expected, value); Log("Waiting for ready pre-failover..."); await WaitForReadyAsync(); // capture current replica var replicas = SentinelServerA.SentinelGetReplicaAddresses(ServiceName); Log("Starting failover..."); var sw = Stopwatch.StartNew(); SentinelServerA.SentinelFailover(ServiceName); // There's no point in doing much for 10 seconds - this is a built-in delay of how Sentinel works. // The actual completion invoking the replication of the former primary is handled via // https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L4799-L4808 // ...which is invoked by INFO polls every 10 seconds (https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L81) // ...which is calling https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L2666 // However, the quicker iteration on INFO during an o_down does not apply here: https://github.com/redis/redis/blob/f233c4c59d24828c77eb1118f837eaee14695f7f/src/sentinel.c#L3089-L3104 // So...we're waiting 10 seconds, no matter what. Might as well just idle to be more stable. await Task.Delay(TimeSpan.FromSeconds(10)); // wait until the replica becomes the primary Log("Waiting for ready post-failover..."); await WaitForReadyAsync(expectedPrimary : replicas[0]); Log($"Time to failover: {sw.Elapsed}"); endpoints = conn.GetEndPoints(); Assert.Equal(2, endpoints.Length); servers = endpoints.Select(e => conn.GetServer(e)).ToArray(); Assert.Equal(2, servers.Length); var newPrimary = servers.FirstOrDefault(s => !s.IsReplica); Assert.NotNull(newPrimary); Assert.Equal(replica.EndPoint.ToString(), newPrimary.EndPoint.ToString()); var newReplica = servers.FirstOrDefault(s => s.IsReplica); Assert.NotNull(newReplica); Assert.Equal(primary.EndPoint.ToString(), newReplica.EndPoint.ToString()); Assert.NotEqual(primary.EndPoint.ToString(), replica.EndPoint.ToString()); value = await db.StringGetAsync(key); Assert.Equal(expected, value); Log("Waiting for second replication check..."); // force read from replica, replication has some lag await WaitForReplicationAsync(newPrimary).ForAwait(); value = await db.StringGetAsync(key, CommandFlags.DemandReplica); Assert.Equal(expected, value); }