public async Task ConfigVerifyReceiveConfigChangeBroadcast()
{
_ = GetConfiguration();
using (var sender = Create(allowAdmin: true))
using (var receiver = Create(syncTimeout: 2000))
{
int total = 0;
receiver.ConfigurationChangedBroadcast += (s, a) =>
{
Log("Config changed: " + (a.EndPoint == null ? "(none)" : a.EndPoint.ToString()));
Interlocked.Increment(ref total);
};
// send a reconfigure/reconnect message
long count = sender.PublishReconfigure();
GetServer(receiver).Ping();
GetServer(receiver).Ping();
await Task.Delay(1000).ConfigureAwait(false);
Assert.True(count == -1 || count >= 2, "subscribers");
Assert.True(Interlocked.CompareExchange(ref total, 0, 0) >= 1, "total (1st)");
Interlocked.Exchange(ref total, 0);
// and send a second time via a re-master operation
var server = GetServer(sender);
if (server.IsReplica)
{
Skip.Inconclusive("didn't expect a replica");
}
server.MakeMaster(ReplicationChangeOptions.Broadcast);
await Task.Delay(1000).ConfigureAwait(false);
GetServer(receiver).Ping();
GetServer(receiver).Ping();
Assert.True(Interlocked.CompareExchange(ref total, 0, 0) >= 1, "total (2nd)");
}
}
public void LuaScriptWithKeys()
{
const string Script = "redis.call('set', @key, @value)";
using (var conn = Create(allowAdmin: true))
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Scripting), f => f.Scripting);
var server = conn.GetServer(TestConfig.Current.MasterServerAndPort);
server.ScriptFlush();
var script = LuaScript.Prepare(Script);
var db = conn.GetDatabase();
var key = Me();
db.KeyDelete(key, CommandFlags.FireAndForget);
var p = new { key = (RedisKey)key, value = 123 };
script.Evaluate(db, p);
var val = db.StringGet(key);
Assert.Equal(123, (int)val);
// no super clean way to extract this; so just abuse InternalsVisibleTo
script.ExtractParameters(p, null, out RedisKey[] keys, out RedisValue[] args);
public void ConnectToAzure(int?port, bool ssl)
{
Skip.IfNoConfig(nameof(TestConfig.Config.AzureCacheServer), TestConfig.Current.AzureCacheServer);
Skip.IfNoConfig(nameof(TestConfig.Config.AzureCachePassword), TestConfig.Current.AzureCachePassword);
var options = new ConfigurationOptions();
if (port == null)
{
options.EndPoints.Add(TestConfig.Current.AzureCacheServer);
}
else
{
options.EndPoints.Add(TestConfig.Current.AzureCacheServer, port.Value);
}
options.Ssl = ssl;
options.Password = TestConfig.Current.AzureCachePassword;
Output.WriteLine(options.ToString());
using (var connection = ConnectionMultiplexer.Connect(options))
{
var ttl = connection.GetDatabase().Ping();
Output.WriteLine(ttl.ToString());
}
}
public void StreamInfoGet()
{
var key = GetUniqueKey("stream_info");
using (var conn = Create())
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Streams), r => r.Streams);
var db = conn.GetDatabase();
var id1 = db.StreamAdd(key, "field1", "value1");
var id2 = db.StreamAdd(key, "field2", "value2");
var id3 = db.StreamAdd(key, "field3", "value3");
var id4 = db.StreamAdd(key, "field4", "value4");
var streamInfo = db.StreamInfo(key);
Assert.Equal(4, streamInfo.Length);
Assert.True(streamInfo.RadixTreeKeys > 0);
Assert.True(streamInfo.RadixTreeNodes > 0);
Assert.Equal(id1, streamInfo.FirstEntry.Id);
Assert.Equal(id4, streamInfo.LastEntry.Id);
}
}
public void StreamReadEmptyStream()
{
var key = GetUniqueKey("read_empty_stream");
using (var conn = Create())
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Streams), r => r.Streams);
var db = conn.GetDatabase();
// Write to a stream to create the key.
var id1 = db.StreamAdd(key, "field1", "value1");
// Delete the key to empty the stream.
db.StreamDelete(key, new RedisValue[] { id1 });
var len = db.StreamLength(key);
// Read the entire stream from the beginning.
var entries = db.StreamRead(key, "0-0");
Assert.True(entries.Length == 0);
Assert.Equal(0, len);
}
}
public void StreamConsumerGroupViewPendingMessageInfoForConsumer()
{
var key = GetUniqueKey("group_pending_for_consumer");
var groupName = "test_group";
var consumer1 = "test_consumer_1";
var consumer2 = "test_consumer_2";
using (var conn = Create())
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Streams), r => r.Streams);
var db = conn.GetDatabase();
var id1 = db.StreamAdd(key, "field1", "value1");
var id2 = db.StreamAdd(key, "field2", "value2");
var id3 = db.StreamAdd(key, "field3", "value3");
var id4 = db.StreamAdd(key, "field4", "value4");
db.StreamCreateConsumerGroup(key, groupName, "-");
// Read a single message into the first consumer.
var consumer1Messages = db.StreamReadGroup(key, groupName, consumer1, count: 1);
// Read the remaining messages into the second consumer.
var consumer2Messages = db.StreamReadGroup(key, groupName, consumer2);
// Get the pending info about the messages themselves.
var pendingMessageInfoList = db.StreamPendingMessages(key,
groupName,
10,
consumer2);
Assert.NotNull(pendingMessageInfoList);
Assert.Equal(3, pendingMessageInfoList.Length);
}
}
public void StreamConsumerGroupViewPendingInfoWhenNothingPending()
{
var key = GetUniqueKey("group_pending_info_nothing_pending");
var groupName = "test_group";
using (var conn = Create())
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Streams), r => r.Streams);
var db = conn.GetDatabase();
var id1 = db.StreamAdd(key, "field1", "value1");
db.StreamCreateConsumerGroup(key, groupName, "0-0");
var pendingMessages = db.StreamPendingMessages(key,
groupName,
10,
consumerName: RedisValue.Null);
Assert.NotNull(pendingMessages);
Assert.True(pendingMessages.Length == 0);
}
}
public void StreamConsumerGroupViewPendingInfoNoConsumers()
{
var key = GetUniqueKey("group_pending_info_no_consumers");
var groupName = "test_group";
using (var conn = Create())
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Streams), r => r.Streams);
var db = conn.GetDatabase();
var id1 = db.StreamAdd(key, "field1", "value1");
db.StreamCreateConsumerGroup(key, groupName, "-");
var pendingInfo = db.StreamPending(key, groupName);
Assert.Equal(0, pendingInfo.PendingMessageCount);
Assert.True(pendingInfo.LowestPendingMessageId == RedisValue.Null);
Assert.True(pendingInfo.HighestPendingMessageId == RedisValue.Null);
Assert.NotNull(pendingInfo.Consumers);
Assert.True(pendingInfo.Consumers.Length == 0);
}
}
public void StreamReadMultipleStreamsWithCount()
{
var key1 = GetUniqueKey("read_multi_count_1");
var key2 = GetUniqueKey("read_multi_count_2");
using (var conn = Create())
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Streams), r => r.Streams);
var db = conn.GetDatabase();
var id1 = db.StreamAdd(key1, "field1", "value1");
var id2 = db.StreamAdd(key1, "fiedl2", "value2");
var id3 = db.StreamAdd(key2, "field3", "value3");
var id4 = db.StreamAdd(key2, "field4", "value4");
var streamList = new StreamIdPair[2]
{
new StreamIdPair(key1, "0-0"),
new StreamIdPair(key2, "0-0")
};
var streams = db.StreamRead(streamList, countPerStream: 1);
// We should get both streams back.
Assert.True(streams.Length == 2);
// Ensure we only got one message per stream.
Assert.True(streams[0].Entries.Length == 1);
Assert.True(streams[1].Entries.Length == 1);
// Check the message IDs as well.
Assert.Equal(id1, streams[0].Entries[0].Id);
Assert.Equal(id3, streams[1].Entries[0].Id);
}
}
public void SortedSetPopMulti_Multi()
{
using (var conn = Create())
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.SortedSetPop), r => r.SortedSetPop);
var db = conn.GetDatabase();
var key = Me();
db.KeyDelete(key, CommandFlags.FireAndForget);
db.SortedSetAdd(key, entries, CommandFlags.FireAndForget);
var first = db.SortedSetPop(key, Order.Ascending);
Assert.True(first.HasValue);
Assert.Equal(entries[0], first.Value);
Assert.Equal(9, db.SortedSetLength(key));
var lasts = db.SortedSetPop(key, 2, Order.Descending);
Assert.Equal(2, lasts.Length);
Assert.Equal(entries[9], lasts[0]);
Assert.Equal(entries[8], lasts[1]);
Assert.Equal(7, db.SortedSetLength(key));
}
}
public async Task TestIncrByFloat()
{
using (var muxer = Create())
{
Skip.IfMissingFeature(muxer, nameof(RedisFeatures.IncrementFloat), r => r.IncrementFloat);
var conn = muxer.GetDatabase();
var key = Me();
_ = conn.KeyDeleteAsync(key).ForAwait();
var aTasks = new Task <double> [1000];
var bTasks = new Task <double> [1000];
for (int i = 1; i < 1001; i++)
{
aTasks[i - 1] = conn.HashIncrementAsync(key, "a", 1.0);
bTasks[i - 1] = conn.HashIncrementAsync(key, "b", -1.0);
}
await Task.WhenAll(bTasks).ForAwait();
for (int i = 1; i < 1001; i++)
{
Assert.Equal(i, aTasks[i - 1].Result);
Assert.Equal(-i, bTasks[i - 1].Result);
}
}
}
public async Task Basic()
{
var fromConfig = new ConfigurationOptions {
EndPoints = { { TestConfig.Current.SecureServer, TestConfig.Current.SecurePort } }, Password = TestConfig.Current.SecurePassword, AllowAdmin = true
};
var toConfig = new ConfigurationOptions {
EndPoints = { { TestConfig.Current.MasterServer, TestConfig.Current.MasterPort } }, AllowAdmin = true
};
using (var from = ConnectionMultiplexer.Connect(fromConfig))
using (var to = ConnectionMultiplexer.Connect(toConfig))
{
if (await IsWindows(from) || await IsWindows(to))
{
Skip.Inconclusive("'migrate' is unreliable on redis-64");
}
RedisKey key = Me();
var fromDb = from.GetDatabase();
var toDb = to.GetDatabase();
fromDb.KeyDelete(key, CommandFlags.FireAndForget);
toDb.KeyDelete(key, CommandFlags.FireAndForget);
fromDb.StringSet(key, "foo", flags: CommandFlags.FireAndForget);
var dest = to.GetEndPoints(true).Single();
fromDb.KeyMigrate(key, dest);
await Task.Delay(1000); // this is *meant* to be synchronous at the redis level, but
// we keep seeing it fail on the CI server where the key has *left* the origin, but
// has *not* yet arrived at the destination; adding a pause while we investigate with
// the redis folks
Assert.False(fromDb.KeyExists(key));
Assert.True(toDb.KeyExists(key));
string s = toDb.StringGet(key);
Assert.Equal("foo", s);
}
}
public async Task GetStats()
{
using (var conn = Create())
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Memory), r => r.Streams);
var server = conn.GetServer(conn.GetEndPoints()[0]);
var stats = server.MemoryStats();
Assert.Equal(ResultType.MultiBulk, stats.Type);
var parsed = stats.ToDictionary();
var alloc = parsed["total.allocated"];
Assert.Equal(ResultType.Integer, alloc.Type);
Assert.True(alloc.AsInt64() > 0);
stats = await server.MemoryStatsAsync();
Assert.Equal(ResultType.MultiBulk, stats.Type);
alloc = parsed["total.allocated"];
Assert.Equal(ResultType.Integer, alloc.Type);
Assert.True(alloc.AsInt64() > 0);
}
}
public void StreamConsumerGroupReadOnlyNewMessagesWithEmptyResponse()
{
var key = GetUniqueKey("group_read");
var groupName = "test_group";
using (var conn = Create())
{
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Streams), r => r.Streams);
var db = conn.GetDatabase();
// Create a stream
db.StreamAdd(key, "field1", "value1");
db.StreamAdd(key, "field2", "value2");
// Create a group.
db.StreamCreateConsumerGroup(key, groupName);
// Read, expect no messages
var entries = db.StreamReadGroup(key, groupName, "test_consumer", "0-0");
Assert.True(entries.Length == 0);
}
}
protected virtual ConnectionMultiplexer Create(
string clientName = null, int?syncTimeout = null, bool?allowAdmin = null, int?keepAlive = null,
int?connectTimeout = null, string password = null, string tieBreaker = null, TextWriter log = null,
bool fail = true, string[] disabledCommands = null, string[] enabledCommands = null,
bool checkConnect = true, bool pause = true, string failMessage = null,
string channelPrefix = null, bool useSharedSocketManager = true, Proxy?proxy = null)
{
if (pause)
{
Thread.Sleep(250); // get a lot of glitches when hammering new socket creations etc; pace it out a bit
}
string configuration = GetConfiguration();
var config = ConfigurationOptions.Parse(configuration);
if (disabledCommands != null && disabledCommands.Length != 0)
{
config.CommandMap = CommandMap.Create(new HashSet <string>(disabledCommands), false);
}
else if (enabledCommands != null && enabledCommands.Length != 0)
{
config.CommandMap = CommandMap.Create(new HashSet <string>(enabledCommands), true);
}
if (Debugger.IsAttached)
{
syncTimeout = int.MaxValue;
}
if (useSharedSocketManager)
{
config.SocketManager = socketManager;
}
if (channelPrefix != null)
{
config.ChannelPrefix = channelPrefix;
}
if (tieBreaker != null)
{
config.TieBreaker = tieBreaker;
}
if (password != null)
{
config.Password = string.IsNullOrEmpty(password) ? null : password;
}
if (clientName != null)
{
config.ClientName = clientName;
}
if (syncTimeout != null)
{
config.SyncTimeout = syncTimeout.Value;
}
if (allowAdmin != null)
{
config.AllowAdmin = allowAdmin.Value;
}
if (keepAlive != null)
{
config.KeepAlive = keepAlive.Value;
}
if (connectTimeout != null)
{
config.ConnectTimeout = connectTimeout.Value;
}
if (proxy != null)
{
config.Proxy = proxy.Value;
}
var watch = Stopwatch.StartNew();
var task = ConnectionMultiplexer.ConnectAsync(config, log ?? Writer);
if (!task.Wait(config.ConnectTimeout >= (int.MaxValue / 2) ? int.MaxValue : config.ConnectTimeout * 2))
{
task.ContinueWith(x =>
{
try
{
GC.KeepAlive(x.Exception);
}
catch
{ }
}, TaskContinuationOptions.OnlyOnFaulted);
throw new TimeoutException("Connect timeout");
}
watch.Stop();
if (Output == null)
{
Assert.True(false, "Failure: Be sure to call the TestBase constuctor like this: BasicOpsTests(ITestOutputHelper output) : base(output) { }");
}
Output.WriteLine("Connect took: " + watch.ElapsedMilliseconds + "ms");
var muxer = task.Result;
if (checkConnect && (muxer == null || !muxer.IsConnected))
{
// If fail is true, we throw.
Assert.False(fail, failMessage + "Server is not available");
Skip.Inconclusive(failMessage + "Server is not available");
}
muxer.InternalError += OnInternalError;
muxer.ConnectionFailed += OnConnectionFailed;
return(muxer);
}
public void ScanResume()
{
using (var conn = Create(allowAdmin: true))
{
// only goes up to 3.*, so...
Skip.IfMissingFeature(conn, nameof(RedisFeatures.Scan), x => x.Scan);
var dbId = TestConfig.GetDedicatedDB(conn);
var db = conn.GetDatabase(dbId);
var prefix = Me();
var server = GetServer(conn);
server.FlushDatabase(dbId);
int i;
for (i = 0; i < 100; i++)
{
db.StringSet(prefix + ":" + i, Guid.NewGuid().ToString());
}
var expected = new HashSet <string>();
long snapCursor = 0;
int snapOffset = 0, snapPageSize = 0;
i = 0;
var seq = server.Keys(dbId, prefix + ":*", pageSize: 15);
foreach (var key in seq)
{
if (i == 57)
{
snapCursor = ((IScanningCursor)seq).Cursor;
snapOffset = ((IScanningCursor)seq).PageOffset;
snapPageSize = ((IScanningCursor)seq).PageSize;
Log($"i: {i}, Cursor: {snapCursor}, Offset: {snapOffset}, PageSize: {snapPageSize}");
}
if (i >= 57)
{
expected.Add((string)key);
}
i++;
}
Log($"Expected: 43, Actual: {expected.Count}, Cursor: {snapCursor}, Offset: {snapOffset}, PageSize: {snapPageSize}");
Assert.Equal(43, expected.Count);
Assert.NotEqual(0, snapCursor);
Assert.Equal(15, snapPageSize);
// note: you might think that we can say "hmmm, 57 when using page-size 15 on an empty (flushed) db (so: no skipped keys); that'll be
// offset 12 in the 4th page; you'd be wrong, though; page size doesn't *actually* mean page size; it is a rough analogue for
// page size, with zero guarantees; in this particular test, the first page actually has 19 elements, for example. So: we cannot
// make the following assertion:
// Assert.Equal(12, snapOffset);
seq = server.Keys(dbId, prefix + ":*", pageSize: 15, cursor: snapCursor, pageOffset: snapOffset);
var seqCur = (IScanningCursor)seq;
Assert.Equal(snapCursor, seqCur.Cursor);
Assert.Equal(snapPageSize, seqCur.PageSize);
Assert.Equal(snapOffset, seqCur.PageOffset);
using (var iter = seq.GetEnumerator())
{
var iterCur = (IScanningCursor)iter;
Assert.Equal(snapCursor, iterCur.Cursor);
Assert.Equal(snapOffset, iterCur.PageOffset);
Assert.Equal(snapCursor, seqCur.Cursor);
Assert.Equal(snapOffset, seqCur.PageOffset);
Assert.True(iter.MoveNext());
Assert.Equal(snapCursor, iterCur.Cursor);
Assert.Equal(snapOffset, iterCur.PageOffset);
Assert.Equal(snapCursor, seqCur.Cursor);
Assert.Equal(snapOffset, seqCur.PageOffset);
Assert.True(iter.MoveNext());
Assert.Equal(snapCursor, iterCur.Cursor);
Assert.Equal(snapOffset + 1, iterCur.PageOffset);
Assert.Equal(snapCursor, seqCur.Cursor);
Assert.Equal(snapOffset + 1, seqCur.PageOffset);
}
int count = 0;
foreach (var key in seq)
{
expected.Remove((string)key);
count++;
}
Assert.Empty(expected);
Assert.Equal(43, count);
}
}
public async Task SubscriptionsSurviveMasterSwitchAsync(bool useSharedSocketManager)
{
if (RunningInCI)
{
Skip.Inconclusive("TODO: Fix race in broadcast reconfig a zero latency.");
}
using (var a = Create(allowAdmin: true, useSharedSocketManager: useSharedSocketManager))
using (var b = Create(allowAdmin: true, useSharedSocketManager: useSharedSocketManager))
{
RedisChannel channel = Me();
var subA = a.GetSubscriber();
var subB = b.GetSubscriber();
long masterChanged = 0, aCount = 0, bCount = 0;
a.ConfigurationChangedBroadcast += delegate
{
Output.WriteLine("A noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
b.ConfigurationChangedBroadcast += delegate
{
Output.WriteLine("B noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
subA.Subscribe(channel, (_, message) =>
{
Output.WriteLine("A got message: " + message);
Interlocked.Increment(ref aCount);
});
subB.Subscribe(channel, (_, message) =>
{
Output.WriteLine("B got message: " + message);
Interlocked.Increment(ref bCount);
});
Assert.False(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
Assert.True(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
Assert.True(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
var epA = subA.SubscribedEndpoint(channel);
var epB = subB.SubscribedEndpoint(channel);
Output.WriteLine("A: " + EndPointCollection.ToString(epA));
Output.WriteLine("B: " + EndPointCollection.ToString(epB));
subA.Publish(channel, "A1");
subB.Publish(channel, "B1");
subA.Ping();
subB.Ping();
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
Assert.Equal(0, Interlocked.Read(ref masterChanged));
try
{
Interlocked.Exchange(ref masterChanged, 0);
Interlocked.Exchange(ref aCount, 0);
Interlocked.Exchange(ref bCount, 0);
Output.WriteLine("Changing master...");
using (var sw = new StringWriter())
{
a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).MakeMaster(ReplicationChangeOptions.All, sw);
Output.WriteLine(sw.ToString());
}
await Task.Delay(5000).ForAwait();
subA.Ping();
subB.Ping();
Output.WriteLine("Pausing...");
Output.WriteLine("A " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Output.WriteLine("A " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
Output.WriteLine("B " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Output.WriteLine("B " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
Assert.True(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
Assert.True(b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
Output.WriteLine("Pause complete");
Output.WriteLine("A outstanding: " + a.GetCounters().TotalOutstanding);
Output.WriteLine("B outstanding: " + b.GetCounters().TotalOutstanding);
subA.Ping();
subB.Ping();
await Task.Delay(2000).ForAwait();
epA = subA.SubscribedEndpoint(channel);
epB = subB.SubscribedEndpoint(channel);
Output.WriteLine("A: " + EndPointCollection.ToString(epA));
Output.WriteLine("B: " + EndPointCollection.ToString(epB));
Output.WriteLine("A2 sent to: " + subA.Publish(channel, "A2"));
Output.WriteLine("B2 sent to: " + subB.Publish(channel, "B2"));
subA.Ping();
subB.Ping();
Output.WriteLine("Checking...");
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
// Expect 6, because a sees a, but b sees a and b due to replication
Assert.Equal(6, Interlocked.CompareExchange(ref masterChanged, 0, 0));
}
finally
{
Output.WriteLine("Restoring configuration...");
try
{
a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).MakeMaster(ReplicationChangeOptions.All);
await Task.Delay(1000).ForAwait();
}
catch { }
}
}
}
public async Task SubscriptionsSurviveMasterSwitchAsync()
{
void TopologyFail() => Skip.Inconclusive("Replication tolopogy change failed...and that's both inconsistent and not what we're testing.");
if (RunningInCI)
{
Skip.Inconclusive("TODO: Fix race in broadcast reconfig a zero latency.");
}
using (var a = Create(allowAdmin: true, shared: false))
using (var b = Create(allowAdmin: true, shared: false))
{
RedisChannel channel = Me();
Log("Using Channel: " + channel);
var subA = a.GetSubscriber();
var subB = b.GetSubscriber();
long masterChanged = 0, aCount = 0, bCount = 0;
a.ConfigurationChangedBroadcast += delegate
{
Log("A noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
b.ConfigurationChangedBroadcast += delegate
{
Log("B noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
subA.Subscribe(channel, (_, message) =>
{
Log("A got message: " + message);
Interlocked.Increment(ref aCount);
});
subB.Subscribe(channel, (_, message) =>
{
Log("B got message: " + message);
Interlocked.Increment(ref bCount);
});
Assert.False(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
if (!a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave)
{
TopologyFail();
}
Assert.True(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
Assert.True(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
Log("Failover 1 Complete");
var epA = subA.SubscribedEndpoint(channel);
var epB = subB.SubscribedEndpoint(channel);
Log(" A: " + EndPointCollection.ToString(epA));
Log(" B: " + EndPointCollection.ToString(epB));
subA.Publish(channel, "A1");
subB.Publish(channel, "B1");
Log(" SubA ping: " + subA.Ping());
Log(" SubB ping: " + subB.Ping());
// If redis is under load due to this suite, it may take a moment to send across.
await UntilCondition(TimeSpan.FromSeconds(5), () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
Assert.Equal(0, Interlocked.Read(ref masterChanged));
try
{
Interlocked.Exchange(ref masterChanged, 0);
Interlocked.Exchange(ref aCount, 0);
Interlocked.Exchange(ref bCount, 0);
Log("Changing master...");
using (var sw = new StringWriter())
{
a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).MakeMaster(ReplicationChangeOptions.All, sw);
Log(sw.ToString());
}
Log("Waiting for connection B to detect...");
await UntilCondition(TimeSpan.FromSeconds(10), () => b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave).ForAwait();
subA.Ping();
subB.Ping();
Log("Falover 2 Attempted. Pausing...");
Log(" A " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Log(" A " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
Log(" B " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Log(" B " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
if (!a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave)
{
TopologyFail();
}
Log("Falover 2 Complete.");
Assert.True(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
await UntilCondition(TimeSpan.FromSeconds(10), () => b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave).ForAwait();
var sanityCheck = b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave;
if (!sanityCheck)
{
Log("FAILURE: B has not detected the topology change.");
foreach (var server in b.GetServerSnapshot().ToArray())
{
Log(" Server" + server.EndPoint);
Log(" State: " + server.ConnectionState);
Log(" IsMaster: " + !server.IsSlave);
Log(" Type: " + server.ServerType);
}
//Skip.Inconclusive("Not enough latency.");
}
Assert.True(sanityCheck, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
Log("Pause complete");
Log(" A outstanding: " + a.GetCounters().TotalOutstanding);
Log(" B outstanding: " + b.GetCounters().TotalOutstanding);
subA.Ping();
subB.Ping();
await Task.Delay(5000).ForAwait();
epA = subA.SubscribedEndpoint(channel);
epB = subB.SubscribedEndpoint(channel);
Log("Subscription complete");
Log(" A: " + EndPointCollection.ToString(epA));
Log(" B: " + EndPointCollection.ToString(epB));
var aSentTo = subA.Publish(channel, "A2");
var bSentTo = subB.Publish(channel, "B2");
Log(" A2 sent to: " + aSentTo);
Log(" B2 sent to: " + bSentTo);
subA.Ping();
subB.Ping();
Log("Ping Complete. Checking...");
await UntilCondition(TimeSpan.FromSeconds(10), () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
Log("Counts so far:");
Log(" aCount: " + Interlocked.Read(ref aCount));
Log(" bCount: " + Interlocked.Read(ref bCount));
Log(" masterChanged: " + Interlocked.Read(ref masterChanged));
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
// Expect 10, because a sees a, but b sees a and b due to replication
Assert.Equal(10, Interlocked.CompareExchange(ref masterChanged, 0, 0));
}
catch
{
LogNoTime("");
Log("ERROR: Something went bad - see above! Roooooolling back. Back it up. Baaaaaack it on up.");
LogNoTime("");
throw;
}
finally
{
Log("Restoring configuration...");
try
{
a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).MakeMaster(ReplicationChangeOptions.All);
await Task.Delay(1000).ForAwait();
}
catch { }
}
}
}
public async Task ConnectToSSLServer(bool useSsl, bool specifyHost)
{
var server = TestConfig.Current.SslServer;
int? port = TestConfig.Current.SslPort;
string password = "";
bool isAzure = false;
if (string.IsNullOrWhiteSpace(server) && useSsl)
{
// we can bounce it past azure instead?
server = TestConfig.Current.AzureCacheServer;
password = TestConfig.Current.AzureCachePassword;
port = null;
isAzure = true;
}
Skip.IfNoConfig(nameof(TestConfig.Config.SslServer), server);
var config = new ConfigurationOptions
{
AllowAdmin = true,
SyncTimeout = Debugger.IsAttached ? int.MaxValue : 5000,
Password = password,
};
var map = new Dictionary <string, string>
{
["config"] = null // don't rely on config working
};
if (!isAzure)
{
map["cluster"] = null;
}
config.CommandMap = CommandMap.Create(map);
if (port != null)
{
config.EndPoints.Add(server, port.Value);
}
else
{
config.EndPoints.Add(server);
}
if (useSsl)
{
config.Ssl = useSsl;
if (specifyHost)
{
config.SslHost = server;
}
config.CertificateValidation += (sender, cert, chain, errors) =>
{
Log("errors: " + errors);
Log("cert issued to: " + cert.Subject);
return(true); // fingers in ears, pretend we don't know this is wrong
};
}
var configString = config.ToString();
Log("config: " + configString);
var clone = ConfigurationOptions.Parse(configString);
Assert.Equal(configString, clone.ToString());
using (var log = new StringWriter())
using (var muxer = ConnectionMultiplexer.Connect(config, log))
{
Log("Connect log:");
lock (log)
{
Log(log.ToString());
}
Log("====");
muxer.ConnectionFailed += OnConnectionFailed;
muxer.InternalError += OnInternalError;
var db = muxer.GetDatabase();
await db.PingAsync().ForAwait();
using (var file = File.Create("ssl-" + useSsl + "-" + specifyHost + ".zip"))
{
muxer.ExportConfiguration(file);
}
RedisKey key = "SE.Redis";
const int AsyncLoop = 2000;
// perf; async
await db.KeyDeleteAsync(key).ForAwait();
var watch = Stopwatch.StartNew();
for (int i = 0; i < AsyncLoop; i++)
{
try
{
await db.StringIncrementAsync(key, flags : CommandFlags.FireAndForget).ForAwait();
}
catch (Exception ex)
{
Log($"Failure on i={i}: {ex.Message}");
throw;
}
}
// need to do this inside the timer to measure the TTLB
long value = (long)await db.StringGetAsync(key).ForAwait();
watch.Stop();
Assert.Equal(AsyncLoop, value);
Log("F&F: {0} INCR, {1:###,##0}ms, {2} ops/s; final value: {3}",
AsyncLoop,
watch.ElapsedMilliseconds,
(long)(AsyncLoop / watch.Elapsed.TotalSeconds),
value);
// perf: sync/multi-threaded
// TestConcurrent(db, key, 30, 10);
//TestConcurrent(db, key, 30, 20);
//TestConcurrent(db, key, 30, 30);
//TestConcurrent(db, key, 30, 40);
//TestConcurrent(db, key, 30, 50);
}
}
public static ConnectionMultiplexer CreateDefault(
TextWriter output,
string clientName = null, int?syncTimeout = null, bool?allowAdmin = null, int?keepAlive = null,
int?connectTimeout = null, string password = null, string tieBreaker = null, TextWriter log = null,
bool fail = true, string[] disabledCommands = null, string[] enabledCommands = null,
bool checkConnect = true, string failMessage = null,
string channelPrefix = null, Proxy?proxy = null,
string configuration = null, bool logTransactionData = true,
int?defaultDatabase = null,
[CallerMemberName] string caller = null)
{
StringWriter localLog = null;
if (log == null)
{
log = localLog = new StringWriter();
}
try
{
var config = ConfigurationOptions.Parse(configuration);
if (disabledCommands != null && disabledCommands.Length != 0)
{
config.CommandMap = CommandMap.Create(new HashSet <string>(disabledCommands), false);
}
else if (enabledCommands != null && enabledCommands.Length != 0)
{
config.CommandMap = CommandMap.Create(new HashSet <string>(enabledCommands), true);
}
if (Debugger.IsAttached)
{
syncTimeout = int.MaxValue;
}
if (channelPrefix != null)
{
config.ChannelPrefix = channelPrefix;
}
if (tieBreaker != null)
{
config.TieBreaker = tieBreaker;
}
if (password != null)
{
config.Password = string.IsNullOrEmpty(password) ? null : password;
}
if (clientName != null)
{
config.ClientName = clientName;
}
else if (caller != null)
{
config.ClientName = caller;
}
if (syncTimeout != null)
{
config.SyncTimeout = syncTimeout.Value;
}
if (allowAdmin != null)
{
config.AllowAdmin = allowAdmin.Value;
}
if (keepAlive != null)
{
config.KeepAlive = keepAlive.Value;
}
if (connectTimeout != null)
{
config.ConnectTimeout = connectTimeout.Value;
}
if (proxy != null)
{
config.Proxy = proxy.Value;
}
if (defaultDatabase != null)
{
config.DefaultDatabase = defaultDatabase.Value;
}
var watch = Stopwatch.StartNew();
var task = ConnectionMultiplexer.ConnectAsync(config, log);
if (!task.Wait(config.ConnectTimeout >= (int.MaxValue / 2) ? int.MaxValue : config.ConnectTimeout * 2))
{
task.ContinueWith(x =>
{
try
{
GC.KeepAlive(x.Exception);
}
catch { /* No boom */ }
}, TaskContinuationOptions.OnlyOnFaulted);
throw new TimeoutException("Connect timeout");
}
watch.Stop();
if (output != null)
{
Log(output, "Connect took: " + watch.ElapsedMilliseconds + "ms");
}
var muxer = task.Result;
if (checkConnect && (muxer == null || !muxer.IsConnected))
{
// If fail is true, we throw.
Assert.False(fail, failMessage + "Server is not available");
Skip.Inconclusive(failMessage + "Server is not available");
}
if (output != null)
{
muxer.MessageFaulted += (msg, ex, origin) =>
{
output?.WriteLine($"Faulted from '{origin}': '{msg}' - '{(ex == null ? "(null)" : ex.Message)}'");
if (ex != null && ex.Data.Contains("got"))
{
output?.WriteLine($"Got: '{ex.Data["got"]}'");
}
};
muxer.Connecting += (e, t) => output?.WriteLine($"Connecting to {Format.ToString(e)} as {t}");
if (logTransactionData)
{
muxer.TransactionLog += msg => output?.WriteLine("tran: " + msg);
}
muxer.InfoMessage += msg => output?.WriteLine(msg);
muxer.Resurrecting += (e, t) => output?.WriteLine($"Resurrecting {Format.ToString(e)} as {t}");
muxer.Closing += complete => output?.WriteLine(complete ? "Closed" : "Closing...");
}
return(muxer);
}
catch
{
if (localLog != null)
{
output?.WriteLine(localLog.ToString());
}
throw;
}
}
public void RedisLabsEnvironmentVariableClientCertificate(bool setEnv)
{
try
{
Skip.IfNoConfig(nameof(TestConfig.Config.RedisLabsSslServer), TestConfig.Current.RedisLabsSslServer);
Skip.IfNoConfig(nameof(TestConfig.Config.RedisLabsPfxPath), TestConfig.Current.RedisLabsPfxPath);
if (setEnv)
{
Environment.SetEnvironmentVariable("SERedis_ClientCertPfxPath", TestConfig.Current.RedisLabsPfxPath);
Environment.SetEnvironmentVariable("SERedis_IssuerCertPath", "redislabs_ca.pem");
// check env worked
Assert.Equal(TestConfig.Current.RedisLabsPfxPath, Environment.GetEnvironmentVariable("SERedis_ClientCertPfxPath"));
Assert.Equal("redislabs_ca.pem", Environment.GetEnvironmentVariable("SERedis_IssuerCertPath"));
}
int timeout = 5000;
if (Debugger.IsAttached)
{
timeout *= 100;
}
var options = new ConfigurationOptions
{
EndPoints = { { TestConfig.Current.RedisLabsSslServer, TestConfig.Current.RedisLabsSslPort } },
ConnectTimeout = timeout,
AllowAdmin = true,
CommandMap = CommandMap.Create(new HashSet <string> {
"subscribe", "unsubscribe", "cluster"
}, false)
};
if (!Directory.Exists(Me()))
{
Directory.CreateDirectory(Me());
}
#if LOGOUTPUT
ConnectionMultiplexer.EchoPath = Me();
#endif
options.Ssl = true;
RedisKey key = Me();
using (var conn = ConnectionMultiplexer.Connect(options))
{
if (!setEnv)
{
Assert.True(false, "Could not set environment");
}
var db = conn.GetDatabase();
db.KeyDelete(key, CommandFlags.FireAndForget);
string s = db.StringGet(key);
Assert.Null(s);
db.StringSet(key, "abc");
s = db.StringGet(key);
Assert.Equal("abc", s);
var latency = db.Ping();
Log("RedisLabs latency: {0:###,##0.##}ms", latency.TotalMilliseconds);
using (var file = File.Create("RedisLabs.zip"))
{
conn.ExportConfiguration(file);
}
}
}
catch (RedisConnectionException ex)
{
if (setEnv || ex.FailureType != ConnectionFailureType.UnableToConnect)
{
throw;
}
}
finally
{
Environment.SetEnvironmentVariable("SERedis_ClientCertPfxPath", null);
}
}
public void ConnectToSSLServer(bool useSsl, bool specifyHost)
{
Skip.IfNoConfig(nameof(TestConfig.Config.SslServer), TestConfig.Current.SslServer);
var config = new ConfigurationOptions
{
CommandMap = CommandMap.Create( // looks like "config" is disabled
new Dictionary <string, string>
{
["config"] = null,
["cluster"] = null
}
),
EndPoints = { { TestConfig.Current.SslServer, TestConfig.Current.SslPort } },
AllowAdmin = true,
SyncTimeout = Debugger.IsAttached ? int.MaxValue : 5000
};
if (useSsl)
{
config.Ssl = useSsl;
if (specifyHost)
{
config.SslHost = TestConfig.Current.SslServer;
}
config.CertificateValidation += (sender, cert, chain, errors) =>
{
Output.WriteLine("errors: " + errors);
Output.WriteLine("cert issued to: " + cert.Subject);
return(true); // fingers in ears, pretend we don't know this is wrong
};
}
var configString = config.ToString();
Output.WriteLine("config: " + configString);
var clone = ConfigurationOptions.Parse(configString);
Assert.Equal(configString, clone.ToString());
using (var log = new StringWriter())
using (var muxer = ConnectionMultiplexer.Connect(config, log))
{
Output.WriteLine("Connect log:");
Output.WriteLine(log.ToString());
Output.WriteLine("====");
muxer.ConnectionFailed += OnConnectionFailed;
muxer.InternalError += OnInternalError;
var db = muxer.GetDatabase();
db.Ping();
using (var file = File.Create("ssl-" + useSsl + "-" + specifyHost + ".zip"))
{
muxer.ExportConfiguration(file);
}
RedisKey key = "SE.Redis";
const int AsyncLoop = 2000;
// perf; async
db.KeyDelete(key, CommandFlags.FireAndForget);
var watch = Stopwatch.StartNew();
for (int i = 0; i < AsyncLoop; i++)
{
db.StringIncrement(key, flags: CommandFlags.FireAndForget);
}
// need to do this inside the timer to measure the TTLB
long value = (long)db.StringGet(key);
watch.Stop();
Assert.Equal(AsyncLoop, value);
Output.WriteLine("F&F: {0} INCR, {1:###,##0}ms, {2} ops/s; final value: {3}",
AsyncLoop,
(long)watch.ElapsedMilliseconds,
(long)(AsyncLoop / watch.Elapsed.TotalSeconds),
value);
// perf: sync/multi-threaded
TestConcurrent(db, key, 30, 10);
//TestConcurrent(db, key, 30, 20);
//TestConcurrent(db, key, 30, 30);
//TestConcurrent(db, key, 30, 40);
//TestConcurrent(db, key, 30, 50);
}
}
public async Task DeslaveGoesToPrimary()
{
ConfigurationOptions config = GetMasterSlaveConfig();
using (var conn = ConnectionMultiplexer.Connect(config))
{
var primary = conn.GetServer(TestConfig.Current.FailoverMasterServerAndPort);
var secondary = conn.GetServer(TestConfig.Current.FailoverSlaveServerAndPort);
primary.Ping();
secondary.Ping();
primary.MakeMaster(ReplicationChangeOptions.SetTiebreaker);
secondary.MakeMaster(ReplicationChangeOptions.None);
await Task.Delay(100).ConfigureAwait(false);
primary.Ping();
secondary.Ping();
using (var writer = new StringWriter())
{
conn.Configure(writer);
string log = writer.ToString();
Writer.WriteLine(log);
bool isUnanimous = log.Contains("tie-break is unanimous at " + TestConfig.Current.FailoverMasterServerAndPort);
if (!isUnanimous)
{
Skip.Inconclusive("this is timing sensitive; unable to verify this time");
}
}
// k, so we know everyone loves 6379; is that what we get?
var db = conn.GetDatabase();
RedisKey key = Me();
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferMaster));
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.DemandMaster));
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferSlave));
var ex = Assert.Throws <RedisConnectionException>(() => db.IdentifyEndpoint(key, CommandFlags.DemandSlave));
Assert.StartsWith("No connection is active/available to service this operation: EXISTS " + Me(), ex.Message);
Writer.WriteLine("Invoking MakeMaster()...");
primary.MakeMaster(ReplicationChangeOptions.Broadcast | ReplicationChangeOptions.EnslaveSubordinates | ReplicationChangeOptions.SetTiebreaker, Writer);
Writer.WriteLine("Finished MakeMaster() call.");
await Task.Delay(100).ConfigureAwait(false);
Writer.WriteLine("Invoking Ping() (post-master)");
primary.Ping();
secondary.Ping();
Writer.WriteLine("Finished Ping() (post-master)");
Assert.True(primary.IsConnected, $"{primary.EndPoint} is not connected.");
Assert.True(secondary.IsConnected, $"{secondary.EndPoint} is not connected.");
Writer.WriteLine($"{primary.EndPoint}: {primary.ServerType}, Mode: {(primary.IsSlave ? "Slave" : "Master")}");
Writer.WriteLine($"{secondary.EndPoint}: {secondary.ServerType}, Mode: {(secondary.IsSlave ? "Slave" : "Master")}");
// Create a separate multiplexer with a valid view of the world to distinguish between failures of
// server topology changes from failures to recognize those changes
Writer.WriteLine("Connecting to secondary validation connection.");
using (var conn2 = ConnectionMultiplexer.Connect(config))
{
var primary2 = conn2.GetServer(TestConfig.Current.FailoverMasterServerAndPort);
var secondary2 = conn2.GetServer(TestConfig.Current.FailoverSlaveServerAndPort);
Writer.WriteLine($"Check: {primary2.EndPoint}: {primary2.ServerType}, Mode: {(primary2.IsSlave ? "Slave" : "Master")}");
Writer.WriteLine($"Check: {secondary2.EndPoint}: {secondary2.ServerType}, Mode: {(secondary2.IsSlave ? "Slave" : "Master")}");
Assert.False(primary2.IsSlave, $"{primary2.EndPoint} should be a master (verification connection).");
Assert.True(secondary2.IsSlave, $"{secondary2.EndPoint} should be a slave (verification connection).");
var db2 = conn2.GetDatabase();
Assert.Equal(primary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.PreferMaster));
Assert.Equal(primary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.DemandMaster));
Assert.Equal(secondary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.PreferSlave));
Assert.Equal(secondary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.DemandSlave));
}
await UntilCondition(TimeSpan.FromSeconds(20), () => !primary.IsSlave && secondary.IsSlave);
Assert.False(primary.IsSlave, $"{primary.EndPoint} should be a master.");
Assert.True(secondary.IsSlave, $"{secondary.EndPoint} should be a slave.");
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferMaster));
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.DemandMaster));
Assert.Equal(secondary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferSlave));
Assert.Equal(secondary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.DemandSlave));
}
}
static void TopologyFail() => Skip.Inconclusive("Replication topology change failed...and that's both inconsistent and not what we're testing.");
public async Task SubscriptionsSurviveMasterSwitchAsync()
{
void TopologyFail() => Skip.Inconclusive("Replication tolopogy change failed...and that's both inconsistent and not what we're testing.");
if (RunningInCI)
{
Skip.Inconclusive("TODO: Fix race in broadcast reconfig a zero latency.");
}
using (var a = Create(allowAdmin: true, shared: false))
using (var b = Create(allowAdmin: true, shared: false))
{
RedisChannel channel = Me();
var subA = a.GetSubscriber();
var subB = b.GetSubscriber();
long masterChanged = 0, aCount = 0, bCount = 0;
a.ConfigurationChangedBroadcast += delegate
{
Log("A noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
b.ConfigurationChangedBroadcast += delegate
{
Log("B noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
subA.Subscribe(channel, (_, message) =>
{
Log("A got message: " + message);
Interlocked.Increment(ref aCount);
});
subB.Subscribe(channel, (_, message) =>
{
Log("B got message: " + message);
Interlocked.Increment(ref bCount);
});
Assert.False(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
if (!a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave)
{
TopologyFail();
}
Assert.True(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
Assert.True(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
var epA = subA.SubscribedEndpoint(channel);
var epB = subB.SubscribedEndpoint(channel);
Log("A: " + EndPointCollection.ToString(epA));
Log("B: " + EndPointCollection.ToString(epB));
subA.Publish(channel, "A1");
subB.Publish(channel, "B1");
Log("SubA ping: " + subA.Ping());
Log("SubB ping: " + subB.Ping());
// If redis is under load due to this suite, it may take a moment to send across.
await UntilCondition(5000, () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
Assert.Equal(0, Interlocked.Read(ref masterChanged));
try
{
Interlocked.Exchange(ref masterChanged, 0);
Interlocked.Exchange(ref aCount, 0);
Interlocked.Exchange(ref bCount, 0);
Log("Changing master...");
using (var sw = new StringWriter())
{
a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).MakeMaster(ReplicationChangeOptions.All, sw);
Log(sw.ToString());
}
await UntilCondition(3000, () => b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave).ForAwait();
subA.Ping();
subB.Ping();
Log("Pausing...");
Log("A " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Log("A " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
Log("B " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Log("B " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
if (!a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave)
{
TopologyFail();
}
Assert.True(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
var sanityCheck = b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave;
if (!sanityCheck)
{
Skip.Inconclusive("Not enough latency.");
}
Assert.True(sanityCheck, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
Log("Pause complete");
Log("A outstanding: " + a.GetCounters().TotalOutstanding);
Log("B outstanding: " + b.GetCounters().TotalOutstanding);
subA.Ping();
subB.Ping();
await Task.Delay(1000).ForAwait();
epA = subA.SubscribedEndpoint(channel);
epB = subB.SubscribedEndpoint(channel);
Log("A: " + EndPointCollection.ToString(epA));
Log("B: " + EndPointCollection.ToString(epB));
Log("A2 sent to: " + subA.Publish(channel, "A2"));
Log("B2 sent to: " + subB.Publish(channel, "B2"));
subA.Ping();
subB.Ping();
Log("Checking...");
await UntilCondition(10000, () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
// Expect 6, because a sees a, but b sees a and b due to replication
Assert.Equal(6, Interlocked.CompareExchange(ref masterChanged, 0, 0));
}
finally
{
Log("Restoring configuration...");
try
{
a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).MakeMaster(ReplicationChangeOptions.All);
await Task.Delay(1000).ForAwait();
}
catch { }
}
}
}
public SentinelBase(ITestOutputHelper output) : base(output)
{
Skip.IfNoConfig(nameof(TestConfig.Config.SentinelServer), TestConfig.Current.SentinelServer);
Skip.IfNoConfig(nameof(TestConfig.Config.SentinelSeviceName), TestConfig.Current.SentinelSeviceName);
}
