public void MuxerIsCollected()
{
#if DEBUG
Skip.Inconclusive("Only predictable in release builds");
#endif
// this is more nuanced than it looks; multiple sockets with
// async callbacks, plus a heartbeat on a timer
// deliberately not "using" - we *want* to leak this
var muxer = Create();
muxer.GetDatabase().Ping(); // smoke-test
ForceGC();
//#if DEBUG // this counter only exists in debug
// int before = ConnectionMultiplexer.CollectedWithoutDispose;
//#endif
var wr = new WeakReference(muxer);
muxer = null;
ForceGC();
Thread.Sleep(2000); // GC is twitchy
ForceGC();
// should be collectable
Assert.Null(wr.Target);
//#if DEBUG // this counter only exists in debug
// int after = ConnectionMultiplexer.CollectedWithoutDispose;
// Assert.Equal(before + 1, after);
//#endif
}
public void TestBasicEnvoyConnection()
{
var sb = new StringBuilder();
Writer.EchoTo(sb);
try
{
using (var muxer = Create(configuration: GetConfiguration(), keepAlive: 1, connectTimeout: 2000, allowAdmin: true, shared: false, proxy: Proxy.Envoyproxy, log: Writer))
{
var db = muxer.GetDatabase();
const string key = "foobar";
const string value = "barfoo";
db.StringSet(key, value);
var expectedVal = db.StringGet(key);
Assert.Equal(expectedVal, value);
}
}
catch (TimeoutException ex) when(ex.Message == "Connect timeout" || sb.ToString().Contains("Returned, but incorrectly"))
{
Skip.Inconclusive("Envoy server not found.");
}
catch (AggregateException)
{
Skip.Inconclusive("Envoy server not found.");
}
catch (RedisConnectionException) when(sb.ToString().Contains("It was not possible to connect to the redis server(s)"))
{
Skip.Inconclusive("Envoy server not found.");
}
}
public void Teardown()
{
int sharedFails;
lock (sharedFailCount)
{
sharedFails = sharedFailCount.Value;
sharedFailCount.Value = 0;
}
if (expectedFailCount >= 0 && (sharedFails + privateFailCount) != expectedFailCount)
{
lock (privateExceptions)
{
foreach (var item in privateExceptions.Take(5))
{
LogNoTime(item);
}
}
lock (backgroundExceptions)
{
foreach (var item in backgroundExceptions.Take(5))
{
LogNoTime(item);
}
}
Skip.Inconclusive($"There were {privateFailCount} private and {sharedFailCount.Value} ambient exceptions; expected {expectedFailCount}.");
}
Log($"Service Counts: (Scheduler) Queue: {SocketManager.Shared?.SchedulerPool?.TotalServicedByQueue.ToString()}, Pool: {SocketManager.Shared?.SchedulerPool?.TotalServicedByPool.ToString()}");
}
public async Task AuthenticationFailureError()
{
Skip.IfNoConfig(nameof(TestConfig.Config.AzureCacheServer), TestConfig.Current.AzureCacheServer);
var options = new ConfigurationOptions();
options.EndPoints.Add(TestConfig.Current.AzureCacheServer);
options.Ssl = true;
options.Password = "";
options.AbortOnConnectFail = false;
options.CertificateValidation += SSL.ShowCertFailures(Writer);
using (var muxer = ConnectionMultiplexer.Connect(options))
{
muxer.ConnectionFailed += (sender, e) =>
{
if (e.FailureType == ConnectionFailureType.SocketFailure)
{
Skip.Inconclusive("socket fail"); // this is OK too
}
Assert.Equal(ConnectionFailureType.AuthenticationFailure, e.FailureType);
};
var ex = Assert.Throws <RedisConnectionException>(() => muxer.GetDatabase().Ping());
Assert.NotNull(ex.InnerException);
var rde = Assert.IsType <RedisConnectionException>(ex.InnerException);
Assert.Equal(CommandStatus.WaitingToBeSent, ex.CommandStatus);
Assert.Equal(ConnectionFailureType.AuthenticationFailure, rde.FailureType);
Assert.Equal("Error: NOAUTH Authentication required. Verify if the Redis password provided is correct.", rde.InnerException.Message);
//wait for a second for connectionfailed event to fire
await Task.Delay(1000).ForAwait();
}
}
public async Task Basic()
{
var fromConfig = new ConfigurationOptions { EndPoints = { { TestConfig.Current.SecureServer, TestConfig.Current.SecurePort } }, Password = TestConfig.Current.SecurePassword, AllowAdmin = true };
var toConfig = new ConfigurationOptions { EndPoints = { { TestConfig.Current.MasterServer, TestConfig.Current.MasterPort } }, AllowAdmin = true };
using (var from = ConnectionMultiplexer.Connect(fromConfig, Writer))
using (var to = ConnectionMultiplexer.Connect(toConfig, Writer))
{
if (await IsWindows(from) || await IsWindows(to))
Skip.Inconclusive("'migrate' is unreliable on redis-64");
RedisKey key = Me();
var fromDb = from.GetDatabase();
var toDb = to.GetDatabase();
fromDb.KeyDelete(key, CommandFlags.FireAndForget);
toDb.KeyDelete(key, CommandFlags.FireAndForget);
fromDb.StringSet(key, "foo", flags: CommandFlags.FireAndForget);
var dest = to.GetEndPoints(true).Single();
Log("Migrating key...");
fromDb.KeyMigrate(key, dest, migrateOptions: MigrateOptions.Replace);
Log("Migration command complete");
// this is *meant* to be synchronous at the redis level, but
// we keep seeing it fail on the CI server where the key has *left* the origin, but
// has *not* yet arrived at the destination; adding a pause while we investigate with
// the redis folks
await UntilCondition(TimeSpan.FromSeconds(15), () => !fromDb.KeyExists(key) && toDb.KeyExists(key));
Assert.False(fromDb.KeyExists(key), "Exists at source");
Assert.True(toDb.KeyExists(key), "Exists at destination");
string s = toDb.StringGet(key);
Assert.Equal("foo", s);
}
}
public void TransactionWithMultiServerKeys()
{
var ex = Assert.Throws <RedisCommandException>(() =>
{
using (var muxer = Create())
{
// connect
var cluster = muxer.GetDatabase();
var anyServer = muxer.GetServer(muxer.GetEndPoints()[0]);
anyServer.Ping();
Assert.Equal(ServerType.Cluster, anyServer.ServerType);
var config = anyServer.ClusterConfiguration;
Assert.NotNull(config);
// invent 2 keys that we believe are served by different nodes
string x = Guid.NewGuid().ToString(), y;
var xNode = config.GetBySlot(x);
int abort = 1000;
do
{
y = Guid.NewGuid().ToString();
} while (--abort > 0 && config.GetBySlot(y) == xNode);
if (abort == 0)
{
Skip.Inconclusive("failed to find a different node to use");
}
var yNode = config.GetBySlot(y);
Output.WriteLine("x={0}, served by {1}", x, xNode.NodeId);
Output.WriteLine("y={0}, served by {1}", y, yNode.NodeId);
Assert.NotEqual(xNode.NodeId, yNode.NodeId);
// wipe those keys
cluster.KeyDelete(x, CommandFlags.FireAndForget);
cluster.KeyDelete(y, CommandFlags.FireAndForget);
// create a transaction that attempts to assign both keys
var tran = cluster.CreateTransaction();
tran.AddCondition(Condition.KeyNotExists(x));
tran.AddCondition(Condition.KeyNotExists(y));
var setX = tran.StringSetAsync(x, "x-val");
var setY = tran.StringSetAsync(y, "y-val");
bool success = tran.Execute();
Assert.True(false, "Expected single-slot rules to apply");
// the rest no longer applies while we are following single-slot rules
//// check that everything was aborted
//Assert.False(success, "tran aborted");
//Assert.True(setX.IsCanceled, "set x cancelled");
//Assert.True(setY.IsCanceled, "set y cancelled");
//var existsX = cluster.KeyExistsAsync(x);
//var existsY = cluster.KeyExistsAsync(y);
//Assert.False(cluster.Wait(existsX), "x exists");
//Assert.False(cluster.Wait(existsY), "y exists");
}
});
Assert.Equal("Multi-key operations must involve a single slot; keys can use 'hash tags' to help this, i.e. '{/users/12345}/account' and '{/users/12345}/contacts' will always be in the same slot", ex.Message);
}
public async Task Exec()
{
Skip.Inconclusive("need to think about CompletedSynchronously");
using (var conn = Create())
{
var parsed = ConfigurationOptions.Parse(conn.Configuration);
Assert.Equal(2, parsed.ConfigCheckSeconds);
var before = conn.GetCounters();
await Task.Delay(7000).ForAwait();
var after = conn.GetCounters();
int done = (int)(after.Interactive.CompletedSynchronously - before.Interactive.CompletedSynchronously);
Assert.True(done >= 2, $"expected >=2, got {done}");
}
}
public async Task ConfigVerifyReceiveConfigChangeBroadcast()
{
var config = GetConfiguration();
using (var sender = Create(allowAdmin: true))
using (var receiver = Create(syncTimeout: 2000))
{
int total = 0;
receiver.ConfigurationChangedBroadcast += (s, a) =>
{
Log("Config changed: " + (a.EndPoint == null ? "(none)" : a.EndPoint.ToString()));
Interlocked.Increment(ref total);
};
// send a reconfigure/reconnect message
long count = sender.PublishReconfigure();
GetServer(receiver).Ping();
GetServer(receiver).Ping();
await Task.Delay(100).ConfigureAwait(false);
Assert.True(count == -1 || count >= 2, "subscribers");
Assert.True(Interlocked.CompareExchange(ref total, 0, 0) >= 1, "total (1st)");
Interlocked.Exchange(ref total, 0);
// and send a second time via a re-master operation
var server = GetServer(sender);
if (server.IsSlave)
{
Skip.Inconclusive("didn't expect a slave");
}
server.MakeMaster(ReplicationChangeOptions.Broadcast);
await Task.Delay(100).ConfigureAwait(false);
GetServer(receiver).Ping();
GetServer(receiver).Ping();
Assert.True(Interlocked.CompareExchange(ref total, 0, 0) >= 1, "total (2nd)");
}
}
public async Task SubscriptionsSurviveMasterSwitchAsync()
{
void TopologyFail() => Skip.Inconclusive("Replication tolopogy change failed...and that's both inconsistent and not what we're testing.");
if (RunningInCI)
{
Skip.Inconclusive("TODO: Fix race in broadcast reconfig a zero latency.");
}
using (var a = Create(allowAdmin: true, shared: false))
using (var b = Create(allowAdmin: true, shared: false))
{
RedisChannel channel = Me();
Log("Using Channel: " + channel);
var subA = a.GetSubscriber();
var subB = b.GetSubscriber();
long masterChanged = 0, aCount = 0, bCount = 0;
a.ConfigurationChangedBroadcast += delegate
{
Log("A noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
b.ConfigurationChangedBroadcast += delegate
{
Log("B noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
subA.Subscribe(channel, (_, message) =>
{
Log("A got message: " + message);
Interlocked.Increment(ref aCount);
});
subB.Subscribe(channel, (_, message) =>
{
Log("B got message: " + message);
Interlocked.Increment(ref bCount);
});
Assert.False(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
if (!a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave)
{
TopologyFail();
}
Assert.True(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
Assert.True(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
Log("Failover 1 Complete");
var epA = subA.SubscribedEndpoint(channel);
var epB = subB.SubscribedEndpoint(channel);
Log(" A: " + EndPointCollection.ToString(epA));
Log(" B: " + EndPointCollection.ToString(epB));
subA.Publish(channel, "A1");
subB.Publish(channel, "B1");
Log(" SubA ping: " + subA.Ping());
Log(" SubB ping: " + subB.Ping());
// If redis is under load due to this suite, it may take a moment to send across.
await UntilCondition(TimeSpan.FromSeconds(5), () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
Assert.Equal(0, Interlocked.Read(ref masterChanged));
try
{
Interlocked.Exchange(ref masterChanged, 0);
Interlocked.Exchange(ref aCount, 0);
Interlocked.Exchange(ref bCount, 0);
Log("Changing master...");
using (var sw = new StringWriter())
{
a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).MakeMaster(ReplicationChangeOptions.All, sw);
Log(sw.ToString());
}
Log("Waiting for connection B to detect...");
await UntilCondition(TimeSpan.FromSeconds(10), () => b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave).ForAwait();
subA.Ping();
subB.Ping();
Log("Falover 2 Attempted. Pausing...");
Log(" A " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Log(" A " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
Log(" B " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Log(" B " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
if (!a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave)
{
TopologyFail();
}
Log("Falover 2 Complete.");
Assert.True(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
await UntilCondition(TimeSpan.FromSeconds(10), () => b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave).ForAwait();
var sanityCheck = b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave;
if (!sanityCheck)
{
Log("FAILURE: B has not detected the topology change.");
foreach (var server in b.GetServerSnapshot().ToArray())
{
Log(" Server" + server.EndPoint);
Log(" State: " + server.ConnectionState);
Log(" IsMaster: " + !server.IsSlave);
Log(" Type: " + server.ServerType);
}
//Skip.Inconclusive("Not enough latency.");
}
Assert.True(sanityCheck, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
Log("Pause complete");
Log(" A outstanding: " + a.GetCounters().TotalOutstanding);
Log(" B outstanding: " + b.GetCounters().TotalOutstanding);
subA.Ping();
subB.Ping();
await Task.Delay(5000).ForAwait();
epA = subA.SubscribedEndpoint(channel);
epB = subB.SubscribedEndpoint(channel);
Log("Subscription complete");
Log(" A: " + EndPointCollection.ToString(epA));
Log(" B: " + EndPointCollection.ToString(epB));
var aSentTo = subA.Publish(channel, "A2");
var bSentTo = subB.Publish(channel, "B2");
Log(" A2 sent to: " + aSentTo);
Log(" B2 sent to: " + bSentTo);
subA.Ping();
subB.Ping();
Log("Ping Complete. Checking...");
await UntilCondition(TimeSpan.FromSeconds(10), () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
Log("Counts so far:");
Log(" aCount: " + Interlocked.Read(ref aCount));
Log(" bCount: " + Interlocked.Read(ref bCount));
Log(" masterChanged: " + Interlocked.Read(ref masterChanged));
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
// Expect 10, because a sees a, but b sees a and b due to replication
Assert.Equal(10, Interlocked.CompareExchange(ref masterChanged, 0, 0));
}
catch
{
LogNoTime("");
Log("ERROR: Something went bad - see above! Roooooolling back. Back it up. Baaaaaack it on up.");
LogNoTime("");
throw;
}
finally
{
Log("Restoring configuration...");
try
{
a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).MakeMaster(ReplicationChangeOptions.All);
await Task.Delay(1000).ForAwait();
}
catch { }
}
}
}
public async Task DeslaveGoesToPrimary()
{
ConfigurationOptions config = GetMasterSlaveConfig();
using (var conn = ConnectionMultiplexer.Connect(config))
{
var primary = conn.GetServer(TestConfig.Current.FailoverMasterServerAndPort);
var secondary = conn.GetServer(TestConfig.Current.FailoverSlaveServerAndPort);
primary.Ping();
secondary.Ping();
primary.MakeMaster(ReplicationChangeOptions.SetTiebreaker);
secondary.MakeMaster(ReplicationChangeOptions.None);
await Task.Delay(100).ConfigureAwait(false);
primary.Ping();
secondary.Ping();
using (var writer = new StringWriter())
{
conn.Configure(writer);
string log = writer.ToString();
Writer.WriteLine(log);
bool isUnanimous = log.Contains("tie-break is unanimous at " + TestConfig.Current.FailoverMasterServerAndPort);
if (!isUnanimous)
{
Skip.Inconclusive("this is timing sensitive; unable to verify this time");
}
}
// k, so we know everyone loves 6379; is that what we get?
var db = conn.GetDatabase();
RedisKey key = Me();
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferMaster));
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.DemandMaster));
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferSlave));
var ex = Assert.Throws <RedisConnectionException>(() => db.IdentifyEndpoint(key, CommandFlags.DemandSlave));
Assert.StartsWith("No connection is active/available to service this operation: EXISTS " + Me(), ex.Message);
Writer.WriteLine("Invoking MakeMaster()...");
primary.MakeMaster(ReplicationChangeOptions.Broadcast | ReplicationChangeOptions.EnslaveSubordinates | ReplicationChangeOptions.SetTiebreaker, Writer);
Writer.WriteLine("Finished MakeMaster() call.");
await Task.Delay(100).ConfigureAwait(false);
Writer.WriteLine("Invoking Ping() (post-master)");
primary.Ping();
secondary.Ping();
Writer.WriteLine("Finished Ping() (post-master)");
Assert.True(primary.IsConnected, $"{primary.EndPoint} is not connected.");
Assert.True(secondary.IsConnected, $"{secondary.EndPoint} is not connected.");
Writer.WriteLine($"{primary.EndPoint}: {primary.ServerType}, Mode: {(primary.IsSlave ? "Slave" : "Master")}");
Writer.WriteLine($"{secondary.EndPoint}: {secondary.ServerType}, Mode: {(secondary.IsSlave ? "Slave" : "Master")}");
// Create a separate multiplexer with a valid view of the world to distinguish between failures of
// server topology changes from failures to recognize those changes
Writer.WriteLine("Connecting to secondary validation connection.");
using (var conn2 = ConnectionMultiplexer.Connect(config))
{
var primary2 = conn2.GetServer(TestConfig.Current.FailoverMasterServerAndPort);
var secondary2 = conn2.GetServer(TestConfig.Current.FailoverSlaveServerAndPort);
Writer.WriteLine($"Check: {primary2.EndPoint}: {primary2.ServerType}, Mode: {(primary2.IsSlave ? "Slave" : "Master")}");
Writer.WriteLine($"Check: {secondary2.EndPoint}: {secondary2.ServerType}, Mode: {(secondary2.IsSlave ? "Slave" : "Master")}");
Assert.False(primary2.IsSlave, $"{primary2.EndPoint} should be a master (verification connection).");
Assert.True(secondary2.IsSlave, $"{secondary2.EndPoint} should be a slave (verification connection).");
var db2 = conn2.GetDatabase();
Assert.Equal(primary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.PreferMaster));
Assert.Equal(primary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.DemandMaster));
Assert.Equal(secondary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.PreferSlave));
Assert.Equal(secondary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.DemandSlave));
}
await UntilCondition(TimeSpan.FromSeconds(20), () => !primary.IsSlave && secondary.IsSlave);
Assert.False(primary.IsSlave, $"{primary.EndPoint} should be a master.");
Assert.True(secondary.IsSlave, $"{secondary.EndPoint} should be a slave.");
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferMaster));
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.DemandMaster));
Assert.Equal(secondary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferSlave));
Assert.Equal(secondary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.DemandSlave));
}
}
protected virtual ConnectionMultiplexer Create(
string clientName = null, int?syncTimeout = null, bool?allowAdmin = null, int?keepAlive = null,
int?connectTimeout = null, string password = null, string tieBreaker = null, TextWriter log = null,
bool fail = true, string[] disabledCommands = null, string[] enabledCommands = null,
bool checkConnect = true, bool pause = true, string failMessage = null,
string channelPrefix = null, bool useSharedSocketManager = true, Proxy?proxy = null)
{
if (pause)
{
Thread.Sleep(250); // get a lot of glitches when hammering new socket creations etc; pace it out a bit
}
string configuration = GetConfiguration();
var config = ConfigurationOptions.Parse(configuration);
if (disabledCommands != null && disabledCommands.Length != 0)
{
config.CommandMap = CommandMap.Create(new HashSet <string>(disabledCommands), false);
}
else if (enabledCommands != null && enabledCommands.Length != 0)
{
config.CommandMap = CommandMap.Create(new HashSet <string>(enabledCommands), true);
}
if (Debugger.IsAttached)
{
syncTimeout = int.MaxValue;
}
if (useSharedSocketManager)
{
config.SocketManager = socketManager;
}
if (channelPrefix != null)
{
config.ChannelPrefix = channelPrefix;
}
if (tieBreaker != null)
{
config.TieBreaker = tieBreaker;
}
if (password != null)
{
config.Password = string.IsNullOrEmpty(password) ? null : password;
}
if (clientName != null)
{
config.ClientName = clientName;
}
if (syncTimeout != null)
{
config.SyncTimeout = syncTimeout.Value;
}
if (allowAdmin != null)
{
config.AllowAdmin = allowAdmin.Value;
}
if (keepAlive != null)
{
config.KeepAlive = keepAlive.Value;
}
if (connectTimeout != null)
{
config.ConnectTimeout = connectTimeout.Value;
}
if (proxy != null)
{
config.Proxy = proxy.Value;
}
var watch = Stopwatch.StartNew();
var task = ConnectionMultiplexer.ConnectAsync(config, log ?? Writer);
if (!task.Wait(config.ConnectTimeout >= (int.MaxValue / 2) ? int.MaxValue : config.ConnectTimeout * 2))
{
task.ContinueWith(x =>
{
try
{
GC.KeepAlive(x.Exception);
}
catch
{ }
}, TaskContinuationOptions.OnlyOnFaulted);
throw new TimeoutException("Connect timeout");
}
watch.Stop();
if (Output == null)
{
Assert.True(false, "Failure: Be sure to call the TestBase constuctor like this: BasicOpsTests(ITestOutputHelper output) : base(output) { }");
}
Output.WriteLine("Connect took: " + watch.ElapsedMilliseconds + "ms");
var muxer = task.Result;
if (checkConnect && (muxer == null || !muxer.IsConnected))
{
// If fail is true, we throw.
Assert.False(fail, failMessage + "Server is not available");
Skip.Inconclusive(failMessage + "Server is not available");
}
muxer.InternalError += OnInternalError;
muxer.ConnectionFailed += OnConnectionFailed;
return(muxer);
}
public static ConnectionMultiplexer CreateDefault(
TextWriter output,
string clientName = null, int?syncTimeout = null, bool?allowAdmin = null, int?keepAlive = null,
int?connectTimeout = null, string password = null, string tieBreaker = null, TextWriter log = null,
bool fail = true, string[] disabledCommands = null, string[] enabledCommands = null,
bool checkConnect = true, string failMessage = null,
string channelPrefix = null, Proxy?proxy = null,
string configuration = null, bool logTransactionData = true,
int?defaultDatabase = null,
[CallerMemberName] string caller = null)
{
StringWriter localLog = null;
if (log == null)
{
log = localLog = new StringWriter();
}
try
{
var config = ConfigurationOptions.Parse(configuration);
if (disabledCommands != null && disabledCommands.Length != 0)
{
config.CommandMap = CommandMap.Create(new HashSet <string>(disabledCommands), false);
}
else if (enabledCommands != null && enabledCommands.Length != 0)
{
config.CommandMap = CommandMap.Create(new HashSet <string>(enabledCommands), true);
}
if (Debugger.IsAttached)
{
syncTimeout = int.MaxValue;
}
if (channelPrefix != null)
{
config.ChannelPrefix = channelPrefix;
}
if (tieBreaker != null)
{
config.TieBreaker = tieBreaker;
}
if (password != null)
{
config.Password = string.IsNullOrEmpty(password) ? null : password;
}
if (clientName != null)
{
config.ClientName = clientName;
}
else if (caller != null)
{
config.ClientName = caller;
}
if (syncTimeout != null)
{
config.SyncTimeout = syncTimeout.Value;
}
if (allowAdmin != null)
{
config.AllowAdmin = allowAdmin.Value;
}
if (keepAlive != null)
{
config.KeepAlive = keepAlive.Value;
}
if (connectTimeout != null)
{
config.ConnectTimeout = connectTimeout.Value;
}
if (proxy != null)
{
config.Proxy = proxy.Value;
}
if (defaultDatabase != null)
{
config.DefaultDatabase = defaultDatabase.Value;
}
var watch = Stopwatch.StartNew();
var task = ConnectionMultiplexer.ConnectAsync(config, log);
if (!task.Wait(config.ConnectTimeout >= (int.MaxValue / 2) ? int.MaxValue : config.ConnectTimeout * 2))
{
task.ContinueWith(x =>
{
try
{
GC.KeepAlive(x.Exception);
}
catch { /* No boom */ }
}, TaskContinuationOptions.OnlyOnFaulted);
throw new TimeoutException("Connect timeout");
}
watch.Stop();
if (output != null)
{
Log(output, "Connect took: " + watch.ElapsedMilliseconds + "ms");
}
var muxer = task.Result;
if (checkConnect && (muxer == null || !muxer.IsConnected))
{
// If fail is true, we throw.
Assert.False(fail, failMessage + "Server is not available");
Skip.Inconclusive(failMessage + "Server is not available");
}
if (output != null)
{
muxer.MessageFaulted += (msg, ex, origin) =>
{
output?.WriteLine($"Faulted from '{origin}': '{msg}' - '{(ex == null ? "(null)" : ex.Message)}'");
if (ex != null && ex.Data.Contains("got"))
{
output?.WriteLine($"Got: '{ex.Data["got"]}'");
}
};
muxer.Connecting += (e, t) => output?.WriteLine($"Connecting to {Format.ToString(e)} as {t}");
if (logTransactionData)
{
muxer.TransactionLog += msg => output?.WriteLine("tran: " + msg);
}
muxer.InfoMessage += msg => output?.WriteLine(msg);
muxer.Resurrecting += (e, t) => output?.WriteLine($"Resurrecting {Format.ToString(e)} as {t}");
muxer.Closing += complete => output?.WriteLine(complete ? "Closed" : "Closing...");
}
return(muxer);
}
catch
{
if (localLog != null)
{
output?.WriteLine(localLog.ToString());
}
throw;
}
}
public async Task SubscriptionsSurviveMasterSwitchAsync(bool useSharedSocketManager)
{
if (RunningInCI)
{
Skip.Inconclusive("TODO: Fix race in broadcast reconfig a zero latency.");
}
using (var a = Create(allowAdmin: true, useSharedSocketManager: useSharedSocketManager))
using (var b = Create(allowAdmin: true, useSharedSocketManager: useSharedSocketManager))
{
RedisChannel channel = Me();
var subA = a.GetSubscriber();
var subB = b.GetSubscriber();
long masterChanged = 0, aCount = 0, bCount = 0;
a.ConfigurationChangedBroadcast += delegate
{
Output.WriteLine("A noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
b.ConfigurationChangedBroadcast += delegate
{
Output.WriteLine("B noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
subA.Subscribe(channel, (_, message) =>
{
Output.WriteLine("A got message: " + message);
Interlocked.Increment(ref aCount);
});
subB.Subscribe(channel, (_, message) =>
{
Output.WriteLine("B got message: " + message);
Interlocked.Increment(ref bCount);
});
Assert.False(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
Assert.True(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
Assert.True(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
var epA = subA.SubscribedEndpoint(channel);
var epB = subB.SubscribedEndpoint(channel);
Output.WriteLine("A: " + EndPointCollection.ToString(epA));
Output.WriteLine("B: " + EndPointCollection.ToString(epB));
subA.Publish(channel, "A1");
subB.Publish(channel, "B1");
subA.Ping();
subB.Ping();
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
Assert.Equal(0, Interlocked.Read(ref masterChanged));
try
{
Interlocked.Exchange(ref masterChanged, 0);
Interlocked.Exchange(ref aCount, 0);
Interlocked.Exchange(ref bCount, 0);
Output.WriteLine("Changing master...");
using (var sw = new StringWriter())
{
a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).MakeMaster(ReplicationChangeOptions.All, sw);
Output.WriteLine(sw.ToString());
}
await Task.Delay(5000).ForAwait();
subA.Ping();
subB.Ping();
Output.WriteLine("Pausing...");
Output.WriteLine("A " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Output.WriteLine("A " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
Output.WriteLine("B " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Output.WriteLine("B " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
Assert.True(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
Assert.True(b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
Output.WriteLine("Pause complete");
Output.WriteLine("A outstanding: " + a.GetCounters().TotalOutstanding);
Output.WriteLine("B outstanding: " + b.GetCounters().TotalOutstanding);
subA.Ping();
subB.Ping();
await Task.Delay(2000).ForAwait();
epA = subA.SubscribedEndpoint(channel);
epB = subB.SubscribedEndpoint(channel);
Output.WriteLine("A: " + EndPointCollection.ToString(epA));
Output.WriteLine("B: " + EndPointCollection.ToString(epB));
Output.WriteLine("A2 sent to: " + subA.Publish(channel, "A2"));
Output.WriteLine("B2 sent to: " + subB.Publish(channel, "B2"));
subA.Ping();
subB.Ping();
Output.WriteLine("Checking...");
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
// Expect 6, because a sees a, but b sees a and b due to replication
Assert.Equal(6, Interlocked.CompareExchange(ref masterChanged, 0, 0));
}
finally
{
Output.WriteLine("Restoring configuration...");
try
{
a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).MakeMaster(ReplicationChangeOptions.All);
await Task.Delay(1000).ForAwait();
}
catch { }
}
}
}
static void TopologyFail() => Skip.Inconclusive("Replication topology change failed...and that's both inconsistent and not what we're testing.");
public async Task SubscriptionsSurviveMasterSwitchAsync()
{
void TopologyFail() => Skip.Inconclusive("Replication tolopogy change failed...and that's both inconsistent and not what we're testing.");
if (RunningInCI)
{
Skip.Inconclusive("TODO: Fix race in broadcast reconfig a zero latency.");
}
using (var a = Create(allowAdmin: true, shared: false))
using (var b = Create(allowAdmin: true, shared: false))
{
RedisChannel channel = Me();
var subA = a.GetSubscriber();
var subB = b.GetSubscriber();
long masterChanged = 0, aCount = 0, bCount = 0;
a.ConfigurationChangedBroadcast += delegate
{
Log("A noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
b.ConfigurationChangedBroadcast += delegate
{
Log("B noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
};
subA.Subscribe(channel, (_, message) =>
{
Log("A got message: " + message);
Interlocked.Increment(ref aCount);
});
subB.Subscribe(channel, (_, message) =>
{
Log("B got message: " + message);
Interlocked.Increment(ref bCount);
});
Assert.False(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
if (!a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave)
{
TopologyFail();
}
Assert.True(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a master");
Assert.True(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a slave");
var epA = subA.SubscribedEndpoint(channel);
var epB = subB.SubscribedEndpoint(channel);
Log("A: " + EndPointCollection.ToString(epA));
Log("B: " + EndPointCollection.ToString(epB));
subA.Publish(channel, "A1");
subB.Publish(channel, "B1");
Log("SubA ping: " + subA.Ping());
Log("SubB ping: " + subB.Ping());
// If redis is under load due to this suite, it may take a moment to send across.
await UntilCondition(5000, () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
Assert.Equal(0, Interlocked.Read(ref masterChanged));
try
{
Interlocked.Exchange(ref masterChanged, 0);
Interlocked.Exchange(ref aCount, 0);
Interlocked.Exchange(ref bCount, 0);
Log("Changing master...");
using (var sw = new StringWriter())
{
a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).MakeMaster(ReplicationChangeOptions.All, sw);
Log(sw.ToString());
}
await UntilCondition(3000, () => b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave).ForAwait();
subA.Ping();
subB.Ping();
Log("Pausing...");
Log("A " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Log("A " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
Log("B " + TestConfig.Current.FailoverMasterServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave ? "Slave" : "Master"));
Log("B " + TestConfig.Current.FailoverSlaveServerAndPort + " status: " + (b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave ? "Slave" : "Master"));
if (!a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave)
{
TopologyFail();
}
Assert.True(a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"A Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
var sanityCheck = b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave;
if (!sanityCheck)
{
Skip.Inconclusive("Not enough latency.");
}
Assert.True(sanityCheck, $"B Connection: {TestConfig.Current.FailoverMasterServerAndPort} should be a slave");
Assert.False(b.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).IsSlave, $"B Connection: {TestConfig.Current.FailoverSlaveServerAndPort} should be a master");
Log("Pause complete");
Log("A outstanding: " + a.GetCounters().TotalOutstanding);
Log("B outstanding: " + b.GetCounters().TotalOutstanding);
subA.Ping();
subB.Ping();
await Task.Delay(1000).ForAwait();
epA = subA.SubscribedEndpoint(channel);
epB = subB.SubscribedEndpoint(channel);
Log("A: " + EndPointCollection.ToString(epA));
Log("B: " + EndPointCollection.ToString(epB));
Log("A2 sent to: " + subA.Publish(channel, "A2"));
Log("B2 sent to: " + subB.Publish(channel, "B2"));
subA.Ping();
subB.Ping();
Log("Checking...");
await UntilCondition(10000, () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
Assert.Equal(2, Interlocked.Read(ref aCount));
Assert.Equal(2, Interlocked.Read(ref bCount));
// Expect 6, because a sees a, but b sees a and b due to replication
Assert.Equal(6, Interlocked.CompareExchange(ref masterChanged, 0, 0));
}
finally
{
Log("Restoring configuration...");
try
{
a.GetServer(TestConfig.Current.FailoverMasterServerAndPort).MakeMaster(ReplicationChangeOptions.All);
await Task.Delay(1000).ForAwait();
}
catch { }
}
}
}
