/// <summary>
/// Prompt all clients to reconnect.
/// </summary>
public static void BroadcastReconnectMessage(RedisConnection connection)
{
if (connection == null)
{
throw new ArgumentNullException("connection");
}
connection.Wait(connection.Publish(RedisMasterChangedChannel, "*"));
}
private void TestSubscriberNameOnRemote(bool setName)
{
string id = Config.CreateUniqueName();
using (var pub = new RedisConnection(Config.RemoteHost, allowAdmin: true))
using (var sub = new RedisSubscriberConnection(Config.RemoteHost))
{
List<string> errors = new List<string>();
EventHandler<BookSleeve.ErrorEventArgs> errorHandler = (sender, args) =>
{
lock (errors) errors.Add(args.Exception.Message);
};
pub.Error += errorHandler;
sub.Error += errorHandler;
if (setName)
{
pub.Name = "pub_" + id;
sub.Name = "sub_" + id;
}
int count = 0;
var subscribe = sub.Subscribe("foo"+id, (key,payload) => Interlocked.Increment(ref count));
Task pOpen = pub.Open(), sOpen = sub.Open();
pub.WaitAll(pOpen, sOpen, subscribe);
Assert.AreEqual(0, Interlocked.CompareExchange(ref count, 0, 0), "init message count");
pub.Wait(pub.Publish("foo" + id, "hello"));
PubSub.AllowReasonableTimeToPublishAndProcess();
var clients = setName ? pub.Wait(pub.Server.ListClients()) : null;
Assert.AreEqual(1, Interlocked.CompareExchange(ref count, 0, 0), "got message");
lock (errors)
{
foreach (var error in errors)
{
Console.WriteLine(error);
}
Assert.AreEqual(0, errors.Count, "zero errors");
}
if (setName)
{
Assert.AreEqual(1, clients.Count(x => x.Name == pub.Name), "pub has name");
Assert.AreEqual(1, clients.Count(x => x.Name == sub.Name), "sub has name");
}
}
}
private static void ProcessQueue(RedisConnection connection, string[] queues)
{
var stopWatch = new Stopwatch();
var formatter = new ObjectFormatter(maxLineLength: 5120);
Logger.Info("ProcessQueue started.");
while (true)
{
var message = connection.Lists.BlockingRemoveFirst(0, queues, DefaultTimeout);
if (message.Result == null)
{
continue;
}
var command = ExecuteCommand.Deserialize(message.Result.Item2);
var timeInQueue = DateTime.UtcNow - command.Submitted;
Logger.Info("Job received after {0:N3} seconds in queue.", timeInQueue.TotalSeconds);
if (timeInQueue > command.TimeoutPeriod)
{
Logger.Warn("Job was in queue for longer than {0} seconds, skipping!", command.TimeoutPeriod.Seconds);
continue;
}
stopWatch.Start();
var result = Executer.Execute(command.Code, command.Classes);
stopWatch.Stop();
Logger.Info("Work completed in {0} milliseconds.", stopWatch.ElapsedMilliseconds);
try
{
var response = JsonConvert.SerializeObject(new
{
code = command.Code,
classes = command.Classes,
result = formatter.FormatObject(result),
time = DateTime.UtcNow,
duration = stopWatch.ElapsedMilliseconds
});
var bytes = Encoding.UTF8.GetBytes(response);
var listeners = connection.Publish("workers:job-done:" + command.ClientId, bytes);
Logger.Info("Work results published to {0} listeners.", listeners.Result);
}
catch (JsonSerializationException ex)
{
Logger.ErrorException("An error occurred while attempting to serialize the JSON result.", ex);
}
stopWatch.Reset();
}
}
private static RedisConnection SelectAndCreateConnection(string configuration, TextWriter log,
out string selectedConfiguration,
out string[] availableEndpoints, bool autoMaster,
string newMaster = null, string tieBreakerKey = null)
{
if (tieBreakerKey == null)
{
tieBreakerKey = "__Booksleeve_TieBreak"; // default tie-breaker key
}
int syncTimeout;
bool allowAdmin;
string serviceName;
string clientName;
if (log == null)
{
log = new StringWriter();
}
string[] arr = GetConfigurationOptions(configuration, out syncTimeout, out allowAdmin, out serviceName,
out clientName);
if (!string.IsNullOrWhiteSpace(newMaster))
{
allowAdmin = true; // need this to diddle the slave/master config
}
log.WriteLine("{0} unique nodes specified", arr.Length);
log.WriteLine("sync timeout: {0}ms, admin commands: {1}", syncTimeout,
allowAdmin ? "enabled" : "disabled");
if (!string.IsNullOrEmpty(serviceName))
{
log.WriteLine("service: {0}", serviceName);
}
if (!string.IsNullOrEmpty(clientName))
{
log.WriteLine("client: {0}", clientName);
}
if (arr.Length == 0)
{
log.WriteLine("No nodes to consider");
selectedConfiguration = null;
availableEndpoints = new string[0];
return(null);
}
var connections = new List <RedisConnection>(arr.Length);
RedisConnection preferred = null;
try
{
var infos = new List <Task <string> >(arr.Length);
var tiebreakers = new List <Task <string> >(arr.Length);
foreach (string option in arr)
{
if (string.IsNullOrWhiteSpace(option))
{
continue;
}
RedisConnection conn = null;
try
{
string[] parts = option.Split(':');
if (parts.Length == 0)
{
continue;
}
string host = parts[0].Trim();
int port = 6379, tmp;
if (parts.Length > 1 && int.TryParse(parts[1].Trim(), out tmp))
{
port = tmp;
}
conn = new RedisConnection(host, port, syncTimeout: syncTimeout, allowAdmin: allowAdmin);
conn.Name = clientName;
log.WriteLine("Opening connection to {0}:{1}...", host, port);
conn.Open();
Task <string> info = conn.GetInfo();
Task <string> tiebreak = conn.Strings.GetString(0, tieBreakerKey);
connections.Add(conn);
infos.Add(info);
tiebreakers.Add(tiebreak);
}
catch (Exception ex)
{
if (conn == null)
{
log.WriteLine("Error parsing option \"{0}\": {1}", option, ex.Message);
}
else
{
log.WriteLine("Error connecting: {0}", ex.Message);
}
}
}
List <RedisConnection> masters = new List <RedisConnection>(), slaves = new List <RedisConnection>();
var breakerScores = new Dictionary <string, int>();
foreach (var tiebreak in tiebreakers)
{
try
{
if (tiebreak.Wait(syncTimeout))
{
string key = tiebreak.Result;
if (string.IsNullOrWhiteSpace(key))
{
continue;
}
int score;
if (breakerScores.TryGetValue(key, out score))
{
breakerScores[key] = score + 1;
}
else
{
breakerScores.Add(key, 1);
}
}
}
catch
{
/* if a node is down, that's fine too */
}
}
// see if any of our nodes are sentinels that know about the named service
List <Tuple <RedisConnection, Task <Tuple <string, int> > > > sentinelNodes = null;
foreach (RedisConnection conn in connections)
{
// the "wait" we did during tie-breaker detection means we should now know what each server is
if (conn.ServerType == ServerType.Sentinel)
{
if (string.IsNullOrEmpty(serviceName))
{
log.WriteLine("Sentinel discovered, but no serviceName was specified; ignoring {0}:{1}",
conn.Host, conn.Port);
}
else
{
log.WriteLine("Querying sentinel {0}:{1} for {2}...", conn.Host, conn.Port, serviceName);
if (sentinelNodes == null)
{
sentinelNodes = new List <Tuple <RedisConnection, Task <Tuple <string, int> > > >();
}
sentinelNodes.Add(Tuple.Create(conn, conn.QuerySentinelMaster(serviceName)));
}
}
}
// wait for sentinel results, if any
if (sentinelNodes != null)
{
var discoveredPairs = new Dictionary <Tuple <string, int>, int>();
foreach (var pair in sentinelNodes)
{
RedisConnection conn = pair.Item1;
try
{
Tuple <string, int> master = conn.Wait(pair.Item2);
if (master == null)
{
log.WriteLine("Sentinel {0}:{1} is not configured for {2}", conn.Host, conn.Port,
serviceName);
}
else
{
log.WriteLine("Sentinel {0}:{1} nominates {2}:{3}", conn.Host, conn.Port, master.Item1,
master.Item2);
int count;
if (discoveredPairs.TryGetValue(master, out count))
{
count = 0;
}
discoveredPairs[master] = count + 1;
}
}
catch (Exception ex)
{
log.WriteLine("Error from sentinel {0}:{1} - {2}", conn.Host, conn.Port, ex.Message);
}
}
Tuple <string, int> finalChoice;
switch (discoveredPairs.Count)
{
case 0:
log.WriteLine("No sentinels nominated a master; unable to connect");
finalChoice = null;
break;
case 1:
finalChoice = discoveredPairs.Single().Key;
log.WriteLine("Sentinels nominated unanimous master: {0}:{1}", finalChoice.Item1,
finalChoice.Item2);
break;
default:
finalChoice = discoveredPairs.OrderByDescending(kvp => kvp.Value).First().Key;
log.WriteLine("Sentinels nominated multiple masters; choosing arbitrarily: {0}:{1}",
finalChoice.Item1, finalChoice.Item2);
break;
}
if (finalChoice != null)
{
RedisConnection toBeDisposed = null;
try
{
// good bet that in this scenario the input didn't specify any actual redis servers, so we'll assume open a new one
log.WriteLine("Opening nominated master: {0}:{1}...", finalChoice.Item1, finalChoice.Item2);
toBeDisposed = new RedisConnection(finalChoice.Item1, finalChoice.Item2,
allowAdmin: allowAdmin, syncTimeout: syncTimeout);
toBeDisposed.Wait(toBeDisposed.Open());
if (toBeDisposed.ServerType == ServerType.Master)
{
RedisConnection tmp = toBeDisposed;
toBeDisposed = null; // so we don't dispose it
selectedConfiguration = tmp.Host + ":" + tmp.Port;
availableEndpoints = new[] { selectedConfiguration };
return(tmp);
}
else
{
log.WriteLine("Server is {0} instead of a master", toBeDisposed.ServerType);
}
}
catch (Exception ex)
{
log.WriteLine("Error: {0}", ex.Message);
}
finally
{
// dispose if something went sour
using (toBeDisposed)
{
}
}
}
// something went south; BUT SENTINEL WINS TRUMPS; quit now
selectedConfiguration = null;
availableEndpoints = new string[0];
return(null);
}
// check for tie-breakers (i.e. when we store which is the master)
switch (breakerScores.Count)
{
case 0:
log.WriteLine("No tie-breakers found ({0})", tieBreakerKey);
break;
case 1:
log.WriteLine("Tie-breaker ({0}) is unanimous: {1}", tieBreakerKey, breakerScores.Keys.Single());
break;
default:
log.WriteLine("Ambiguous tie-breakers ({0}):", tieBreakerKey);
foreach (var kvp in breakerScores.OrderByDescending(x => x.Value))
{
log.WriteLine("\t{0}: {1}", kvp.Key, kvp.Value);
}
break;
}
for (int i = 0; i < connections.Count; i++)
{
log.WriteLine("Reading configuration from {0}:{1}...", connections[i].Host, connections[i].Port);
try
{
if (!infos[i].Wait(syncTimeout))
{
log.WriteLine("\tTimeout fetching INFO");
continue;
}
var infoPairs = new StringDictionary();
using (var sr = new StringReader(infos[i].Result))
{
string line;
while ((line = sr.ReadLine()) != null)
{
int idx = line.IndexOf(':');
if (idx < 0)
{
continue;
}
string key = line.Substring(0, idx).Trim(),
value = line.Substring(idx + 1, line.Length - (idx + 1)).Trim();
infoPairs[key] = value;
}
}
string role = infoPairs["role"];
switch (role)
{
case "slave":
log.WriteLine("\tServer is SLAVE of {0}:{1}",
infoPairs["master_host"], infoPairs["master_port"]);
log.Write("\tLink is {0}, seen {1} seconds ago",
infoPairs["master_link_status"], infoPairs["master_last_io_seconds_ago"]);
if (infoPairs["master_sync_in_progress"] == "1")
{
log.Write(" (sync is in progress)");
}
log.WriteLine();
slaves.Add(connections[i]);
break;
case "master":
log.WriteLine("\tServer is MASTER, with {0} slaves", infoPairs["connected_slaves"]);
masters.Add(connections[i]);
break;
default:
log.WriteLine("\tUnknown role: {0}", role);
break;
}
string tmp = infoPairs["connected_clients"];
int clientCount, channelCount, patternCount;
if (string.IsNullOrWhiteSpace(tmp) || !int.TryParse(tmp, out clientCount))
{
clientCount = -1;
}
tmp = infoPairs["pubsub_channels"];
if (string.IsNullOrWhiteSpace(tmp) || !int.TryParse(tmp, out channelCount))
{
channelCount = -1;
}
tmp = infoPairs["pubsub_patterns"];
if (string.IsNullOrWhiteSpace(tmp) || !int.TryParse(tmp, out patternCount))
{
patternCount = -1;
}
log.WriteLine("\tClients: {0}; channels: {1}; patterns: {2}", clientCount, channelCount,
patternCount);
}
catch (Exception ex)
{
log.WriteLine("\tError reading INFO results: {0}", ex.Message);
}
}
if (newMaster == null)
{
switch (masters.Count)
{
case 0:
switch (slaves.Count)
{
case 0:
log.WriteLine("No masters or slaves found");
break;
case 1:
log.WriteLine("No masters found; selecting single slave");
preferred = slaves[0];
break;
default:
log.WriteLine("No masters found; considering {0} slaves...", slaves.Count);
preferred = SelectWithTieBreak(log, slaves, breakerScores);
break;
}
if (preferred != null)
{
if (autoMaster)
{
//LogException("Promoting redis SLAVE to MASTER");
log.WriteLine("Promoting slave to master...");
if (allowAdmin)
{
// can do on this connection
preferred.Wait(preferred.Server.MakeMaster());
}
else
{
// need an admin connection for this
using (
var adminPreferred = new RedisConnection(preferred.Host, preferred.Port,
allowAdmin: true,
syncTimeout: syncTimeout))
{
adminPreferred.Open();
adminPreferred.Wait(adminPreferred.Server.MakeMaster());
}
}
}
else
{
log.WriteLine("Slave should be promoted to master (but not done yet)...");
}
}
break;
case 1:
log.WriteLine("One master found; selecting");
preferred = masters[0];
break;
default:
log.WriteLine("Considering {0} masters...", masters.Count);
preferred = SelectWithTieBreak(log, masters, breakerScores);
break;
}
}
else
{
// we have been instructed to change master server
preferred = masters.Concat(slaves)
.FirstOrDefault(conn => (conn.Host + ":" + conn.Port) == newMaster);
if (preferred == null)
{
log.WriteLine("Selected new master not available: {0}", newMaster);
}
else
{
int errorCount = 0;
try
{
log.WriteLine("Promoting to master: {0}:{1}...", preferred.Host, preferred.Port);
preferred.Wait(preferred.Server.MakeMaster());
// if this is a master, we expect set/publish to work, even on 2.6
preferred.Strings.Set(0, tieBreakerKey, newMaster);
preferred.Wait(preferred.Publish(RedisMasterChangedChannel, newMaster));
}
catch (Exception ex)
{
log.WriteLine("\t{0}", ex.Message);
errorCount++;
}
if (errorCount == 0) // only make slaves if the master was happy
{
foreach (RedisConnection conn in masters.Concat(slaves))
{
if (conn == preferred)
{
continue; // can't make self a slave!
}
try
{
log.WriteLine("Enslaving: {0}:{1}...", conn.Host, conn.Port);
// try to set the tie-breaker **first** in case of problems
Task didSet = conn.Strings.Set(0, tieBreakerKey, newMaster);
// and broadcast to anyone who thinks this is the master
Task <long> didPublish = conn.Publish(RedisMasterChangedChannel, newMaster);
// now make it a slave
Task didEnslave = conn.Server.MakeSlave(preferred.Host, preferred.Port);
// these are best-effort only; from 2.6, readonly slave servers may reject these commands
try
{
conn.Wait(didSet);
}
catch
{
}
try
{
conn.Wait(didPublish);
}
catch
{
}
// but this one we'll log etc
conn.Wait(didEnslave);
}
catch (Exception ex)
{
log.WriteLine("\t{0}", ex.Message);
errorCount++;
}
}
}
if (errorCount != 0)
{
log.WriteLine("Things didn't go smoothly; CHECK WHAT HAPPENED!");
}
// want the connection disposed etc
preferred = null;
}
}
if (preferred == null)
{
selectedConfiguration = null;
}
else
{
selectedConfiguration = preferred.Host + ":" + preferred.Port;
log.WriteLine("Selected server {0}", selectedConfiguration);
}
availableEndpoints = (from conn in masters.Concat(slaves)
select conn.Host + ":" + conn.Port).ToArray();
return(preferred);
}
finally
{
foreach (RedisConnection conn in connections)
{
if (conn != null && conn != preferred)
{
try
{
conn.Dispose();
}
catch
{
}
}
}
}
}
/// <summary>
/// Prompt all clients to reconnect.
/// </summary>
public static void BroadcastReconnectMessage(RedisConnection connection)
{
if(connection == null) throw new ArgumentNullException("connection");
connection.Wait(connection.Publish(RedisMasterChangedChannel, "*"));
}
private static RedisConnection SelectAndCreateConnection(string configuration, TextWriter log, out string selectedConfiguration, out string[] availableEndpoints, bool autoMaster, string newMaster = null)
{
int syncTimeout;
bool allowAdmin;
if(log == null) log = new StringWriter();
var arr = GetConfigurationOptions(configuration, out syncTimeout, out allowAdmin);
if (!string.IsNullOrWhiteSpace(newMaster)) allowAdmin = true; // need this to diddle the slave/master config
log.WriteLine("{0} unique nodes specified", arr.Length);
log.WriteLine("sync timeout: {0}ms, admin commands: {1}", syncTimeout,
allowAdmin ? "enabled" : "disabled");
if (arr.Length == 0)
{
log.WriteLine("No nodes to consider");
selectedConfiguration = null;
availableEndpoints = new string[0];
return null;
}
var connections = new List<RedisConnection>(arr.Length);
RedisConnection preferred = null;
try
{
var infos = new List<Task<string>>(arr.Length);
var tiebreakers = new List<Task<string>>(arr.Length);
foreach (var option in arr)
{
if (string.IsNullOrWhiteSpace(option)) continue;
RedisConnection conn = null;
try
{
var parts = option.Split(':');
if (parts.Length == 0) continue;
string host = parts[0].Trim();
int port = 6379, tmp;
if (parts.Length > 1 && int.TryParse(parts[1].Trim(), out tmp)) port = tmp;
conn = new RedisConnection(host, port, syncTimeout: syncTimeout, allowAdmin: allowAdmin);
log.WriteLine("Opening connection to {0}:{1}...", host, port);
conn.Open();
var info = conn.GetInfo();
var tiebreak = conn.Strings.GetString(0, TieBreakerKey);
connections.Add(conn);
infos.Add(info);
tiebreakers.Add(tiebreak);
}
catch (Exception ex)
{
if (conn == null)
{
log.WriteLine("Error parsing option \"{0}\": {1}", option, ex.Message);
}
else
{
log.WriteLine("Error connecting: {0}", ex.Message);
}
}
}
List<RedisConnection> masters = new List<RedisConnection>(), slaves = new List<RedisConnection>();
var breakerScores = new Dictionary<string, int>();
foreach (var tiebreak in tiebreakers)
{
try
{
if (tiebreak.Wait(syncTimeout))
{
string key = tiebreak.Result;
if (string.IsNullOrWhiteSpace(key)) continue;
int score;
if (breakerScores.TryGetValue(key, out score)) breakerScores[key] = score + 1;
else breakerScores.Add(key, 1);
}
}
catch { /* if a node is down, that's fine too */ }
}
// check for tie-breakers (i.e. when we store which is the master)
switch (breakerScores.Count)
{
case 0:
log.WriteLine("No tie-breakers found");
break;
case 1:
log.WriteLine("Tie-breaker is unanimous: {0}", breakerScores.Keys.Single());
break;
default:
log.WriteLine("Ambiguous tie-breakers:");
foreach (var kvp in breakerScores.OrderByDescending(x => x.Value))
{
log.WriteLine("\t{0}: {1}", kvp.Key, kvp.Value);
}
break;
}
for (int i = 0; i < connections.Count; i++)
{
log.WriteLine("Reading configuration from {0}:{1}...", connections[i].Host, connections[i].Port);
try
{
if (!infos[i].Wait(syncTimeout))
{
log.WriteLine("\tTimeout fetching INFO");
continue;
}
var infoPairs = new StringDictionary();
using (var sr = new StringReader(infos[i].Result))
{
string line;
while ((line = sr.ReadLine()) != null)
{
int idx = line.IndexOf(':');
if (idx < 0) continue;
string key = line.Substring(0, idx).Trim(),
value = line.Substring(idx + 1, line.Length - (idx + 1)).Trim();
infoPairs[key] = value;
}
}
string role = infoPairs["role"];
switch (role)
{
case "slave":
log.WriteLine("\tServer is SLAVE of {0}:{1}",
infoPairs["master_host"], infoPairs["master_port"]);
log.Write("\tLink is {0}, seen {1} seconds ago",
infoPairs["master_link_status"], infoPairs["master_last_io_seconds_ago"]);
if (infoPairs["master_sync_in_progress"] == "1") log.Write(" (sync is in progress)");
log.WriteLine();
slaves.Add(connections[i]);
break;
case "master":
log.WriteLine("\tServer is MASTER, with {0} slaves", infoPairs["connected_slaves"]);
masters.Add(connections[i]);
break;
default:
log.WriteLine("\tUnknown role: {0}", role);
break;
}
string tmp = infoPairs["connected_clients"];
int clientCount, channelCount, patternCount;
if (string.IsNullOrWhiteSpace(tmp) || !int.TryParse(tmp, out clientCount)) clientCount = -1;
tmp = infoPairs["pubsub_channels"];
if (string.IsNullOrWhiteSpace(tmp) || !int.TryParse(tmp, out channelCount)) channelCount = -1;
tmp = infoPairs["pubsub_patterns"];
if (string.IsNullOrWhiteSpace(tmp) || !int.TryParse(tmp, out patternCount)) patternCount = -1;
log.WriteLine("\tClients: {0}; channels: {1}; patterns: {2}", clientCount, channelCount, patternCount);
}
catch (Exception ex)
{
log.WriteLine("\tError reading INFO results: {0}", ex.Message);
}
}
if (newMaster == null)
{
switch (masters.Count)
{
case 0:
switch (slaves.Count)
{
case 0:
log.WriteLine("No masters or slaves found");
break;
case 1:
log.WriteLine("No masters found; selecting single slave");
preferred = slaves[0];
break;
default:
log.WriteLine("No masters found; considering {0} slaves...", slaves.Count);
preferred = SelectWithTieBreak(log, slaves, breakerScores);
break;
}
if (preferred != null)
{
if (autoMaster)
{
//LogException("Promoting redis SLAVE to MASTER");
log.WriteLine("Promoting slave to master...");
if (allowAdmin)
{ // can do on this connection
preferred.Wait(preferred.Server.MakeMaster());
}
else
{ // need an admin connection for this
using (var adminPreferred = new RedisConnection(preferred.Host, preferred.Port, allowAdmin: true, syncTimeout: syncTimeout))
{
adminPreferred.Open();
adminPreferred.Wait(adminPreferred.Server.MakeMaster());
}
}
}
else
{
log.WriteLine("Slave should be promoted to master (but not done yet)...");
}
}
break;
case 1:
log.WriteLine("One master found; selecting");
preferred = masters[0];
break;
default:
log.WriteLine("Considering {0} masters...", masters.Count);
preferred = SelectWithTieBreak(log, masters, breakerScores);
break;
}
}
else
{ // we have been instructed to change master server
preferred = masters.Concat(slaves).FirstOrDefault(conn => (conn.Host + ":" + conn.Port) == newMaster);
if (preferred == null)
{
log.WriteLine("Selected new master not available: {0}", newMaster);
}
else
{
int errorCount = 0;
try
{
log.WriteLine("Promoting to master: {0}:{1}...", preferred.Host, preferred.Port);
preferred.Wait(preferred.Server.MakeMaster());
preferred.Strings.Set(0, TieBreakerKey, newMaster);
preferred.Wait(preferred.Publish(RedisMasterChangedChannel, newMaster));
}
catch (Exception ex)
{
log.WriteLine("\t{0}", ex.Message);
errorCount++;
}
if (errorCount == 0) // only make slaves if the master was happy
{
foreach (var conn in masters.Concat(slaves))
{
if (conn == preferred) continue; // can't make self a slave!
try
{
log.WriteLine("Enslaving: {0}:{1}...", conn.Host, conn.Port);
// set the tie-breaker **first** in case of problems
conn.Strings.Set(0, TieBreakerKey, newMaster);
// and broadcast to anyone who thinks this is the master
conn.Publish(RedisMasterChangedChannel, newMaster);
// now make it a slave
conn.Wait(conn.Server.MakeSlave(preferred.Host, preferred.Port));
}
catch (Exception ex)
{
log.WriteLine("\t{0}",ex.Message);
errorCount++;
}
}
}
if (errorCount != 0)
{
log.WriteLine("Things didn't go smoothly; CHECK WHAT HAPPENED!");
}
// want the connection disposed etc
preferred = null;
}
}
if (preferred == null)
{
selectedConfiguration = null;
}
else
{
selectedConfiguration = preferred.Host + ":" + preferred.Port;
log.WriteLine("Selected server {0}", selectedConfiguration);
}
availableEndpoints = (from conn in masters.Concat(slaves)
select conn.Host + ":" + conn.Port).ToArray();
return preferred;
}
finally
{
foreach (var conn in connections)
{
if (conn != null && conn != preferred) try { conn.Dispose(); }
catch { }
}
}
}
private static RedisConnection SelectAndCreateConnection(string configuration, TextWriter log,
out string selectedConfiguration,
out string[] availableEndpoints, bool autoMaster,
string newMaster = null, string tieBreakerKey = null)
{
if (tieBreakerKey == null) tieBreakerKey = "__Booksleeve_TieBreak"; // default tie-breaker key
int syncTimeout;
bool allowAdmin;
string serviceName;
string clientName;
if (log == null) log = new StringWriter();
string[] arr = GetConfigurationOptions(configuration, out syncTimeout, out allowAdmin, out serviceName,
out clientName);
if (!string.IsNullOrWhiteSpace(newMaster)) allowAdmin = true; // need this to diddle the slave/master config
log.WriteLine("{0} unique nodes specified", arr.Length);
log.WriteLine("sync timeout: {0}ms, admin commands: {1}", syncTimeout,
allowAdmin ? "enabled" : "disabled");
if (!string.IsNullOrEmpty(serviceName)) log.WriteLine("service: {0}", serviceName);
if (!string.IsNullOrEmpty(clientName)) log.WriteLine("client: {0}", clientName);
if (arr.Length == 0)
{
log.WriteLine("No nodes to consider");
selectedConfiguration = null;
availableEndpoints = new string[0];
return null;
}
var connections = new List<RedisConnection>(arr.Length);
RedisConnection preferred = null;
try
{
var infos = new List<Task<string>>(arr.Length);
var tiebreakers = new List<Task<string>>(arr.Length);
foreach (string option in arr)
{
if (string.IsNullOrWhiteSpace(option)) continue;
RedisConnection conn = null;
try
{
string[] parts = option.Split(':');
if (parts.Length == 0) continue;
string host = parts[0].Trim();
int port = 6379, tmp;
if (parts.Length > 1 && int.TryParse(parts[1].Trim(), out tmp)) port = tmp;
conn = new RedisConnection(host, port, syncTimeout: syncTimeout, allowAdmin: allowAdmin);
conn.Name = clientName;
log.WriteLine("Opening connection to {0}:{1}...", host, port);
conn.Open();
Task<string> info = conn.GetInfo();
Task<string> tiebreak = conn.Strings.GetString(0, tieBreakerKey);
connections.Add(conn);
infos.Add(info);
tiebreakers.Add(tiebreak);
}
catch (Exception ex)
{
if (conn == null)
{
log.WriteLine("Error parsing option \"{0}\": {1}", option, ex.Message);
}
else
{
log.WriteLine("Error connecting: {0}", ex.Message);
}
}
}
List<RedisConnection> masters = new List<RedisConnection>(), slaves = new List<RedisConnection>();
var breakerScores = new Dictionary<string, int>();
foreach (var tiebreak in tiebreakers)
{
try
{
if (tiebreak.Wait(syncTimeout))
{
string key = tiebreak.Result;
if (string.IsNullOrWhiteSpace(key)) continue;
int score;
if (breakerScores.TryGetValue(key, out score)) breakerScores[key] = score + 1;
else breakerScores.Add(key, 1);
}
}
catch
{
/* if a node is down, that's fine too */
}
}
// see if any of our nodes are sentinels that know about the named service
List<Tuple<RedisConnection, Task<Tuple<string, int>>>> sentinelNodes = null;
foreach (RedisConnection conn in connections)
{
// the "wait" we did during tie-breaker detection means we should now know what each server is
if (conn.ServerType == ServerType.Sentinel)
{
if (string.IsNullOrEmpty(serviceName))
{
log.WriteLine("Sentinel discovered, but no serviceName was specified; ignoring {0}:{1}",
conn.Host, conn.Port);
}
else
{
log.WriteLine("Querying sentinel {0}:{1} for {2}...", conn.Host, conn.Port, serviceName);
if (sentinelNodes == null)
sentinelNodes = new List<Tuple<RedisConnection, Task<Tuple<string, int>>>>();
sentinelNodes.Add(Tuple.Create(conn, conn.QuerySentinelMaster(serviceName)));
}
}
}
// wait for sentinel results, if any
if (sentinelNodes != null)
{
var discoveredPairs = new Dictionary<Tuple<string, int>, int>();
foreach (var pair in sentinelNodes)
{
RedisConnection conn = pair.Item1;
try
{
Tuple<string, int> master = conn.Wait(pair.Item2);
if (master == null)
{
log.WriteLine("Sentinel {0}:{1} is not configured for {2}", conn.Host, conn.Port,
serviceName);
}
else
{
log.WriteLine("Sentinel {0}:{1} nominates {2}:{3}", conn.Host, conn.Port, master.Item1,
master.Item2);
int count;
if (discoveredPairs.TryGetValue(master, out count)) count = 0;
discoveredPairs[master] = count + 1;
}
}
catch (Exception ex)
{
log.WriteLine("Error from sentinel {0}:{1} - {2}", conn.Host, conn.Port, ex.Message);
}
}
Tuple<string, int> finalChoice;
switch (discoveredPairs.Count)
{
case 0:
log.WriteLine("No sentinels nominated a master; unable to connect");
finalChoice = null;
break;
case 1:
finalChoice = discoveredPairs.Single().Key;
log.WriteLine("Sentinels nominated unanimous master: {0}:{1}", finalChoice.Item1,
finalChoice.Item2);
break;
default:
finalChoice = discoveredPairs.OrderByDescending(kvp => kvp.Value).First().Key;
log.WriteLine("Sentinels nominated multiple masters; choosing arbitrarily: {0}:{1}",
finalChoice.Item1, finalChoice.Item2);
break;
}
if (finalChoice != null)
{
RedisConnection toBeDisposed = null;
try
{
// good bet that in this scenario the input didn't specify any actual redis servers, so we'll assume open a new one
log.WriteLine("Opening nominated master: {0}:{1}...", finalChoice.Item1, finalChoice.Item2);
toBeDisposed = new RedisConnection(finalChoice.Item1, finalChoice.Item2,
allowAdmin: allowAdmin, syncTimeout: syncTimeout);
toBeDisposed.Wait(toBeDisposed.Open());
if (toBeDisposed.ServerType == ServerType.Master)
{
RedisConnection tmp = toBeDisposed;
toBeDisposed = null; // so we don't dispose it
selectedConfiguration = tmp.Host + ":" + tmp.Port;
availableEndpoints = new[] {selectedConfiguration};
return tmp;
}
else
{
log.WriteLine("Server is {0} instead of a master", toBeDisposed.ServerType);
}
}
catch (Exception ex)
{
log.WriteLine("Error: {0}", ex.Message);
}
finally
{
// dispose if something went sour
using (toBeDisposed)
{
}
}
}
// something went south; BUT SENTINEL WINS TRUMPS; quit now
selectedConfiguration = null;
availableEndpoints = new string[0];
return null;
}
// check for tie-breakers (i.e. when we store which is the master)
switch (breakerScores.Count)
{
case 0:
log.WriteLine("No tie-breakers found ({0})", tieBreakerKey);
break;
case 1:
log.WriteLine("Tie-breaker ({0}) is unanimous: {1}", tieBreakerKey, breakerScores.Keys.Single());
break;
default:
log.WriteLine("Ambiguous tie-breakers ({0}):", tieBreakerKey);
foreach (var kvp in breakerScores.OrderByDescending(x => x.Value))
{
log.WriteLine("\t{0}: {1}", kvp.Key, kvp.Value);
}
break;
}
for (int i = 0; i < connections.Count; i++)
{
log.WriteLine("Reading configuration from {0}:{1}...", connections[i].Host, connections[i].Port);
try
{
if (!infos[i].Wait(syncTimeout))
{
log.WriteLine("\tTimeout fetching INFO");
continue;
}
var infoPairs = new StringDictionary();
using (var sr = new StringReader(infos[i].Result))
{
string line;
while ((line = sr.ReadLine()) != null)
{
int idx = line.IndexOf(':');
if (idx < 0) continue;
string key = line.Substring(0, idx).Trim(),
value = line.Substring(idx + 1, line.Length - (idx + 1)).Trim();
infoPairs[key] = value;
}
}
string role = infoPairs["role"];
switch (role)
{
case "slave":
log.WriteLine("\tServer is SLAVE of {0}:{1}",
infoPairs["master_host"], infoPairs["master_port"]);
log.Write("\tLink is {0}, seen {1} seconds ago",
infoPairs["master_link_status"], infoPairs["master_last_io_seconds_ago"]);
if (infoPairs["master_sync_in_progress"] == "1") log.Write(" (sync is in progress)");
log.WriteLine();
slaves.Add(connections[i]);
break;
case "master":
log.WriteLine("\tServer is MASTER, with {0} slaves", infoPairs["connected_slaves"]);
masters.Add(connections[i]);
break;
default:
log.WriteLine("\tUnknown role: {0}", role);
break;
}
string tmp = infoPairs["connected_clients"];
int clientCount, channelCount, patternCount;
if (string.IsNullOrWhiteSpace(tmp) || !int.TryParse(tmp, out clientCount)) clientCount = -1;
tmp = infoPairs["pubsub_channels"];
if (string.IsNullOrWhiteSpace(tmp) || !int.TryParse(tmp, out channelCount)) channelCount = -1;
tmp = infoPairs["pubsub_patterns"];
if (string.IsNullOrWhiteSpace(tmp) || !int.TryParse(tmp, out patternCount)) patternCount = -1;
log.WriteLine("\tClients: {0}; channels: {1}; patterns: {2}", clientCount, channelCount,
patternCount);
}
catch (Exception ex)
{
log.WriteLine("\tError reading INFO results: {0}", ex.Message);
}
}
if (newMaster == null)
{
switch (masters.Count)
{
case 0:
switch (slaves.Count)
{
case 0:
log.WriteLine("No masters or slaves found");
break;
case 1:
log.WriteLine("No masters found; selecting single slave");
preferred = slaves[0];
break;
default:
log.WriteLine("No masters found; considering {0} slaves...", slaves.Count);
preferred = SelectWithTieBreak(log, slaves, breakerScores);
break;
}
if (preferred != null)
{
if (autoMaster)
{
//LogException("Promoting redis SLAVE to MASTER");
log.WriteLine("Promoting slave to master...");
if (allowAdmin)
{
// can do on this connection
preferred.Wait(preferred.Server.MakeMaster());
}
else
{
// need an admin connection for this
using (
var adminPreferred = new RedisConnection(preferred.Host, preferred.Port,
allowAdmin: true,
syncTimeout: syncTimeout))
{
adminPreferred.Open();
adminPreferred.Wait(adminPreferred.Server.MakeMaster());
}
}
}
else
{
log.WriteLine("Slave should be promoted to master (but not done yet)...");
}
}
break;
case 1:
log.WriteLine("One master found; selecting");
preferred = masters[0];
break;
default:
log.WriteLine("Considering {0} masters...", masters.Count);
preferred = SelectWithTieBreak(log, masters, breakerScores);
break;
}
}
else
{
// we have been instructed to change master server
preferred = masters.Concat(slaves)
.FirstOrDefault(conn => (conn.Host + ":" + conn.Port) == newMaster);
if (preferred == null)
{
log.WriteLine("Selected new master not available: {0}", newMaster);
}
else
{
int errorCount = 0;
try
{
log.WriteLine("Promoting to master: {0}:{1}...", preferred.Host, preferred.Port);
preferred.Wait(preferred.Server.MakeMaster());
// if this is a master, we expect set/publish to work, even on 2.6
preferred.Strings.Set(0, tieBreakerKey, newMaster);
preferred.Wait(preferred.Publish(RedisMasterChangedChannel, newMaster));
}
catch (Exception ex)
{
log.WriteLine("\t{0}", ex.Message);
errorCount++;
}
if (errorCount == 0) // only make slaves if the master was happy
{
foreach (RedisConnection conn in masters.Concat(slaves))
{
if (conn == preferred) continue; // can't make self a slave!
try
{
log.WriteLine("Enslaving: {0}:{1}...", conn.Host, conn.Port);
// try to set the tie-breaker **first** in case of problems
Task didSet = conn.Strings.Set(0, tieBreakerKey, newMaster);
// and broadcast to anyone who thinks this is the master
Task<long> didPublish = conn.Publish(RedisMasterChangedChannel, newMaster);
// now make it a slave
Task didEnslave = conn.Server.MakeSlave(preferred.Host, preferred.Port);
// these are best-effort only; from 2.6, readonly slave servers may reject these commands
try
{
conn.Wait(didSet);
}
catch
{
}
try
{
conn.Wait(didPublish);
}
catch
{
}
// but this one we'll log etc
conn.Wait(didEnslave);
}
catch (Exception ex)
{
log.WriteLine("\t{0}", ex.Message);
errorCount++;
}
}
}
if (errorCount != 0)
{
log.WriteLine("Things didn't go smoothly; CHECK WHAT HAPPENED!");
}
// want the connection disposed etc
preferred = null;
}
}
if (preferred == null)
{
selectedConfiguration = null;
}
else
{
selectedConfiguration = preferred.Host + ":" + preferred.Port;
log.WriteLine("Selected server {0}", selectedConfiguration);
}
availableEndpoints = (from conn in masters.Concat(slaves)
select conn.Host + ":" + conn.Port).ToArray();
return preferred;
}
finally
{
foreach (RedisConnection conn in connections)
{
if (conn != null && conn != preferred)
try
{
conn.Dispose();
}
catch
{
}
}
}
}
private void TestMassivePublishWithWithoutFlush(RedisConnection conn, string caption)
{
const int loop = 100000;
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
GC.WaitForPendingFinalizers();
var tasks = new Task[loop];
var withFlush = Stopwatch.StartNew();
for (int i = 0; i < loop; i++)
tasks[i] = conn.Publish("foo", "bar");
conn.WaitAll(tasks);
withFlush.Stop();
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
GC.WaitForPendingFinalizers();
conn.SuspendFlush();
var withoutFlush = Stopwatch.StartNew();
for (int i = 0; i < loop; i++)
tasks[i] = conn.Publish("foo", "bar");
conn.ResumeFlush();
conn.WaitAll(tasks);
withoutFlush.Stop();
Assert.Less(1, 2, "sanity check");
Assert.Less(withoutFlush.ElapsedMilliseconds, withFlush.ElapsedMilliseconds, caption);
Console.WriteLine("{2}: {0}ms (eager-flush) vs {1}ms (suspend-flush)",
withFlush.ElapsedMilliseconds, withoutFlush.ElapsedMilliseconds, caption);
}