/// <summary>
/// For testing only
/// </summary>
internal void SimulateConnectionFailure()
{
if (!Multiplexer.RawConfig.AllowAdmin)
{
throw ExceptionFactory.AdminModeNotEnabled(Multiplexer.IncludeDetailInExceptions, RedisCommand.DEBUG, null, ServerEndPoint); // close enough
}
physical?.RecordConnectionFailed(ConnectionFailureType.SocketFailure);
}
private async Task OnEstablishingAsyncAwaited(PhysicalConnection connection, Task handshake)
{
try
{
await handshake.ForAwait();
}
catch (Exception ex)
{
connection.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
}
}
public override bool SetResult(PhysicalConnection connection, Message message, RawResult result)
{
var final = base.SetResult(connection, message, result);
if (result.IsError)
{
if (result.IsEqual(authFail))
{
connection.RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure);
}
else if (result.AssertStarts(loading))
{
connection.RecordConnectionFailed(ConnectionFailureType.Loading);
}
else
{
connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure);
}
}
return(final);
}
internal void OnEstablishing(PhysicalConnection connection, TextWriter log)
{
try
{
if (connection == null)
{
return;
}
Handshake(connection, log);
}
catch (Exception ex)
{
connection.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
}
}
private static Action <Task <int> > EndReadFactory(PhysicalConnection physical)
{
return(result =>
{ // can't capture AsyncState on SocketRead, so we'll do it once per physical instead
try
{
physical.Multiplexer.Trace("Completed asynchronously: processing in callback", physical.physicalName);
if (physical.EndReading(result))
{
physical.BeginReading();
}
}
catch (Exception ex)
{
physical.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
}
});
}
internal void OnFullyEstablished(PhysicalConnection connection)
{
try
{
if (connection == null)
{
return;
}
var bridge = connection.Bridge;
if (bridge == subscription)
{
multiplexer.ResendSubscriptions(this);
}
multiplexer.OnConnectionRestored(endpoint, bridge.ConnectionType);
}
catch (Exception ex)
{
connection.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
}
}
protected override bool SetResultCore(PhysicalConnection connection, Message message, RawResult result)
{
bool happy;
switch (message.Command)
{
case RedisCommand.ECHO:
happy = result.Type == ResultType.BulkString && (!establishConnection || result.IsEqual(connection.Multiplexer.UniqueId));
break;
case RedisCommand.PING:
happy = result.Type == ResultType.SimpleString && result.IsEqual(RedisLiterals.BytesPONG);
break;
case RedisCommand.TIME:
happy = result.Type == ResultType.MultiBulk && result.GetItems().Length == 2;
break;
case RedisCommand.EXISTS:
happy = result.Type == ResultType.Integer;
break;
default:
happy = true;
break;
}
if (happy)
{
if (establishConnection)
{
connection.Bridge.OnFullyEstablished(connection);
}
SetResult(message, happy);
return(true);
}
else
{
connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure);
return(false);
}
}
internal Task OnEstablishingAsync(PhysicalConnection connection, TextWriter log)
{
try
{
if (connection == null)
{
return(Task.CompletedTask);
}
var handshake = HandshakeAsync(connection, log);
if (handshake.Status != TaskStatus.RanToCompletion)
{
return(OnEstablishingAsyncAwaited(connection, handshake));
}
}
catch (Exception ex)
{
connection.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
}
return(Task.CompletedTask);
}
internal void OnFullyEstablished(PhysicalConnection connection)
{
try
{
if (connection == null)
{
return;
}
var bridge = connection.BridgeCouldBeNull;
if (bridge != null)
{
if (bridge == subscription)
{
Multiplexer.ResendSubscriptions(this);
}
Multiplexer.OnConnectionRestored(EndPoint, bridge.ConnectionType, connection?.ToString());
}
}
catch (Exception ex)
{
connection.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
}
}
private bool WriteMessageToServer(PhysicalConnection connection, Message message)
{
if (message == null) return true;
try
{
var cmd = message.Command;
bool isMasterOnly = message.IsMasterOnly();
if (isMasterOnly && serverEndPoint.IsSlave && (serverEndPoint.SlaveReadOnly || !serverEndPoint.AllowSlaveWrites))
{
throw ExceptionFactory.MasterOnly(multiplexer.IncludeDetailInExceptions, message.Command, message, ServerEndPoint);
}
SelectDatabase(connection, message);
if (!connection.TransactionActive)
{
var readmode = connection.GetReadModeCommand(isMasterOnly);
if (readmode != null)
{
connection.Enqueue(readmode);
readmode.WriteTo(connection);
readmode.SetRequestSent();
IncrementOpCount();
}
if (message.IsAsking)
{
var asking = ReusableAskingCommand;
connection.Enqueue(asking);
asking.WriteImpl(connection);
asking.SetRequestSent();
IncrementOpCount();
}
}
switch (cmd)
{
case RedisCommand.WATCH:
case RedisCommand.MULTI:
connection.TransactionActive = true;
break;
case RedisCommand.UNWATCH:
case RedisCommand.EXEC:
case RedisCommand.DISCARD:
connection.TransactionActive = false;
break;
}
connection.Enqueue(message);
message.WriteImpl(connection);
message.SetRequestSent();
IncrementOpCount();
// some commands smash our ability to trust the database; some commands
// demand an immediate flush
switch (cmd)
{
case RedisCommand.EVAL:
case RedisCommand.EVALSHA:
if(!serverEndPoint.GetFeatures().ScriptingDatabaseSafe)
{
connection.SetUnknownDatabase();
}
break;
case RedisCommand.DISCARD:
case RedisCommand.EXEC:
connection.SetUnknownDatabase();
break;
}
return true;
}
catch (RedisCommandException ex)
{
Trace("Write failed: " + ex.Message);
message.Fail(ConnectionFailureType.ProtocolFailure, ex);
CompleteSyncOrAsync(message);
// this failed without actually writing; we're OK with that... unless there's a transaction
if (connection != null && connection.TransactionActive)
{
// we left it in a broken state; need to kill the connection
connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, ex);
return false;
}
return true;
}
catch (Exception ex)
{
Trace("Write failed: " + ex.Message);
message.Fail(ConnectionFailureType.InternalFailure, ex);
CompleteSyncOrAsync(message);
// we're not sure *what* happened here; probably an IOException; kill the connection
if(connection != null) connection.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
return false;
}
}
private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection connection, Message message)
{
if (message == null)
{
return(WriteResult.Success); // for some definition of success
}
bool isQueued = false;
try
{
var cmd = message.Command;
LastCommand = cmd;
bool isMasterOnly = message.IsMasterOnly();
if (isMasterOnly && ServerEndPoint.IsSlave && (ServerEndPoint.SlaveReadOnly || !ServerEndPoint.AllowSlaveWrites))
{
throw ExceptionFactory.MasterOnly(Multiplexer.IncludeDetailInExceptions, message.Command, message, ServerEndPoint);
}
switch (cmd)
{
case RedisCommand.QUIT:
connection.RecordQuit();
break;
case RedisCommand.EXEC:
Multiplexer.OnPreTransactionExec(message); // testing purposes, to force certain errors
break;
}
SelectDatabaseInsideWriteLock(connection, message);
if (!connection.TransactionActive)
{
var readmode = connection.GetReadModeCommand(isMasterOnly);
if (readmode != null)
{
connection.EnqueueInsideWriteLock(readmode);
readmode.WriteTo(connection);
readmode.SetRequestSent();
IncrementOpCount();
}
if (message.IsAsking)
{
var asking = ReusableAskingCommand;
connection.EnqueueInsideWriteLock(asking);
asking.WriteTo(connection);
asking.SetRequestSent();
IncrementOpCount();
}
}
switch (cmd)
{
case RedisCommand.WATCH:
case RedisCommand.MULTI:
connection.TransactionActive = true;
break;
case RedisCommand.UNWATCH:
case RedisCommand.EXEC:
case RedisCommand.DISCARD:
connection.TransactionActive = false;
break;
}
connection.EnqueueInsideWriteLock(message);
isQueued = true;
message.WriteTo(connection);
message.SetRequestSent();
IncrementOpCount();
// some commands smash our ability to trust the database; some commands
// demand an immediate flush
switch (cmd)
{
case RedisCommand.EVAL:
case RedisCommand.EVALSHA:
if (!ServerEndPoint.GetFeatures().ScriptingDatabaseSafe)
{
connection.SetUnknownDatabase();
}
break;
case RedisCommand.UNKNOWN:
case RedisCommand.DISCARD:
case RedisCommand.EXEC:
connection.SetUnknownDatabase();
break;
}
return(WriteResult.Success);
}
catch (RedisCommandException ex) when(!isQueued)
{
Trace("Write failed: " + ex.Message);
message.Fail(ConnectionFailureType.InternalFailure, ex, null);
this.CompleteSyncOrAsync(message);
// this failed without actually writing; we're OK with that... unless there's a transaction
if (connection?.TransactionActive == true)
{
// we left it in a broken state; need to kill the connection
connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, ex);
return(WriteResult.WriteFailure);
}
return(WriteResult.Success);
}
catch (Exception ex)
{
Trace("Write failed: " + ex.Message);
message.Fail(ConnectionFailureType.InternalFailure, ex, null);
this.CompleteSyncOrAsync(message);
// we're not sure *what* happened here; probably an IOException; kill the connection
connection?.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
return(WriteResult.WriteFailure);
}
}
internal WriteResult WriteQueue(int maxWork)
{
bool weAreWriter = false;
PhysicalConnection conn = null;
try
{
Trace("Writing queue from bridge");
weAreWriter = Interlocked.CompareExchange(ref activeWriters, 1, 0) == 0;
if (!weAreWriter)
{
Trace("(aborting: existing writer)");
return(WriteResult.CompetingWriter);
}
conn = GetConnection(null);
if (conn == null)
{
AbortUnsent();
Trace("Connection not available; exiting");
return(WriteResult.NoConnection);
}
Message last;
int count = 0;
while (true)
{
var next = queue.Dequeue();
if (next == null)
{
Trace("Nothing to write; exiting");
if (count == 0)
{
conn.Flush(); // only flush on an empty run
return(WriteResult.NothingToDo);
}
return(WriteResult.QueueEmptyAfterWrite);
}
last = next;
Trace("Now pending: " + GetPendingCount());
if (!WriteMessageDirect(conn, next))
{
AbortUnsent();
Trace("write failed; connection is toast; exiting");
return(WriteResult.NoConnection);
}
count++;
if (maxWork > 0 && count >= maxWork)
{
Trace("Work limit; exiting");
Trace(last != null, "Flushed up to: " + last);
conn.Flush();
break;
}
}
}
catch (IOException ex)
{
if (conn != null)
{
conn.RecordConnectionFailed(ConnectionFailureType.SocketFailure, ex);
conn = null;
}
AbortUnsent();
}
catch (Exception ex)
{
AbortUnsent();
OnInternalError(ex);
}
finally
{
if (weAreWriter)
{
Interlocked.Exchange(ref activeWriters, 0);
Trace("Exiting writer");
}
}
return(queue.Any() ? WriteResult.MoreWork : WriteResult.QueueEmptyAfterWrite);
}
private bool WriteMessageToServer(PhysicalConnection connection, Message message)
{
if (message == null)
{
return(true);
}
try
{
var cmd = message.Command;
bool isMasterOnly = message.IsMasterOnly();
if (isMasterOnly && serverEndPoint.IsSlave && (serverEndPoint.SlaveReadOnly || !serverEndPoint.AllowSlaveWrites))
{
throw ExceptionFactory.MasterOnly(multiplexer.IncludeDetailInExceptions, message.Command, message, ServerEndPoint);
}
SelectDatabase(connection, message);
if (!connection.TransactionActive)
{
var readmode = connection.GetReadModeCommand(isMasterOnly);
if (readmode != null)
{
connection.Enqueue(readmode);
readmode.WriteTo(connection);
readmode.SetRequestSent();
IncrementOpCount();
}
if (message.IsAsking)
{
var asking = ReusableAskingCommand;
connection.Enqueue(asking);
asking.WriteImpl(connection);
asking.SetRequestSent();
IncrementOpCount();
}
}
switch (cmd)
{
case RedisCommand.WATCH:
case RedisCommand.MULTI:
connection.TransactionActive = true;
break;
case RedisCommand.UNWATCH:
case RedisCommand.EXEC:
case RedisCommand.DISCARD:
connection.TransactionActive = false;
break;
}
connection.Enqueue(message);
message.WriteImpl(connection);
message.SetRequestSent();
IncrementOpCount();
// some commands smash our ability to trust the database; some commands
// demand an immediate flush
switch (cmd)
{
case RedisCommand.EVAL:
case RedisCommand.EVALSHA:
if (!serverEndPoint.GetFeatures().ScriptingDatabaseSafe)
{
connection.SetUnknownDatabase();
}
break;
case RedisCommand.DISCARD:
case RedisCommand.EXEC:
connection.SetUnknownDatabase();
break;
}
return(true);
}
catch (RedisCommandException ex)
{
Trace("Write failed: " + ex.Message);
message.Fail(ConnectionFailureType.ProtocolFailure, ex);
CompleteSyncOrAsync(message);
// this failed without actually writing; we're OK with that... unless there's a transaction
if (connection != null && connection.TransactionActive)
{
// we left it in a broken state; need to kill the connection
connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, ex);
return(false);
}
return(true);
}
catch (Exception ex)
{
Trace("Write failed: " + ex.Message);
message.Fail(ConnectionFailureType.InternalFailure, ex);
CompleteSyncOrAsync(message);
// we're not sure *what* happened here; probably an IOException; kill the connection
if (connection != null)
{
connection.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
}
return(false);
}
}
