private void SessionAbortedByPartner(bool isInbound, PartitionKey partition, IReliableMessagingSession session)
{
var snapshot = session.GetDataSnapshot();
if (isInbound)
{
FabricEvents.Events.SessionAbortedByPartner("Inbound@" + this.streamManager.TraceType, partition.ToString(), snapshot.SessionId.ToString());
}
else
{
FabricEvents.Events.SessionAbortedByPartner("Outbound@" + this.streamManager.TraceType, partition.ToString(), snapshot.SessionId.ToString());
}
}
/// <summary>
/// Find or create a session for a given partition.
/// This is called under a critical section by FindOrCreateDriverAsync and is not designed to be re-entrant
/// This routine must be called with a resolved and normalized partition, which FindOrCreateDriverAsync does
/// </summary>
/// <param name="targetPartition">Find the session for the given target partition</param>
/// <param name="timeout">Timeout for the operation to complete</param>
/// <returns>Return the reliable messaging session</returns>
internal async Task <IReliableMessagingSession> FindOrCreateOutboundSessionAsync(PartitionKey targetPartition, TimeSpan timeout)
{
FabricEvents.Events.FindOrCreateOutboundSession("Start@" + this.streamManager.TraceType, targetPartition.ToString(), "");
IReliableMessagingSession outboundSession = null;
var found = this.activeOutboundSessions.TryGetValue(targetPartition, out outboundSession);
if (found)
{
var snapshot = outboundSession.GetDataSnapshot();
FabricEvents.Events.FindOrCreateOutboundSession(
"Found@" + this.streamManager.TraceType,
targetPartition.ToString(),
snapshot.SessionId.ToString());
}
else
{
FabricEvents.Events.FindOrCreateOutboundSession("NotFound.SettingUp@" + this.streamManager.TraceType, targetPartition.ToString(), "");
try
{
var newSession = await this.SetupSession(targetPartition, timeout);
var added = this.activeOutboundSessions.TryAdd(targetPartition, newSession);
Diagnostics.Assert(added, "Failed to add new session to cache in FindOrCreateOutboundSession");
var snapshot = newSession.GetDataSnapshot();
FabricEvents.Events.FindOrCreateOutboundSession(
"SetupFinish@" + this.streamManager.TraceType,
targetPartition.ToString(),
snapshot.SessionId.ToString());
outboundSession = newSession;
}
catch (TimeoutException)
{
throw;
}
catch (FabricObjectClosedException)
{
throw;
}
catch (Exception e)
{
// TODO: overactive assert meant to discover failure modes
Diagnostics.Assert(false, "{0} SetupSession failure for Target: {1} with Exception: {2}", this.tracer, targetPartition, e);
}
}
return(outboundSession);
}
/// <summary>
/// Constructor to create an instance of the OutboundSessionDriver for a given Partition.
/// </summary>
/// <param name="targetPartition">Session for the Target Partition</param>
/// <param name="streamManager">Steam Manager where messages are sent to the session queue</param>
/// <param name="sessionConnectionManager">Session Connection manager, which creates and actively manages the session</param>
/// <param name="outboundSession">The session we will use to send out messages</param>
internal OutboundSessionDriver(
PartitionKey targetPartition, StreamManager streamManager, SessionConnectionManager sessionConnectionManager,
IReliableMessagingSession outboundSession)
{
// Assign relevent association to targetPartitions, Stream manager, and connection manager.
this.targetPartition = targetPartition;
this.streamManager = streamManager;
this.sessionConnectionManager = sessionConnectionManager;
// Instantiate the message queue.
this.sessionQueue = new ConcurrentQueue <IOperationData>();
// The targetPartition Session
this.outboundSession = outboundSession;
// The appropriate Session ID for tracing
var snapshot = outboundSession.GetDataSnapshot();
this.SessionId = snapshot.SessionId;
// Set default values.
this.drainQueueBarrier = 0;
this.tracer = streamManager.Tracer + " Target: " + this.targetPartition;
}
/// <summary>
/// Initializes a new instance of the InboundSessionDriver class for a given partition.
/// </summary>
/// <param name="partnerKey">Partner Key</param>
/// <param name="inboundSession">Active Inbound Session</param>
/// <param name="streamManager">Update corresponding streams in the StreamManager</param>
internal InboundSessionDriver(PartitionKey partnerKey, IReliableMessagingSession inboundSession, StreamManager streamManager)
{
this.partnerKey = partnerKey;
this.inboundSession = inboundSession;
this.streamManager = streamManager;
this.tracer = this.streamManager.Tracer + " Source: " + this.partnerKey;
this.traceType = this.streamManager.TraceType;
var snapshot = inboundSession.GetDataSnapshot();
this.SessionId = snapshot.SessionId;
// Start some async receive operations and wait for message to come through.
// Start the corresponding continuation operations when a message is received.
// TODO: Should the number of receive correspond to the sessionReceiveQuota.
// park some receive operations with the session
for (var i = 0; i < StreamConstants.ConcurrentReceiveOperationCount; i++)
{
this.ReceiveOnSession();
}
}
/// <summary>
/// When the OutBoundSessionDriver is disposed, Abort and clear the corresponding outbound session.
/// </summary>
/// <param name="targetPartition">Clear the Reliable MessagingSession for the target Partition</param>
internal void ClearOutboundSession(PartitionKey targetPartition)
{
FabricEvents.Events.ClearOutboundSession("Start@" + this.streamManager.TraceType, targetPartition.ToString(), "");
IReliableMessagingSession outboundSession = null;
var normalizedPartition = this.GetNormalizedPartitionKey(targetPartition);
var found = this.activeOutboundSessions.TryRemove(normalizedPartition, out outboundSession);
if (found)
{
var snapshot = outboundSession.GetDataSnapshot();
outboundSession.Abort();
FabricEvents.Events.ClearOutboundSession(
"Aborted@" + this.streamManager.TraceType,
normalizedPartition.ToString(),
snapshot.SessionId.ToString());
}
else
{
FabricEvents.Events.ClearOutboundSession("NotFound@" + this.streamManager.TraceType, normalizedPartition.ToString(), "");
}
}
/// <summary>
/// For the given source partition and inbound session, update(replace or create) the InboundSessionDriver cache.
/// </summary>
/// <param name="key">Source Partition</param>
/// <param name="session">Corresponding inbound messaging session</param>
/// <returns>true if accepted, false otherwise</returns>
private bool AcceptInboundSession(PartitionKey key, IReliableMessagingSession session)
{
var snapshot = session.GetDataSnapshot();
FabricEvents.Events.AcceptInboundSession("Start@" + this.streamManager.TraceType, key.ToString(), snapshot.SessionId.ToString());
InboundSessionDriver oldDriver = null;
var removeSuccess = this.streamManager.RuntimeResources.InboundSessionDrivers.TryRemove(key, out oldDriver);
if (removeSuccess)
{
FabricEvents.Events.AcceptInboundSession("ClearedOldDriver@" + this.streamManager.TraceType, key.ToString(), oldDriver.SessionId.ToString());
oldDriver.Dispose();
}
var driver = new InboundSessionDriver(key, session, this.streamManager);
var addSuccess = this.streamManager.RuntimeResources.InboundSessionDrivers.TryAdd(key, driver);
Diagnostics.Assert(addSuccess, "Inbound session accepted twice");
FabricEvents.Events.AcceptInboundSession("Finish@" + this.streamManager.TraceType, key.ToString(), snapshot.SessionId.ToString());
return(true);
}
/// <summary>
/// Instantiates a Reliable Messaging Session for a given target partition.
/// </summary>
/// <param name="targetPartition">Target Partition</param>
/// <param name="timeout">Timeout for the operation to complete</param>
/// <returns>Reliable Messaging Session for the Target Partition</returns>
internal async Task <IReliableMessagingSession> SetupSession(PartitionKey targetPartition, TimeSpan timeout)
{
string endpoint = null;
var setupSuccess = false;
IReliableMessagingSession session = null;
var baseOpenAsyncTimeout = 0;
var remainingTimeout = timeout;
SessionDataSnapshot snapshot = null;
try
{
while (!setupSuccess)
{
var beforeGetEndpoint = DateTime.UtcNow;
// Get End point for this Partition
var partitionInfo = await this.GetEndpointAsync(targetPartition, remainingTimeout);
endpoint = partitionInfo.EndPoint;
FabricEvents.Events.SetupSession("Start@" + this.streamManager.TraceType, targetPartition.ToString(), endpoint, Guid.Empty.ToString());
var afterGetEndpoint = DateTime.UtcNow;
remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeGetEndpoint, afterGetEndpoint);
session = await this.CreateSession(targetPartition, endpoint);
var retry = false;
snapshot = session.GetDataSnapshot();
FabricEvents.Events.SetupSession(
"OpenSession.Start@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpoint,
snapshot.SessionId.ToString());
var openTask = session.OpenAsync(CancellationToken.None);
var beforeOpenWait = DateTime.UtcNow;
baseOpenAsyncTimeout = baseOpenAsyncTimeout >= StreamConstants.MaxDelayForValidSessionEndpoint
? StreamConstants.MaxDelayForValidSessionEndpoint
: baseOpenAsyncTimeout + StreamConstants.BaseDelayForValidSessionEndpoint;
var baseTimeSpan = new TimeSpan(0, 0, 0, 0, baseOpenAsyncTimeout);
var openAsyncTimeout = (baseTimeSpan < remainingTimeout) || (remainingTimeout == Timeout.InfiniteTimeSpan) ? baseTimeSpan : remainingTimeout;
// this timeout is a retry mechanism in case we are going to the wrong endpoint because the primary replica is moving
var openTaskCompleted = await TimeoutHandler.WaitWithDelay(openAsyncTimeout, openTask);
var afterOpenWait = DateTime.UtcNow;
remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeOpenWait, afterOpenWait);
if (openTaskCompleted && openTask.Exception == null)
{
FabricEvents.Events.SetupSession(
"OpenSession.Finish@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpoint,
snapshot.SessionId.ToString());
setupSuccess = true;
}
else if (openTask.Exception != null)
{
if (openTask.Exception.InnerException.GetType() == typeof(FabricException))
{
var fabricException = openTask.Exception.InnerException as FabricException;
if (fabricException.ErrorCode == FabricErrorCode.ReliableSessionManagerNotFound)
{
// Target partition primary may be moving to a different host process before session manager was created
retry = true;
}
}
else
{
Tracing.WriteExceptionAsError(
"SetupSession.OpenSession.UnrecoverableError",
openTask.Exception,
"{0} Target: {1}",
this.tracer,
targetPartition);
throw openTask.Exception;
}
}
else
{
FabricEvents.Events.SetupSession(
"OpenSession.RetryTimeout@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpoint,
snapshot.SessionId.ToString());
retry = true;
}
if (retry)
{
session.Abort();
session = null;
setupSuccess = false;
}
else
{
// control should never come here if we did not succeed; we should have thrown any non-retriable exception or set retry=true for a retriable exception
Diagnostics.Assert(setupSuccess, "SetupSession.CodingError {0} Target: {1}", this.tracer, targetPartition);
}
}
}
catch (TimeoutException)
{
if (session != null)
{
// Abort the session and leave a clean slate for the next time we try to set up the session: we don't want a ReliableSessionAlreadyExists exception
session.Abort();
session = null;
}
throw;
}
catch (FabricObjectClosedException)
{
// this can happen if the session manager was closed by a parallel thread due to role change
var endpointString = endpoint ?? "NullEndpoint";
var sessionIdString = snapshot == null ? "NoSessionCreated" : snapshot.SessionId.ToString();
FabricEvents.Events.SetupSession(
"ObjectClosed@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpointString,
sessionIdString);
throw;
}
catch (Exception e)
{
Tracing.WriteExceptionAsError("SetupSession.Failure", e, "Source: {0} Target: {1}", this.tracer, targetPartition);
Diagnostics.Assert(
false,
"{0} Unexpected exception {1} in SessionConnectionManager.SetupSession for Target: {2}",
this.tracer,
e.GetType(),
targetPartition);
}
Diagnostics.Assert(
session != null,
"{0} SessionConnectionManager.SetupSession returning null session for Target: {1}",
this.tracer,
targetPartition);
if (session != null)
{
snapshot = session.GetDataSnapshot();
Diagnostics.Assert(
snapshot.IsOpenForSend,
"SessionConnectionManager.SetupSession returning session that is not open for send for Source: {0} Target: {1}",
this.tracer,
targetPartition);
FabricEvents.Events.SetupSession(
"Finish@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpoint,
snapshot.SessionId.ToString());
}
return(session);
}