/// <summary>
/// Retry continuation.
/// </summary>
/// <param name="task"></param>
internal void QuotaExceededRetryContinuation(Task task)
{
if (task.Exception != null)
{
Tracing.WriteExceptionAsError("OutboundSessionDriver.QuotaExceededRetry.Failure", task.Exception, "{0}", this.tracer);
}
}
/// <summary>
/// Handler for session send() operation.
/// Process various exception use cases here.
/// </summary>
/// <param name="sendTask">Session Continuation task</param>
internal void SendContinuation(Task sendTask)
{
if (sendTask.Exception != null)
{
var realException = sendTask.Exception.InnerException;
if (realException is InvalidOperationException || realException is OperationCanceledException)
{
FabricEvents.Events.SessionSendFailure("SessionAborted@" + this.streamManager.TraceType, this.SessionId.ToString());
// the driver was Disposed usually due to partner reset, this driver is no longer valid: confirm that
var snapshot = this.outboundSession.GetDataSnapshot();
Diagnostics.Assert(
!snapshot.IsOpenForSend,
"{0} InvalidOperationException when outboundSession is open for Send in SendContinuation",
this.tracer);
}
else
{
Tracing.WriteExceptionAsError(
"SessionSend.Failure",
realException,
"{0} OutboundSessionDriver.SendContinuation encountered unexpected exception",
this.tracer);
Diagnostics.Assert(
false,
"{0} Unexpected exception {1} in SendContinuation",
this.tracer,
realException.GetType());
}
}
}
/// <summary>
/// Converts the InboundStreamStableParameters to byte array
/// </summary>
/// <returns>Byte array</returns>
public void ToByteArray(InboundStreamStableParameters value, BinaryWriter writer)
{
try
{
value.WriteBaseParameters(writer);
}
catch (Exception e)
{
Tracing.WriteExceptionAsError("InboundStreamStableParameters.ToByteArray", e, "Content: {0}", this);
}
}
/// <summary>
/// Converts the OutboundStreamStableParameters to byte array
/// </summary>
/// <returns>Byte array</returns>
public void ToByteArray(OutboundStreamStableParameters value, BinaryWriter writer)
{
try
{
value.WriteBaseParameters(writer);
writer.Write(value.MessageQuota);
writer.Write((int)value.AcceptanceResponse);
}
catch (Exception e)
{
Tracing.WriteExceptionAsError("OutboundStreamStableParameters.ToByteArray", e, "Content: {0}", this);
}
}
/// <summary>
/// Delete End point Info from Property Manager when role(primary) is shutdown.
/// </summary>
/// <param name="key">Partition Key to be deleted</param>
/// <returns />
internal async Task RevokeEnpointAsync(PartitionKey key)
{
var parentName = key.ServiceInstanceName;
var propertyName = key.PublishedPropertyName("TransportEndpoint");
var deleteSuccess = false;
while (!deleteSuccess)
{
try
{
await this.client.PropertyManager.DeletePropertyAsync(parentName, propertyName);
FabricEvents.Events.DeleteEndpointProperty("FoundAndDeleted@" + this.streamManager.TraceType, key.ToString());
deleteSuccess = true;
}
catch (FabricElementNotFoundException)
{
FabricEvents.Events.DeleteEndpointProperty("NotFound@" + this.streamManager.TraceType, key.ToString());
deleteSuccess = true;
}
catch (FabricTransientException)
{
FabricEvents.Events.DeleteEndpointProperty("TransientError.Retrying@" + this.streamManager.TraceType, key.ToString());
}
catch (TimeoutException)
{
FabricEvents.Events.DeleteEndpointProperty("Timeout.Retrying@" + this.streamManager.TraceType, key.ToString());
}
catch (OperationCanceledException)
{
FabricEvents.Events.DeleteEndpointProperty("Canceled.Retrying@" + this.streamManager.TraceType, key.ToString());
}
catch (FabricException fabricException)
{
if (fabricException.ErrorCode == FabricErrorCode.CommunicationError)
{
FabricEvents.Events.DeleteEndpointProperty("CommunicationError.Retrying@" + this.streamManager.TraceType, key.ToString());
}
else
{
Tracing.WriteExceptionAsError("RegisterEndpointAsync.DeletePropertyAsync.UnrecoverableError", fabricException, "{0}", this.tracer);
throw;
}
}
catch (Exception otherException)
{
Tracing.WriteExceptionAsError("RegisterEndpointAsync.DeletePropertyAsync.UnrecoverableError", otherException, "{0}", this.tracer);
throw;
}
}
}
/// <summary>
/// handler in lieu of await for StreamManager.CompleteDeleteStreamProtocol
/// </summary>
/// <param name="task"></param>
/// <param name="streamId"></param>
private void CompleteDeleteStreamProtocolContinuation(Task task, Guid streamId)
{
// TODO: exception handling
if (task.Exception != null)
{
if (task.Exception.InnerException is FabricObjectClosedException || task.Exception.InnerException is FabricNotPrimaryException)
{
Tracing.WriteExceptionAsWarning("Stream.CompleteDeleteStreamProtocol.Failure", task.Exception, "{0} StreamId: {1}", this.tracer, streamId);
}
else
{
Tracing.WriteExceptionAsError("Stream.CompleteDeleteStreamProtocol.Failure", task.Exception, "{0} StreamId: {1}", this.tracer, streamId);
}
}
}
/// <summary>
/// Handler in lieu of await for StreamManager.RestartPartnerStreamsAsync
/// </summary>
/// <param name="task"></param>
private void RestartPartnerStreamsAsyncContinuation(Task task)
{
// TODO: exception handling
if (task.Exception != null)
{
if (task.Exception.InnerException is FabricObjectClosedException || task.Exception.InnerException is FabricNotPrimaryException)
{
Tracing.WriteExceptionAsWarning("InboundSessionDriver.RestartPartnerStreamsAsync.Failure", task.Exception, "{0}", this.tracer);
}
else
{
Tracing.WriteExceptionAsError("InboundSessionDriver.RestartPartnerStreamsAsync.Failure", task.Exception, "{0}", this.tracer);
}
}
}
/// <summary>
/// Handler in lieu of await for StreamManager.CompleteOpenStreamProtocol
/// </summary>
/// <param name="task"></param>
private void CompleteOpenStreamProtocolContinuation(Task task)
{
// TODO: exception handling
if (task.Exception != null)
{
if (task.Exception.InnerException is FabricObjectClosedException || task.Exception.InnerException is FabricNotPrimaryException)
{
Tracing.WriteExceptionAsWarning("InboundSessionDriver.CompleteOpenStreamProtocol.Failure", task.Exception, "{0}", this.tracer);
}
else
{
Tracing.WriteExceptionAsError("InboundSessionDriver.CompleteOpenStreamProtocol.Failure", task.Exception, "{0}", this.tracer);
}
}
}
/// <summary>
/// Handler for messages received on the session, will also deal with exceptions that occur when the session is closed or aborted
/// </summary>
/// <param name="receiveTask"></param>
private void ReceiveContinuation(Task <IOperationData> receiveTask)
{
// Check for aborted session and other failures
if (receiveTask.Exception != null)
{
var realException = receiveTask.Exception.InnerException;
if (realException is InvalidOperationException || realException is OperationCanceledException)
{
FabricEvents.Events.SessionReceiveFailure("SessionAborted@" + this.traceType, this.SessionId.ToString());
}
else
{
Tracing.WriteExceptionAsError("InboundSessionDriver.ReceiveAsync.Failure", realException, "{0}", this.tracer);
Diagnostics.Assert(false, "{0} InboundSessionDriver.ReceiveAsync Unexpected Exception {1}", this.tracer, receiveTask.Exception.GetType());
}
}
else
{
var wireMessage = receiveTask.Result;
var inboundStreamMessage = new InboundBaseStreamWireMessage(wireMessage);
var kind = inboundStreamMessage.Kind;
// Check if Stream Manager is available for updates.
if (this.streamManager.NotWritable())
{
// drop the message
FabricEvents.Events.WireMessageReceived(
"DroppedNotWritable@" + this.traceType + "@" + this.SessionId.ToString(),
inboundStreamMessage.StreamIdentity.ToString(),
inboundStreamMessage.MessageSequenceNumber,
kind.ToString());
return;
}
FabricEvents.Events.WireMessageReceived(
this.traceType + "@" + this.SessionId.ToString(),
inboundStreamMessage.StreamIdentity.ToString(),
inboundStreamMessage.MessageSequenceNumber,
kind.ToString());
/*
* There are two basic approaches to dealing with messages here
* If the message is intended for a stream and the stream is not found it means we are in the middle of a restore process
* so we simply drop the message and build it into the protocol that it will be retried eventually
*
* If the message is intended for the stream manager to control restart or to drive stream lifecycle the stream manager
* will ensure it is in operating state and if not has a mechanism to wait for that state
*/
switch (kind)
{
case StreamWireProtocolMessageKind.ServiceData:
this.streamManager.MessageReceived(inboundStreamMessage)
.ContinueWith(antecedent => Task.Run(() => this.MessageReceivedContinuation(antecedent, inboundStreamMessage.StreamIdentity)));
break;
case StreamWireProtocolMessageKind.SequenceAck:
this.streamManager.AckReceived(inboundStreamMessage)
.ContinueWith(antecedent => Task.Run(() => this.AckReceivedContinuation(antecedent, inboundStreamMessage.StreamIdentity)));
break;
case StreamWireProtocolMessageKind.OpenStream:
this.streamManager.InboundStreamRequested(this.partnerKey, inboundStreamMessage)
.ContinueWith(this.InboundStreamRequestedContinuation);
break;
case StreamWireProtocolMessageKind.CloseStream:
this.streamManager.MessageReceived(inboundStreamMessage)
.ContinueWith(antecedent => Task.Run(() => this.MessageReceivedContinuation(antecedent, inboundStreamMessage.StreamIdentity)));
break;
case StreamWireProtocolMessageKind.OpenStreamResponse:
this.streamManager.CompleteOpenStreamProtocol(inboundStreamMessage)
.ContinueWith(this.CompleteOpenStreamProtocolContinuation);
break;
case StreamWireProtocolMessageKind.CloseStreamResponse:
this.streamManager.CompleteCloseStreamProtocol(inboundStreamMessage)
.ContinueWith(this.CompleteCloseStreamProtocolContinuation);
break;
case StreamWireProtocolMessageKind.ResetPartnerStreams:
// The StreamIdentity slot is actually carrying the Guid representing the era of the requester
this.streamManager.ResetPartnerStreamsAsync(inboundStreamMessage.StreamIdentity, this.partnerKey)
.ContinueWith(this.ResetPartnerStreamsAsyncContinuation);
break;
case StreamWireProtocolMessageKind.ResetPartnerStreamsResponse:
var resetResponseValue = new InboundResetPartnerStreamsResponseValue(inboundStreamMessage);
Diagnostics.Assert(
resetResponseValue.Response == StreamWireProtocolResponse.ResetPartnerStreamsCompleted,
"{0} Unexpected reset partner streams response",
this.tracer);
// The StreamIdentity slot is actually carrying the Guid representing the era when the ResetPartnerStreams request was sent
// We will only process the respone if the request was sent in the current era
if (inboundStreamMessage.StreamIdentity == this.streamManager.Era)
{
this.streamManager.RestartPartnerStreamsAsync(this.partnerKey, this.streamManager.RuntimeResources.OutboundStreams, false)
.ContinueWith(this.RestartPartnerStreamsAsyncContinuation);
}
break;
case StreamWireProtocolMessageKind.DeleteStream:
this.streamManager.DeleteInboundStreamRequested(this.partnerKey, inboundStreamMessage)
.ContinueWith(
antecedent => Task.Run(() => this.DeleteInboundStreamRequestedContinuation(antecedent, inboundStreamMessage.StreamIdentity)));
break;
case StreamWireProtocolMessageKind.DeleteStreamResponse:
this.streamManager.CompleteDeleteStreamProtocol(inboundStreamMessage)
.ContinueWith(
antecedent => Task.Run(() => this.CompleteDeleteStreamProtocolContinuation(antecedent, inboundStreamMessage.StreamIdentity)));
break;
default:
Diagnostics.Assert(
false,
"{0} Unknown message kind in InboundSessionDriver Kind-as-int: {1}",
this.tracer,
(int)inboundStreamMessage.Kind);
break;
}
// Restart the receive operation
Diagnostics.Assert(this.inboundSession != null, "InboundSessionDriver.ReceiveContinuation.RestartReceive found null session");
this.ReceiveOnSession();
}
}
/// <summary>
/// Send messages from the session (message) queue in a FIFO fashion.
/// </summary>
/// <returns></returns>
internal bool TryDrainSessionSendQueue()
{
bool barrierOpen;
var messageExists = true;
var sessionQuotaExceeded = false;
var driverDisposed = false;
// if we got through the barrier and exited because we found no message but the sessionQueue is not empty then there was a race that
// a Send operation lost so we circle back and try sending the message that the Send operation would have tried to send
do
{
// low impact lock - while sending a message to ensure we process sends synchronously in FIFO
var value = Interlocked.CompareExchange(ref this.drainQueueBarrier, 1, 0);
barrierOpen = value == 0;
if (barrierOpen)
{
try
{
while (!sessionQuotaExceeded && !driverDisposed && messageExists)
{
IOperationData nextMessage = null;
messageExists = this.sessionQueue.TryPeek(out nextMessage);
// IMPORTANT: This try-catch code relies on the fact that the COM-interop layer SYNCHRONOUSLY THROWS A FAULT from the BeginFunc OF SendAsync
// TODO: find a way to push handling of this transient fault down into the managed wrapper for sessions
// TODO: what happens if there is a session Abort? Or a failure of the partner?
if (messageExists)
{
try
{
// drop the Task returned since the only possible exceptions in the task results reflect session abort
// we will get a notification for session abort through the registered interface and deal with it there
this.outboundSession.SendAsync(nextMessage, CancellationToken.None).ContinueWith(this.SendContinuation);
}
catch (FabricException fabricException)
{
var errorCode = fabricException.ErrorCode;
if (errorCode == FabricErrorCode.ReliableSessionQuotaExceeded)
{
FabricEvents.Events.SessionSendFailure("QuotaExceeded@" + this.streamManager.TraceType, this.SessionId.ToString());
sessionQuotaExceeded = true;
}
}
catch (Exception e)
{
if (e is InvalidOperationException || e is OperationCanceledException)
{
FabricEvents.Events.SessionSendFailure("SessionAborted@" + this.streamManager.TraceType, this.SessionId.ToString());
// the driver was Disposed usually due to partner reset, this driver is no longer valid: confirm that
var snapshot = this.outboundSession.GetDataSnapshot();
Diagnostics.Assert(
!snapshot.IsOpenForSend,
"{0} InvalidOperationException when outboundSession is open for Send in TryDrainSessionSendQueue",
this.tracer);
driverDisposed = true;
}
else
{
Tracing.WriteExceptionAsError(
"SendAsync.Failure",
e,
"{0} OutboundSessionDriver.SendContinuation encountered unexpected exception",
this.tracer);
Diagnostics.Assert(
false,
"{0} Unexpected exception {1} in OutboundSessionDriver.SendContinuation",
this.tracer,
e.GetType());
}
}
// dequeue the message only if the message was successfully handed out to the session for send proc.
if (!sessionQuotaExceeded && !driverDisposed)
{
var popSuccess = this.sessionQueue.TryDequeue(out nextMessage);
Diagnostics.Assert(
popSuccess,
"{0} Unexpected missing message in OutboundSessionDriver.TryDrainSessionSendQueue",
this.tracer);
}
}
}
}
finally
{
value = Interlocked.CompareExchange(ref this.drainQueueBarrier, 0, 1);
Diagnostics.Assert(
value == 1,
"drainQueueBarrier was reset to 0 while inside critical section in OutboundSessionDriver.TryDrainSessionSendQueue");
}
}
} // Finally check if messages got added to the queue while we were busy processing the earlier message.
while (barrierOpen && !driverDisposed && !messageExists && this.sessionQueue.Count > 0);
return(sessionQuotaExceeded);
}
/// <summary>
/// Instantiates a Reliable Messaging Session for a given target partition.
/// </summary>
/// <param name="targetPartition">Target Partition</param>
/// <param name="timeout">Timeout for the operation to complete</param>
/// <returns>Reliable Messaging Session for the Target Partition</returns>
internal async Task <IReliableMessagingSession> SetupSession(PartitionKey targetPartition, TimeSpan timeout)
{
string endpoint = null;
var setupSuccess = false;
IReliableMessagingSession session = null;
var baseOpenAsyncTimeout = 0;
var remainingTimeout = timeout;
SessionDataSnapshot snapshot = null;
try
{
while (!setupSuccess)
{
var beforeGetEndpoint = DateTime.UtcNow;
// Get End point for this Partition
var partitionInfo = await this.GetEndpointAsync(targetPartition, remainingTimeout);
endpoint = partitionInfo.EndPoint;
FabricEvents.Events.SetupSession("Start@" + this.streamManager.TraceType, targetPartition.ToString(), endpoint, Guid.Empty.ToString());
var afterGetEndpoint = DateTime.UtcNow;
remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeGetEndpoint, afterGetEndpoint);
session = await this.CreateSession(targetPartition, endpoint);
var retry = false;
snapshot = session.GetDataSnapshot();
FabricEvents.Events.SetupSession(
"OpenSession.Start@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpoint,
snapshot.SessionId.ToString());
var openTask = session.OpenAsync(CancellationToken.None);
var beforeOpenWait = DateTime.UtcNow;
baseOpenAsyncTimeout = baseOpenAsyncTimeout >= StreamConstants.MaxDelayForValidSessionEndpoint
? StreamConstants.MaxDelayForValidSessionEndpoint
: baseOpenAsyncTimeout + StreamConstants.BaseDelayForValidSessionEndpoint;
var baseTimeSpan = new TimeSpan(0, 0, 0, 0, baseOpenAsyncTimeout);
var openAsyncTimeout = (baseTimeSpan < remainingTimeout) || (remainingTimeout == Timeout.InfiniteTimeSpan) ? baseTimeSpan : remainingTimeout;
// this timeout is a retry mechanism in case we are going to the wrong endpoint because the primary replica is moving
var openTaskCompleted = await TimeoutHandler.WaitWithDelay(openAsyncTimeout, openTask);
var afterOpenWait = DateTime.UtcNow;
remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeOpenWait, afterOpenWait);
if (openTaskCompleted && openTask.Exception == null)
{
FabricEvents.Events.SetupSession(
"OpenSession.Finish@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpoint,
snapshot.SessionId.ToString());
setupSuccess = true;
}
else if (openTask.Exception != null)
{
if (openTask.Exception.InnerException.GetType() == typeof(FabricException))
{
var fabricException = openTask.Exception.InnerException as FabricException;
if (fabricException.ErrorCode == FabricErrorCode.ReliableSessionManagerNotFound)
{
// Target partition primary may be moving to a different host process before session manager was created
retry = true;
}
}
else
{
Tracing.WriteExceptionAsError(
"SetupSession.OpenSession.UnrecoverableError",
openTask.Exception,
"{0} Target: {1}",
this.tracer,
targetPartition);
throw openTask.Exception;
}
}
else
{
FabricEvents.Events.SetupSession(
"OpenSession.RetryTimeout@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpoint,
snapshot.SessionId.ToString());
retry = true;
}
if (retry)
{
session.Abort();
session = null;
setupSuccess = false;
}
else
{
// control should never come here if we did not succeed; we should have thrown any non-retriable exception or set retry=true for a retriable exception
Diagnostics.Assert(setupSuccess, "SetupSession.CodingError {0} Target: {1}", this.tracer, targetPartition);
}
}
}
catch (TimeoutException)
{
if (session != null)
{
// Abort the session and leave a clean slate for the next time we try to set up the session: we don't want a ReliableSessionAlreadyExists exception
session.Abort();
session = null;
}
throw;
}
catch (FabricObjectClosedException)
{
// this can happen if the session manager was closed by a parallel thread due to role change
var endpointString = endpoint ?? "NullEndpoint";
var sessionIdString = snapshot == null ? "NoSessionCreated" : snapshot.SessionId.ToString();
FabricEvents.Events.SetupSession(
"ObjectClosed@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpointString,
sessionIdString);
throw;
}
catch (Exception e)
{
Tracing.WriteExceptionAsError("SetupSession.Failure", e, "Source: {0} Target: {1}", this.tracer, targetPartition);
Diagnostics.Assert(
false,
"{0} Unexpected exception {1} in SessionConnectionManager.SetupSession for Target: {2}",
this.tracer,
e.GetType(),
targetPartition);
}
Diagnostics.Assert(
session != null,
"{0} SessionConnectionManager.SetupSession returning null session for Target: {1}",
this.tracer,
targetPartition);
if (session != null)
{
snapshot = session.GetDataSnapshot();
Diagnostics.Assert(
snapshot.IsOpenForSend,
"SessionConnectionManager.SetupSession returning session that is not open for send for Source: {0} Target: {1}",
this.tracer,
targetPartition);
FabricEvents.Events.SetupSession(
"Finish@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpoint,
snapshot.SessionId.ToString());
}
return(session);
}
/// <summary>
/// Given a Partition and its endpoint, create a reliable session and return it.
/// Exceptions this may throw TimeoutException,FabricObjectClosedException
/// </summary>
/// <param name="targetPartition">Target Partition</param>
/// <param name="endpoint">Target End point</param>
/// <returns>Reliable Messaging Session object</returns>
private async Task <IReliableMessagingSession> CreateSession(PartitionKey targetPartition, string endpoint)
{
Task <IReliableMessagingSession> sessionCreateTask = null;
IReliableMessagingSession session;
// CreateOutboundSessionAsync is really synchronous at the implementation level, and will throw exceptions synchronously
try
{
FabricEvents.Events.SetupSession(
"CreateSession.Start@" + this.streamManager.TraceType,
targetPartition.ToString(),
endpoint,
Guid.Empty.ToString());
switch (targetPartition.Kind)
{
case PartitionKind.Singleton:
sessionCreateTask = this.sessionManager.CreateOutboundSessionAsync(
targetPartition.ServiceInstanceName,
endpoint,
CancellationToken.None);
break;
case PartitionKind.Numbered:
sessionCreateTask = this.sessionManager.CreateOutboundSessionAsync(
targetPartition.ServiceInstanceName,
targetPartition.PartitionRange.IntegerKeyLow,
endpoint,
CancellationToken.None);
break;
case PartitionKind.Named:
sessionCreateTask = this.sessionManager.CreateOutboundSessionAsync(
targetPartition.ServiceInstanceName,
targetPartition.PartitionName,
endpoint,
CancellationToken.None);
break;
}
// session creation is essentially synchronous so we won't try timeout on it
await sessionCreateTask;
session = sessionCreateTask.Result;
}
catch (FabricObjectClosedException)
{
// this can happen if the session manager was closed by a parallel thread due to role change
throw;
}
catch (Exception e)
{
if (e is FabricException)
{
var fabricException = e as FabricException;
Diagnostics.Assert(
fabricException.ErrorCode != FabricErrorCode.ReliableSessionAlreadyExists,
"{0} Unexpected duplicate session in setup session for Target: {1}",
this.tracer,
targetPartition);
}
Tracing.WriteExceptionAsError(
"SetupSession",
e,
"{0} Unexpected exception in SetupSession from CreateOutboundSessionAsync for Target: {1}",
this.tracer,
targetPartition);
throw;
}
return(session);
}
/// <summary>
/// Get the partition end point info for the given partition key
/// Exceptions this may throw TimeoutException, FabricObjectClosedException, more to be discovered
/// </summary>
/// <param name="key">Get end point for this given partition key</param>
/// <param name="timeout">Timeout for the operation to complete</param>
/// <returns>Partition Body</returns>
private async Task <PartitionInfo> GetEndpointAsync(PartitionKey key, TimeSpan timeout)
{
PartitionInfo endpoint = null;
var parentName = key.ServiceInstanceName;
var propertyName = key.PublishedPropertyName("TransportEndpoint");
var getEndpointDelay = 0;
var getCompleted = false;
// TODO: revisit the timeout behavior of GetPropertyAsync to clarify and adjust accordingly
var remainingTimeout = timeout;
FabricEvents.Events.GetEndpointProperty("Start@" + this.streamManager.TraceType, key.ToString(), "");
while (!getCompleted)
{
var beforeGetPropertyAsync = DateTime.UtcNow;
try
{
var property = await this.client.PropertyManager.GetPropertyAsync(parentName, propertyName);
endpoint = property.GetValue <byte[]>().Deserialize <PartitionInfo>();
getCompleted = true;
}
catch (FabricElementNotFoundException)
{
FabricEvents.Events.GetEndpointProperty("NotFound.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
catch (FabricTransientException)
{
FabricEvents.Events.GetEndpointProperty("TransientError.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
catch (TimeoutException)
{
FabricEvents.Events.GetEndpointProperty("Timeout.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
catch (OperationCanceledException)
{
FabricEvents.Events.GetEndpointProperty("Canceled.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
catch (FabricException fabricException)
{
if (fabricException.ErrorCode == FabricErrorCode.CommunicationError)
{
FabricEvents.Events.GetEndpointProperty("CommunicationError.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
else
{
Tracing.WriteExceptionAsError("RegisterEndpointAsync.GetPropertyAsync.UnrecoverableError", fabricException, "{0}", this.tracer);
throw;
}
}
catch (Exception otherException)
{
Tracing.WriteExceptionAsError("RegisterEndpointAsync.GetPropertyAsync.UnrecoverableError", otherException, "{0}", this.tracer);
throw;
}
if (endpoint == null)
{
var afterGetPropertyAsync = DateTime.UtcNow;
remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeGetPropertyAsync, afterGetPropertyAsync);
getEndpointDelay = getEndpointDelay >= StreamConstants.MaxDelayForValidSessionEndpoint
? StreamConstants.MaxDelayForValidSessionEndpoint
: getEndpointDelay + StreamConstants.BaseDelayForValidSessionEndpoint;
await Task.Delay(getEndpointDelay);
}
}
bool replaced;
// Update resolved end points cache
lock (this.resolvedEndpointsLock)
{
replaced = this.resolvedEndpoints.Remove(key);
this.resolvedEndpoints.Add(key, endpoint);
}
if (replaced)
{
FabricEvents.Events.GetEndpointProperty("FinishReplaced@" + this.streamManager.TraceType, key.ToString(), endpoint.ToString());
}
else
{
FabricEvents.Events.GetEndpointProperty("FinishNew@" + this.streamManager.TraceType, key.ToString(), endpoint.ToString());
}
return(endpoint);
}
//
/// <summary>
/// TODO: this may throw FabricObjectClosedException
/// Use naming to find the integer partition range for a numbered target partition given a single partition number target
/// </summary>
/// <param name="key">Partition Key to resolve</param>
/// <param name="timeout">Timeout for this operation to complete by</param>
/// <returns>Resolved (and in case of key Kind = numbered) normalized partition key</returns>
internal async Task <PartitionKey> ResolveAndNormalizeTargetPartition(PartitionKey key, TimeSpan timeout)
{
FabricEvents.Events.ResolveTargetPartition("Start@" + this.streamManager.TraceType, key.ToString(), "");
PartitionKey foundKey = null;
var found = this.GetNormalizedPartitionKey(key, out foundKey);
if (found)
{
FabricEvents.Events.ResolveTargetPartition("FoundCached@" + this.streamManager.TraceType, key.ToString(), foundKey.ToString());
return(foundKey);
}
var resolveSuccess = false;
ResolvedServicePartition resolvedPartition = null;
var resolveDelay = StreamConstants.BaseDelayForValidSessionEndpoint;
var remainingTimeout = timeout;
while (!resolveSuccess)
{
var beforeResolveAttempt = DateTime.UtcNow;
// TODO: ResolveServicePartitionAsync seems to throw TimeoutException in unexpected ways: investigate and fix this, leaving out the timeout param for now
try
{
switch (key.Kind)
{
case PartitionKind.Singleton:
resolvedPartition = await this.client.ServiceManager.ResolveServicePartitionAsync(key.ServiceInstanceName);
break;
case PartitionKind.Named:
resolvedPartition = await this.client.ServiceManager.ResolveServicePartitionAsync(key.ServiceInstanceName, key.PartitionName);
break;
case PartitionKind.Numbered:
resolvedPartition =
await this.client.ServiceManager.ResolveServicePartitionAsync(key.ServiceInstanceName, key.PartitionRange.IntegerKeyHigh);
break;
default:
Diagnostics.Assert(false, "{0} Unknown partition Kind {1} in ResolveAndNormalizeTargetPartition", this.tracer, key);
break;
}
resolveSuccess = true;
}
catch (FabricTransientException)
{
FabricEvents.Events.ResolveTargetPartition("TransientException.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
catch (TimeoutException)
{
FabricEvents.Events.ResolveTargetPartition("Timeout.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
catch (OperationCanceledException)
{
FabricEvents.Events.ResolveTargetPartition("Canceled.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
catch (FabricException fabricException)
{
if (fabricException.ErrorCode == FabricErrorCode.InvalidPartitionKey)
{
FabricEvents.Events.ResolveTargetPartition("KeyNotFound.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
else if (fabricException.ErrorCode == FabricErrorCode.CommunicationError)
{
FabricEvents.Events.ResolveTargetPartition("CommunicationError.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
else if (fabricException.ErrorCode == FabricErrorCode.ServiceNotFound)
{
FabricEvents.Events.ResolveTargetPartition("ServiceNotFound.Retrying@" + this.streamManager.TraceType, key.ToString(), "");
}
else
{
Tracing.WriteExceptionAsError(
"ResolveAndNormalizeTargetPartition.UnrecoverableError",
fabricException,
"{0} Target: {1}",
this.tracer,
key);
throw;
}
}
catch (Exception e)
{
Tracing.WriteExceptionAsError("ResolveAndNormalizeTargetPartition.UnrecoverableError", e, "{0} Target: {1}", this.tracer, key);
throw;
}
if (!resolveSuccess)
{
var afterResolveAttempt = DateTime.UtcNow;
remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeResolveAttempt, afterResolveAttempt);
// TODO: the behavior of ResolveServicePartitionAsync is unclear as far as timeout goes, this code needs fixing
if ((afterResolveAttempt - beforeResolveAttempt).TotalMilliseconds < resolveDelay)
{
await Task.Delay(resolveDelay);
}
var afterDelay = DateTime.UtcNow;
remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, afterResolveAttempt, afterDelay);
}
}
PartitionKey result;
// if we had a numbered partition stream target with a single number then we need to change the key to the range actually supported by the resolved target partition
if ((key.Kind == PartitionKind.Numbered) && (key.PartitionRange.IntegerKeyLow == key.PartitionRange.IntegerKeyHigh))
{
var partitionInfo = resolvedPartition.Info as Int64RangePartitionInformation;
result = new PartitionKey(key.ServiceInstanceName, partitionInfo.LowKey, partitionInfo.HighKey);
}
else
{
result = key;
}
// Add partition keys to the cache.
this.normalizedPartitionKeys.TryAdd(key, result);
FabricEvents.Events.ResolveTargetPartition("Finish@" + this.streamManager.TraceType, key.ToString(), result.ToString());
return(result);
}
/// <summary>
/// Register my(this instance of) service endpoint info (endpoint and era)
/// This method is called only once at startup when the service becomes primary
/// </summary>
/// <param name="key">Partition to register</param>
/// <param name="endpoint">Endpoint of the partition</param>
/// <param name="era">Era of the partition</param>
/// <returns></returns>
internal async Task RegisterPartitionAsync(PartitionKey key, string endpoint, Guid era)
{
var parentName = key.ServiceInstanceName;
var propertyName = key.PublishedPropertyName("TransportEndpoint");
var endPointInfo = new PartitionInfo(endpoint, era);
FabricEvents.Events.RegisterEndpointProperty("Start@" + this.streamManager.TraceType, key.ToString(), endPointInfo.ToString());
await this.RevokeEnpointAsync(key);
var putSuccess = false;
while (!putSuccess)
{
try
{
await this.client.PropertyManager.PutPropertyAsync(parentName, propertyName, endPointInfo.Serialize());
putSuccess = true;
}
catch (FabricTransientException)
{
FabricEvents.Events.RegisterEndpointProperty(
"TransientException.Retrying@" + this.streamManager.TraceType,
key.ToString(),
endPointInfo.ToString());
}
catch (TimeoutException)
{
FabricEvents.Events.RegisterEndpointProperty(
"Timeout.Retrying@" + this.streamManager.TraceType,
key.ToString(),
endPointInfo.ToString());
}
catch (OperationCanceledException)
{
FabricEvents.Events.RegisterEndpointProperty(
"Canceled.Retrying@" + this.streamManager.TraceType,
key.ToString(),
endPointInfo.ToString());
}
catch (FabricException fabricException)
{
if (fabricException.ErrorCode == FabricErrorCode.CommunicationError)
{
FabricEvents.Events.RegisterEndpointProperty(
"CommunicationError.Retrying@" + this.streamManager.TraceType,
key.ToString(),
endPointInfo.ToString());
}
else
{
Tracing.WriteExceptionAsError("RegisterEndpointAsync.PutPropertyAsync.UnrecoverableError", fabricException, "{0}", this.tracer);
throw;
}
}
catch (Exception otherException)
{
Tracing.WriteExceptionAsError("RegisterEndpointAsync.PutPropertyAsync.UnrecoverableError", otherException, "{0}", this.tracer);
throw;
}
}
FabricEvents.Events.RegisterEndpointProperty("Finish@" + this.streamManager.TraceType, key.ToString(), endPointInfo.ToString());
}
// Hides the base SetResult method which we need to override
// This method is called when the message is actually delivered to a ReceiveAsync call
// The update to the CloseMessageSequenceNumber property of Inbound is transactional
internal async Task SetResult(InboundBaseStreamWireMessage message)
{
var closeSeqNumber = this.stream.CloseMessageSequenceNumber;
if (message.Kind == StreamWireProtocolMessageKind.CloseStream)
{
// The close message can be a repeat in recovery cases, but the close sequence number must be unique
// It is also possible that the transaction surrounding this ReceiveAsync will abort and the message
// will not in fact be consumed -- and will be received again, but the close sequence number must be unique
Diagnostics.Assert(
closeSeqNumber == StreamConstants.InitialValueOfLastSequenceNumberInStream || closeSeqNumber == message.MessageSequenceNumber,
"{0} TraceId::{1} encountered multiple close sequence number values {2} and {3}",
this.stream.Tracer,
this.traceId,
closeSeqNumber,
message.MessageSequenceNumber);
try
{
// this method is idempotent; the close sequence number for a stream is invariant
await this.stream.SetCloseMessageSequenceNumber(message.MessageSequenceNumber, this.transactionContext);
await this.stream.CloseInboundStream(this.transactionContext);
}
catch (Exception e)
{
if (e is FabricObjectClosedException)
{
FabricEvents.Events.DataMessageDelivery(
string.Format(CultureInfo.InvariantCulture, "ObjectClosed@{0}{1}", this.traceId, this.stream.TraceType),
this.stream.StreamIdentity.ToString(),
message.MessageSequenceNumber,
this.transactionContext.Id.ToString());
this.TrySetException(e);
return;
}
if (e is FabricNotPrimaryException)
{
FabricEvents.Events.DataMessageDelivery(
string.Format(CultureInfo.InvariantCulture, "NotPrimary@{0}{1}", this.traceId, this.stream.TraceType),
this.stream.StreamIdentity.ToString(),
message.MessageSequenceNumber,
this.transactionContext.Id.ToString());
this.TrySetException(e);
return;
}
if (e is FabricNotReadableException)
{
FabricEvents.Events.DataMessageDelivery(
string.Format(CultureInfo.InvariantCulture, "NotReadable@{0}{1}", this.traceId, this.stream.TraceType),
this.stream.StreamIdentity.ToString(),
message.MessageSequenceNumber,
this.transactionContext.Id.ToString());
this.TrySetException(e);
return;
}
Tracing.WriteExceptionAsError(
"DataMessageDelivery.Failure",
e,
"Partition::Replica {0} TraceId: {1} MessageNumber: {2} TransactionId: {3}",
this.stream.TraceType,
this.traceId,
message.MessageSequenceNumber,
this.transactionContext.Id);
Diagnostics.Assert(
false,
"Unexpected Exception In {0} TraceId: {1} MessageNumber: {2} TransactionId: {3}",
this.stream.TraceType,
this.traceId,
message.MessageSequenceNumber,
this.transactionContext.Id);
}
}
else
{
Diagnostics.Assert(
closeSeqNumber == StreamConstants.InitialValueOfLastSequenceNumberInStream || closeSeqNumber > message.MessageSequenceNumber,
"{0} received payload message with sequence number {1} which is greater than the close sequence number {2}",
this.stream.Tracer,
message.MessageSequenceNumber,
closeSeqNumber);
}
var setResultSuccess = this.TrySetResult(message);
if (setResultSuccess)
{
FabricEvents.Events.DataMessageDelivery(
"Success::" + this.traceId + "@" + this.stream.TraceType,
this.stream.StreamIdentity.ToString(),
message.MessageSequenceNumber,
this.transactionContext.Id.ToString());
}
else
{
FabricEvents.Events.DataMessageDelivery(
"Failure::" + this.traceId + "@" + this.stream.TraceType,
this.stream.StreamIdentity.ToString(),
message.MessageSequenceNumber,
this.transactionContext.Id.ToString());
}
}
