/// <summary> /// Retry continuation. /// </summary> /// <param name="task"></param> internal void QuotaExceededRetryContinuation(Task task) { if (task.Exception != null) { Tracing.WriteExceptionAsError("OutboundSessionDriver.QuotaExceededRetry.Failure", task.Exception, "{0}", this.tracer); } }
/// <summary> /// Handler for session send() operation. /// Process various exception use cases here. /// </summary> /// <param name="sendTask">Session Continuation task</param> internal void SendContinuation(Task sendTask) { if (sendTask.Exception != null) { var realException = sendTask.Exception.InnerException; if (realException is InvalidOperationException || realException is OperationCanceledException) { FabricEvents.Events.SessionSendFailure("SessionAborted@" + this.streamManager.TraceType, this.SessionId.ToString()); // the driver was Disposed usually due to partner reset, this driver is no longer valid: confirm that var snapshot = this.outboundSession.GetDataSnapshot(); Diagnostics.Assert( !snapshot.IsOpenForSend, "{0} InvalidOperationException when outboundSession is open for Send in SendContinuation", this.tracer); } else { Tracing.WriteExceptionAsError( "SessionSend.Failure", realException, "{0} OutboundSessionDriver.SendContinuation encountered unexpected exception", this.tracer); Diagnostics.Assert( false, "{0} Unexpected exception {1} in SendContinuation", this.tracer, realException.GetType()); } } }
/// <summary> /// Converts the InboundStreamStableParameters to byte array /// </summary> /// <returns>Byte array</returns> public void ToByteArray(InboundStreamStableParameters value, BinaryWriter writer) { try { value.WriteBaseParameters(writer); } catch (Exception e) { Tracing.WriteExceptionAsError("InboundStreamStableParameters.ToByteArray", e, "Content: {0}", this); } }
/// <summary> /// Converts the OutboundStreamStableParameters to byte array /// </summary> /// <returns>Byte array</returns> public void ToByteArray(OutboundStreamStableParameters value, BinaryWriter writer) { try { value.WriteBaseParameters(writer); writer.Write(value.MessageQuota); writer.Write((int)value.AcceptanceResponse); } catch (Exception e) { Tracing.WriteExceptionAsError("OutboundStreamStableParameters.ToByteArray", e, "Content: {0}", this); } }
/// <summary> /// Delete End point Info from Property Manager when role(primary) is shutdown. /// </summary> /// <param name="key">Partition Key to be deleted</param> /// <returns /> internal async Task RevokeEnpointAsync(PartitionKey key) { var parentName = key.ServiceInstanceName; var propertyName = key.PublishedPropertyName("TransportEndpoint"); var deleteSuccess = false; while (!deleteSuccess) { try { await this.client.PropertyManager.DeletePropertyAsync(parentName, propertyName); FabricEvents.Events.DeleteEndpointProperty("FoundAndDeleted@" + this.streamManager.TraceType, key.ToString()); deleteSuccess = true; } catch (FabricElementNotFoundException) { FabricEvents.Events.DeleteEndpointProperty("NotFound@" + this.streamManager.TraceType, key.ToString()); deleteSuccess = true; } catch (FabricTransientException) { FabricEvents.Events.DeleteEndpointProperty("TransientError.Retrying@" + this.streamManager.TraceType, key.ToString()); } catch (TimeoutException) { FabricEvents.Events.DeleteEndpointProperty("Timeout.Retrying@" + this.streamManager.TraceType, key.ToString()); } catch (OperationCanceledException) { FabricEvents.Events.DeleteEndpointProperty("Canceled.Retrying@" + this.streamManager.TraceType, key.ToString()); } catch (FabricException fabricException) { if (fabricException.ErrorCode == FabricErrorCode.CommunicationError) { FabricEvents.Events.DeleteEndpointProperty("CommunicationError.Retrying@" + this.streamManager.TraceType, key.ToString()); } else { Tracing.WriteExceptionAsError("RegisterEndpointAsync.DeletePropertyAsync.UnrecoverableError", fabricException, "{0}", this.tracer); throw; } } catch (Exception otherException) { Tracing.WriteExceptionAsError("RegisterEndpointAsync.DeletePropertyAsync.UnrecoverableError", otherException, "{0}", this.tracer); throw; } } }
/// <summary> /// handler in lieu of await for StreamManager.CompleteDeleteStreamProtocol /// </summary> /// <param name="task"></param> /// <param name="streamId"></param> private void CompleteDeleteStreamProtocolContinuation(Task task, Guid streamId) { // TODO: exception handling if (task.Exception != null) { if (task.Exception.InnerException is FabricObjectClosedException || task.Exception.InnerException is FabricNotPrimaryException) { Tracing.WriteExceptionAsWarning("Stream.CompleteDeleteStreamProtocol.Failure", task.Exception, "{0} StreamId: {1}", this.tracer, streamId); } else { Tracing.WriteExceptionAsError("Stream.CompleteDeleteStreamProtocol.Failure", task.Exception, "{0} StreamId: {1}", this.tracer, streamId); } } }
/// <summary> /// Handler in lieu of await for StreamManager.RestartPartnerStreamsAsync /// </summary> /// <param name="task"></param> private void RestartPartnerStreamsAsyncContinuation(Task task) { // TODO: exception handling if (task.Exception != null) { if (task.Exception.InnerException is FabricObjectClosedException || task.Exception.InnerException is FabricNotPrimaryException) { Tracing.WriteExceptionAsWarning("InboundSessionDriver.RestartPartnerStreamsAsync.Failure", task.Exception, "{0}", this.tracer); } else { Tracing.WriteExceptionAsError("InboundSessionDriver.RestartPartnerStreamsAsync.Failure", task.Exception, "{0}", this.tracer); } } }
/// <summary> /// Handler in lieu of await for StreamManager.CompleteOpenStreamProtocol /// </summary> /// <param name="task"></param> private void CompleteOpenStreamProtocolContinuation(Task task) { // TODO: exception handling if (task.Exception != null) { if (task.Exception.InnerException is FabricObjectClosedException || task.Exception.InnerException is FabricNotPrimaryException) { Tracing.WriteExceptionAsWarning("InboundSessionDriver.CompleteOpenStreamProtocol.Failure", task.Exception, "{0}", this.tracer); } else { Tracing.WriteExceptionAsError("InboundSessionDriver.CompleteOpenStreamProtocol.Failure", task.Exception, "{0}", this.tracer); } } }
/// <summary> /// Handler for messages received on the session, will also deal with exceptions that occur when the session is closed or aborted /// </summary> /// <param name="receiveTask"></param> private void ReceiveContinuation(Task <IOperationData> receiveTask) { // Check for aborted session and other failures if (receiveTask.Exception != null) { var realException = receiveTask.Exception.InnerException; if (realException is InvalidOperationException || realException is OperationCanceledException) { FabricEvents.Events.SessionReceiveFailure("SessionAborted@" + this.traceType, this.SessionId.ToString()); } else { Tracing.WriteExceptionAsError("InboundSessionDriver.ReceiveAsync.Failure", realException, "{0}", this.tracer); Diagnostics.Assert(false, "{0} InboundSessionDriver.ReceiveAsync Unexpected Exception {1}", this.tracer, receiveTask.Exception.GetType()); } } else { var wireMessage = receiveTask.Result; var inboundStreamMessage = new InboundBaseStreamWireMessage(wireMessage); var kind = inboundStreamMessage.Kind; // Check if Stream Manager is available for updates. if (this.streamManager.NotWritable()) { // drop the message FabricEvents.Events.WireMessageReceived( "DroppedNotWritable@" + this.traceType + "@" + this.SessionId.ToString(), inboundStreamMessage.StreamIdentity.ToString(), inboundStreamMessage.MessageSequenceNumber, kind.ToString()); return; } FabricEvents.Events.WireMessageReceived( this.traceType + "@" + this.SessionId.ToString(), inboundStreamMessage.StreamIdentity.ToString(), inboundStreamMessage.MessageSequenceNumber, kind.ToString()); /* * There are two basic approaches to dealing with messages here * If the message is intended for a stream and the stream is not found it means we are in the middle of a restore process * so we simply drop the message and build it into the protocol that it will be retried eventually * * If the message is intended for the stream manager to control restart or to drive stream lifecycle the stream manager * will ensure it is in operating state and if not has a mechanism to wait for that state */ switch (kind) { case StreamWireProtocolMessageKind.ServiceData: this.streamManager.MessageReceived(inboundStreamMessage) .ContinueWith(antecedent => Task.Run(() => this.MessageReceivedContinuation(antecedent, inboundStreamMessage.StreamIdentity))); break; case StreamWireProtocolMessageKind.SequenceAck: this.streamManager.AckReceived(inboundStreamMessage) .ContinueWith(antecedent => Task.Run(() => this.AckReceivedContinuation(antecedent, inboundStreamMessage.StreamIdentity))); break; case StreamWireProtocolMessageKind.OpenStream: this.streamManager.InboundStreamRequested(this.partnerKey, inboundStreamMessage) .ContinueWith(this.InboundStreamRequestedContinuation); break; case StreamWireProtocolMessageKind.CloseStream: this.streamManager.MessageReceived(inboundStreamMessage) .ContinueWith(antecedent => Task.Run(() => this.MessageReceivedContinuation(antecedent, inboundStreamMessage.StreamIdentity))); break; case StreamWireProtocolMessageKind.OpenStreamResponse: this.streamManager.CompleteOpenStreamProtocol(inboundStreamMessage) .ContinueWith(this.CompleteOpenStreamProtocolContinuation); break; case StreamWireProtocolMessageKind.CloseStreamResponse: this.streamManager.CompleteCloseStreamProtocol(inboundStreamMessage) .ContinueWith(this.CompleteCloseStreamProtocolContinuation); break; case StreamWireProtocolMessageKind.ResetPartnerStreams: // The StreamIdentity slot is actually carrying the Guid representing the era of the requester this.streamManager.ResetPartnerStreamsAsync(inboundStreamMessage.StreamIdentity, this.partnerKey) .ContinueWith(this.ResetPartnerStreamsAsyncContinuation); break; case StreamWireProtocolMessageKind.ResetPartnerStreamsResponse: var resetResponseValue = new InboundResetPartnerStreamsResponseValue(inboundStreamMessage); Diagnostics.Assert( resetResponseValue.Response == StreamWireProtocolResponse.ResetPartnerStreamsCompleted, "{0} Unexpected reset partner streams response", this.tracer); // The StreamIdentity slot is actually carrying the Guid representing the era when the ResetPartnerStreams request was sent // We will only process the respone if the request was sent in the current era if (inboundStreamMessage.StreamIdentity == this.streamManager.Era) { this.streamManager.RestartPartnerStreamsAsync(this.partnerKey, this.streamManager.RuntimeResources.OutboundStreams, false) .ContinueWith(this.RestartPartnerStreamsAsyncContinuation); } break; case StreamWireProtocolMessageKind.DeleteStream: this.streamManager.DeleteInboundStreamRequested(this.partnerKey, inboundStreamMessage) .ContinueWith( antecedent => Task.Run(() => this.DeleteInboundStreamRequestedContinuation(antecedent, inboundStreamMessage.StreamIdentity))); break; case StreamWireProtocolMessageKind.DeleteStreamResponse: this.streamManager.CompleteDeleteStreamProtocol(inboundStreamMessage) .ContinueWith( antecedent => Task.Run(() => this.CompleteDeleteStreamProtocolContinuation(antecedent, inboundStreamMessage.StreamIdentity))); break; default: Diagnostics.Assert( false, "{0} Unknown message kind in InboundSessionDriver Kind-as-int: {1}", this.tracer, (int)inboundStreamMessage.Kind); break; } // Restart the receive operation Diagnostics.Assert(this.inboundSession != null, "InboundSessionDriver.ReceiveContinuation.RestartReceive found null session"); this.ReceiveOnSession(); } }
/// <summary> /// Send messages from the session (message) queue in a FIFO fashion. /// </summary> /// <returns></returns> internal bool TryDrainSessionSendQueue() { bool barrierOpen; var messageExists = true; var sessionQuotaExceeded = false; var driverDisposed = false; // if we got through the barrier and exited because we found no message but the sessionQueue is not empty then there was a race that // a Send operation lost so we circle back and try sending the message that the Send operation would have tried to send do { // low impact lock - while sending a message to ensure we process sends synchronously in FIFO var value = Interlocked.CompareExchange(ref this.drainQueueBarrier, 1, 0); barrierOpen = value == 0; if (barrierOpen) { try { while (!sessionQuotaExceeded && !driverDisposed && messageExists) { IOperationData nextMessage = null; messageExists = this.sessionQueue.TryPeek(out nextMessage); // IMPORTANT: This try-catch code relies on the fact that the COM-interop layer SYNCHRONOUSLY THROWS A FAULT from the BeginFunc OF SendAsync // TODO: find a way to push handling of this transient fault down into the managed wrapper for sessions // TODO: what happens if there is a session Abort? Or a failure of the partner? if (messageExists) { try { // drop the Task returned since the only possible exceptions in the task results reflect session abort // we will get a notification for session abort through the registered interface and deal with it there this.outboundSession.SendAsync(nextMessage, CancellationToken.None).ContinueWith(this.SendContinuation); } catch (FabricException fabricException) { var errorCode = fabricException.ErrorCode; if (errorCode == FabricErrorCode.ReliableSessionQuotaExceeded) { FabricEvents.Events.SessionSendFailure("QuotaExceeded@" + this.streamManager.TraceType, this.SessionId.ToString()); sessionQuotaExceeded = true; } } catch (Exception e) { if (e is InvalidOperationException || e is OperationCanceledException) { FabricEvents.Events.SessionSendFailure("SessionAborted@" + this.streamManager.TraceType, this.SessionId.ToString()); // the driver was Disposed usually due to partner reset, this driver is no longer valid: confirm that var snapshot = this.outboundSession.GetDataSnapshot(); Diagnostics.Assert( !snapshot.IsOpenForSend, "{0} InvalidOperationException when outboundSession is open for Send in TryDrainSessionSendQueue", this.tracer); driverDisposed = true; } else { Tracing.WriteExceptionAsError( "SendAsync.Failure", e, "{0} OutboundSessionDriver.SendContinuation encountered unexpected exception", this.tracer); Diagnostics.Assert( false, "{0} Unexpected exception {1} in OutboundSessionDriver.SendContinuation", this.tracer, e.GetType()); } } // dequeue the message only if the message was successfully handed out to the session for send proc. if (!sessionQuotaExceeded && !driverDisposed) { var popSuccess = this.sessionQueue.TryDequeue(out nextMessage); Diagnostics.Assert( popSuccess, "{0} Unexpected missing message in OutboundSessionDriver.TryDrainSessionSendQueue", this.tracer); } } } } finally { value = Interlocked.CompareExchange(ref this.drainQueueBarrier, 0, 1); Diagnostics.Assert( value == 1, "drainQueueBarrier was reset to 0 while inside critical section in OutboundSessionDriver.TryDrainSessionSendQueue"); } } } // Finally check if messages got added to the queue while we were busy processing the earlier message. while (barrierOpen && !driverDisposed && !messageExists && this.sessionQueue.Count > 0); return(sessionQuotaExceeded); }
/// <summary> /// Instantiates a Reliable Messaging Session for a given target partition. /// </summary> /// <param name="targetPartition">Target Partition</param> /// <param name="timeout">Timeout for the operation to complete</param> /// <returns>Reliable Messaging Session for the Target Partition</returns> internal async Task <IReliableMessagingSession> SetupSession(PartitionKey targetPartition, TimeSpan timeout) { string endpoint = null; var setupSuccess = false; IReliableMessagingSession session = null; var baseOpenAsyncTimeout = 0; var remainingTimeout = timeout; SessionDataSnapshot snapshot = null; try { while (!setupSuccess) { var beforeGetEndpoint = DateTime.UtcNow; // Get End point for this Partition var partitionInfo = await this.GetEndpointAsync(targetPartition, remainingTimeout); endpoint = partitionInfo.EndPoint; FabricEvents.Events.SetupSession("Start@" + this.streamManager.TraceType, targetPartition.ToString(), endpoint, Guid.Empty.ToString()); var afterGetEndpoint = DateTime.UtcNow; remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeGetEndpoint, afterGetEndpoint); session = await this.CreateSession(targetPartition, endpoint); var retry = false; snapshot = session.GetDataSnapshot(); FabricEvents.Events.SetupSession( "OpenSession.Start@" + this.streamManager.TraceType, targetPartition.ToString(), endpoint, snapshot.SessionId.ToString()); var openTask = session.OpenAsync(CancellationToken.None); var beforeOpenWait = DateTime.UtcNow; baseOpenAsyncTimeout = baseOpenAsyncTimeout >= StreamConstants.MaxDelayForValidSessionEndpoint ? StreamConstants.MaxDelayForValidSessionEndpoint : baseOpenAsyncTimeout + StreamConstants.BaseDelayForValidSessionEndpoint; var baseTimeSpan = new TimeSpan(0, 0, 0, 0, baseOpenAsyncTimeout); var openAsyncTimeout = (baseTimeSpan < remainingTimeout) || (remainingTimeout == Timeout.InfiniteTimeSpan) ? baseTimeSpan : remainingTimeout; // this timeout is a retry mechanism in case we are going to the wrong endpoint because the primary replica is moving var openTaskCompleted = await TimeoutHandler.WaitWithDelay(openAsyncTimeout, openTask); var afterOpenWait = DateTime.UtcNow; remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeOpenWait, afterOpenWait); if (openTaskCompleted && openTask.Exception == null) { FabricEvents.Events.SetupSession( "OpenSession.Finish@" + this.streamManager.TraceType, targetPartition.ToString(), endpoint, snapshot.SessionId.ToString()); setupSuccess = true; } else if (openTask.Exception != null) { if (openTask.Exception.InnerException.GetType() == typeof(FabricException)) { var fabricException = openTask.Exception.InnerException as FabricException; if (fabricException.ErrorCode == FabricErrorCode.ReliableSessionManagerNotFound) { // Target partition primary may be moving to a different host process before session manager was created retry = true; } } else { Tracing.WriteExceptionAsError( "SetupSession.OpenSession.UnrecoverableError", openTask.Exception, "{0} Target: {1}", this.tracer, targetPartition); throw openTask.Exception; } } else { FabricEvents.Events.SetupSession( "OpenSession.RetryTimeout@" + this.streamManager.TraceType, targetPartition.ToString(), endpoint, snapshot.SessionId.ToString()); retry = true; } if (retry) { session.Abort(); session = null; setupSuccess = false; } else { // control should never come here if we did not succeed; we should have thrown any non-retriable exception or set retry=true for a retriable exception Diagnostics.Assert(setupSuccess, "SetupSession.CodingError {0} Target: {1}", this.tracer, targetPartition); } } } catch (TimeoutException) { if (session != null) { // Abort the session and leave a clean slate for the next time we try to set up the session: we don't want a ReliableSessionAlreadyExists exception session.Abort(); session = null; } throw; } catch (FabricObjectClosedException) { // this can happen if the session manager was closed by a parallel thread due to role change var endpointString = endpoint ?? "NullEndpoint"; var sessionIdString = snapshot == null ? "NoSessionCreated" : snapshot.SessionId.ToString(); FabricEvents.Events.SetupSession( "ObjectClosed@" + this.streamManager.TraceType, targetPartition.ToString(), endpointString, sessionIdString); throw; } catch (Exception e) { Tracing.WriteExceptionAsError("SetupSession.Failure", e, "Source: {0} Target: {1}", this.tracer, targetPartition); Diagnostics.Assert( false, "{0} Unexpected exception {1} in SessionConnectionManager.SetupSession for Target: {2}", this.tracer, e.GetType(), targetPartition); } Diagnostics.Assert( session != null, "{0} SessionConnectionManager.SetupSession returning null session for Target: {1}", this.tracer, targetPartition); if (session != null) { snapshot = session.GetDataSnapshot(); Diagnostics.Assert( snapshot.IsOpenForSend, "SessionConnectionManager.SetupSession returning session that is not open for send for Source: {0} Target: {1}", this.tracer, targetPartition); FabricEvents.Events.SetupSession( "Finish@" + this.streamManager.TraceType, targetPartition.ToString(), endpoint, snapshot.SessionId.ToString()); } return(session); }
/// <summary> /// Given a Partition and its endpoint, create a reliable session and return it. /// Exceptions this may throw TimeoutException,FabricObjectClosedException /// </summary> /// <param name="targetPartition">Target Partition</param> /// <param name="endpoint">Target End point</param> /// <returns>Reliable Messaging Session object</returns> private async Task <IReliableMessagingSession> CreateSession(PartitionKey targetPartition, string endpoint) { Task <IReliableMessagingSession> sessionCreateTask = null; IReliableMessagingSession session; // CreateOutboundSessionAsync is really synchronous at the implementation level, and will throw exceptions synchronously try { FabricEvents.Events.SetupSession( "CreateSession.Start@" + this.streamManager.TraceType, targetPartition.ToString(), endpoint, Guid.Empty.ToString()); switch (targetPartition.Kind) { case PartitionKind.Singleton: sessionCreateTask = this.sessionManager.CreateOutboundSessionAsync( targetPartition.ServiceInstanceName, endpoint, CancellationToken.None); break; case PartitionKind.Numbered: sessionCreateTask = this.sessionManager.CreateOutboundSessionAsync( targetPartition.ServiceInstanceName, targetPartition.PartitionRange.IntegerKeyLow, endpoint, CancellationToken.None); break; case PartitionKind.Named: sessionCreateTask = this.sessionManager.CreateOutboundSessionAsync( targetPartition.ServiceInstanceName, targetPartition.PartitionName, endpoint, CancellationToken.None); break; } // session creation is essentially synchronous so we won't try timeout on it await sessionCreateTask; session = sessionCreateTask.Result; } catch (FabricObjectClosedException) { // this can happen if the session manager was closed by a parallel thread due to role change throw; } catch (Exception e) { if (e is FabricException) { var fabricException = e as FabricException; Diagnostics.Assert( fabricException.ErrorCode != FabricErrorCode.ReliableSessionAlreadyExists, "{0} Unexpected duplicate session in setup session for Target: {1}", this.tracer, targetPartition); } Tracing.WriteExceptionAsError( "SetupSession", e, "{0} Unexpected exception in SetupSession from CreateOutboundSessionAsync for Target: {1}", this.tracer, targetPartition); throw; } return(session); }
/// <summary> /// Get the partition end point info for the given partition key /// Exceptions this may throw TimeoutException, FabricObjectClosedException, more to be discovered /// </summary> /// <param name="key">Get end point for this given partition key</param> /// <param name="timeout">Timeout for the operation to complete</param> /// <returns>Partition Body</returns> private async Task <PartitionInfo> GetEndpointAsync(PartitionKey key, TimeSpan timeout) { PartitionInfo endpoint = null; var parentName = key.ServiceInstanceName; var propertyName = key.PublishedPropertyName("TransportEndpoint"); var getEndpointDelay = 0; var getCompleted = false; // TODO: revisit the timeout behavior of GetPropertyAsync to clarify and adjust accordingly var remainingTimeout = timeout; FabricEvents.Events.GetEndpointProperty("Start@" + this.streamManager.TraceType, key.ToString(), ""); while (!getCompleted) { var beforeGetPropertyAsync = DateTime.UtcNow; try { var property = await this.client.PropertyManager.GetPropertyAsync(parentName, propertyName); endpoint = property.GetValue <byte[]>().Deserialize <PartitionInfo>(); getCompleted = true; } catch (FabricElementNotFoundException) { FabricEvents.Events.GetEndpointProperty("NotFound.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } catch (FabricTransientException) { FabricEvents.Events.GetEndpointProperty("TransientError.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } catch (TimeoutException) { FabricEvents.Events.GetEndpointProperty("Timeout.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } catch (OperationCanceledException) { FabricEvents.Events.GetEndpointProperty("Canceled.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } catch (FabricException fabricException) { if (fabricException.ErrorCode == FabricErrorCode.CommunicationError) { FabricEvents.Events.GetEndpointProperty("CommunicationError.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } else { Tracing.WriteExceptionAsError("RegisterEndpointAsync.GetPropertyAsync.UnrecoverableError", fabricException, "{0}", this.tracer); throw; } } catch (Exception otherException) { Tracing.WriteExceptionAsError("RegisterEndpointAsync.GetPropertyAsync.UnrecoverableError", otherException, "{0}", this.tracer); throw; } if (endpoint == null) { var afterGetPropertyAsync = DateTime.UtcNow; remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeGetPropertyAsync, afterGetPropertyAsync); getEndpointDelay = getEndpointDelay >= StreamConstants.MaxDelayForValidSessionEndpoint ? StreamConstants.MaxDelayForValidSessionEndpoint : getEndpointDelay + StreamConstants.BaseDelayForValidSessionEndpoint; await Task.Delay(getEndpointDelay); } } bool replaced; // Update resolved end points cache lock (this.resolvedEndpointsLock) { replaced = this.resolvedEndpoints.Remove(key); this.resolvedEndpoints.Add(key, endpoint); } if (replaced) { FabricEvents.Events.GetEndpointProperty("FinishReplaced@" + this.streamManager.TraceType, key.ToString(), endpoint.ToString()); } else { FabricEvents.Events.GetEndpointProperty("FinishNew@" + this.streamManager.TraceType, key.ToString(), endpoint.ToString()); } return(endpoint); }
// /// <summary> /// TODO: this may throw FabricObjectClosedException /// Use naming to find the integer partition range for a numbered target partition given a single partition number target /// </summary> /// <param name="key">Partition Key to resolve</param> /// <param name="timeout">Timeout for this operation to complete by</param> /// <returns>Resolved (and in case of key Kind = numbered) normalized partition key</returns> internal async Task <PartitionKey> ResolveAndNormalizeTargetPartition(PartitionKey key, TimeSpan timeout) { FabricEvents.Events.ResolveTargetPartition("Start@" + this.streamManager.TraceType, key.ToString(), ""); PartitionKey foundKey = null; var found = this.GetNormalizedPartitionKey(key, out foundKey); if (found) { FabricEvents.Events.ResolveTargetPartition("FoundCached@" + this.streamManager.TraceType, key.ToString(), foundKey.ToString()); return(foundKey); } var resolveSuccess = false; ResolvedServicePartition resolvedPartition = null; var resolveDelay = StreamConstants.BaseDelayForValidSessionEndpoint; var remainingTimeout = timeout; while (!resolveSuccess) { var beforeResolveAttempt = DateTime.UtcNow; // TODO: ResolveServicePartitionAsync seems to throw TimeoutException in unexpected ways: investigate and fix this, leaving out the timeout param for now try { switch (key.Kind) { case PartitionKind.Singleton: resolvedPartition = await this.client.ServiceManager.ResolveServicePartitionAsync(key.ServiceInstanceName); break; case PartitionKind.Named: resolvedPartition = await this.client.ServiceManager.ResolveServicePartitionAsync(key.ServiceInstanceName, key.PartitionName); break; case PartitionKind.Numbered: resolvedPartition = await this.client.ServiceManager.ResolveServicePartitionAsync(key.ServiceInstanceName, key.PartitionRange.IntegerKeyHigh); break; default: Diagnostics.Assert(false, "{0} Unknown partition Kind {1} in ResolveAndNormalizeTargetPartition", this.tracer, key); break; } resolveSuccess = true; } catch (FabricTransientException) { FabricEvents.Events.ResolveTargetPartition("TransientException.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } catch (TimeoutException) { FabricEvents.Events.ResolveTargetPartition("Timeout.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } catch (OperationCanceledException) { FabricEvents.Events.ResolveTargetPartition("Canceled.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } catch (FabricException fabricException) { if (fabricException.ErrorCode == FabricErrorCode.InvalidPartitionKey) { FabricEvents.Events.ResolveTargetPartition("KeyNotFound.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } else if (fabricException.ErrorCode == FabricErrorCode.CommunicationError) { FabricEvents.Events.ResolveTargetPartition("CommunicationError.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } else if (fabricException.ErrorCode == FabricErrorCode.ServiceNotFound) { FabricEvents.Events.ResolveTargetPartition("ServiceNotFound.Retrying@" + this.streamManager.TraceType, key.ToString(), ""); } else { Tracing.WriteExceptionAsError( "ResolveAndNormalizeTargetPartition.UnrecoverableError", fabricException, "{0} Target: {1}", this.tracer, key); throw; } } catch (Exception e) { Tracing.WriteExceptionAsError("ResolveAndNormalizeTargetPartition.UnrecoverableError", e, "{0} Target: {1}", this.tracer, key); throw; } if (!resolveSuccess) { var afterResolveAttempt = DateTime.UtcNow; remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, beforeResolveAttempt, afterResolveAttempt); // TODO: the behavior of ResolveServicePartitionAsync is unclear as far as timeout goes, this code needs fixing if ((afterResolveAttempt - beforeResolveAttempt).TotalMilliseconds < resolveDelay) { await Task.Delay(resolveDelay); } var afterDelay = DateTime.UtcNow; remainingTimeout = TimeoutHandler.UpdateTimeout(remainingTimeout, afterResolveAttempt, afterDelay); } } PartitionKey result; // if we had a numbered partition stream target with a single number then we need to change the key to the range actually supported by the resolved target partition if ((key.Kind == PartitionKind.Numbered) && (key.PartitionRange.IntegerKeyLow == key.PartitionRange.IntegerKeyHigh)) { var partitionInfo = resolvedPartition.Info as Int64RangePartitionInformation; result = new PartitionKey(key.ServiceInstanceName, partitionInfo.LowKey, partitionInfo.HighKey); } else { result = key; } // Add partition keys to the cache. this.normalizedPartitionKeys.TryAdd(key, result); FabricEvents.Events.ResolveTargetPartition("Finish@" + this.streamManager.TraceType, key.ToString(), result.ToString()); return(result); }
/// <summary> /// Register my(this instance of) service endpoint info (endpoint and era) /// This method is called only once at startup when the service becomes primary /// </summary> /// <param name="key">Partition to register</param> /// <param name="endpoint">Endpoint of the partition</param> /// <param name="era">Era of the partition</param> /// <returns></returns> internal async Task RegisterPartitionAsync(PartitionKey key, string endpoint, Guid era) { var parentName = key.ServiceInstanceName; var propertyName = key.PublishedPropertyName("TransportEndpoint"); var endPointInfo = new PartitionInfo(endpoint, era); FabricEvents.Events.RegisterEndpointProperty("Start@" + this.streamManager.TraceType, key.ToString(), endPointInfo.ToString()); await this.RevokeEnpointAsync(key); var putSuccess = false; while (!putSuccess) { try { await this.client.PropertyManager.PutPropertyAsync(parentName, propertyName, endPointInfo.Serialize()); putSuccess = true; } catch (FabricTransientException) { FabricEvents.Events.RegisterEndpointProperty( "TransientException.Retrying@" + this.streamManager.TraceType, key.ToString(), endPointInfo.ToString()); } catch (TimeoutException) { FabricEvents.Events.RegisterEndpointProperty( "Timeout.Retrying@" + this.streamManager.TraceType, key.ToString(), endPointInfo.ToString()); } catch (OperationCanceledException) { FabricEvents.Events.RegisterEndpointProperty( "Canceled.Retrying@" + this.streamManager.TraceType, key.ToString(), endPointInfo.ToString()); } catch (FabricException fabricException) { if (fabricException.ErrorCode == FabricErrorCode.CommunicationError) { FabricEvents.Events.RegisterEndpointProperty( "CommunicationError.Retrying@" + this.streamManager.TraceType, key.ToString(), endPointInfo.ToString()); } else { Tracing.WriteExceptionAsError("RegisterEndpointAsync.PutPropertyAsync.UnrecoverableError", fabricException, "{0}", this.tracer); throw; } } catch (Exception otherException) { Tracing.WriteExceptionAsError("RegisterEndpointAsync.PutPropertyAsync.UnrecoverableError", otherException, "{0}", this.tracer); throw; } } FabricEvents.Events.RegisterEndpointProperty("Finish@" + this.streamManager.TraceType, key.ToString(), endPointInfo.ToString()); }
// Hides the base SetResult method which we need to override // This method is called when the message is actually delivered to a ReceiveAsync call // The update to the CloseMessageSequenceNumber property of Inbound is transactional internal async Task SetResult(InboundBaseStreamWireMessage message) { var closeSeqNumber = this.stream.CloseMessageSequenceNumber; if (message.Kind == StreamWireProtocolMessageKind.CloseStream) { // The close message can be a repeat in recovery cases, but the close sequence number must be unique // It is also possible that the transaction surrounding this ReceiveAsync will abort and the message // will not in fact be consumed -- and will be received again, but the close sequence number must be unique Diagnostics.Assert( closeSeqNumber == StreamConstants.InitialValueOfLastSequenceNumberInStream || closeSeqNumber == message.MessageSequenceNumber, "{0} TraceId::{1} encountered multiple close sequence number values {2} and {3}", this.stream.Tracer, this.traceId, closeSeqNumber, message.MessageSequenceNumber); try { // this method is idempotent; the close sequence number for a stream is invariant await this.stream.SetCloseMessageSequenceNumber(message.MessageSequenceNumber, this.transactionContext); await this.stream.CloseInboundStream(this.transactionContext); } catch (Exception e) { if (e is FabricObjectClosedException) { FabricEvents.Events.DataMessageDelivery( string.Format(CultureInfo.InvariantCulture, "ObjectClosed@{0}{1}", this.traceId, this.stream.TraceType), this.stream.StreamIdentity.ToString(), message.MessageSequenceNumber, this.transactionContext.Id.ToString()); this.TrySetException(e); return; } if (e is FabricNotPrimaryException) { FabricEvents.Events.DataMessageDelivery( string.Format(CultureInfo.InvariantCulture, "NotPrimary@{0}{1}", this.traceId, this.stream.TraceType), this.stream.StreamIdentity.ToString(), message.MessageSequenceNumber, this.transactionContext.Id.ToString()); this.TrySetException(e); return; } if (e is FabricNotReadableException) { FabricEvents.Events.DataMessageDelivery( string.Format(CultureInfo.InvariantCulture, "NotReadable@{0}{1}", this.traceId, this.stream.TraceType), this.stream.StreamIdentity.ToString(), message.MessageSequenceNumber, this.transactionContext.Id.ToString()); this.TrySetException(e); return; } Tracing.WriteExceptionAsError( "DataMessageDelivery.Failure", e, "Partition::Replica {0} TraceId: {1} MessageNumber: {2} TransactionId: {3}", this.stream.TraceType, this.traceId, message.MessageSequenceNumber, this.transactionContext.Id); Diagnostics.Assert( false, "Unexpected Exception In {0} TraceId: {1} MessageNumber: {2} TransactionId: {3}", this.stream.TraceType, this.traceId, message.MessageSequenceNumber, this.transactionContext.Id); } } else { Diagnostics.Assert( closeSeqNumber == StreamConstants.InitialValueOfLastSequenceNumberInStream || closeSeqNumber > message.MessageSequenceNumber, "{0} received payload message with sequence number {1} which is greater than the close sequence number {2}", this.stream.Tracer, message.MessageSequenceNumber, closeSeqNumber); } var setResultSuccess = this.TrySetResult(message); if (setResultSuccess) { FabricEvents.Events.DataMessageDelivery( "Success::" + this.traceId + "@" + this.stream.TraceType, this.stream.StreamIdentity.ToString(), message.MessageSequenceNumber, this.transactionContext.Id.ToString()); } else { FabricEvents.Events.DataMessageDelivery( "Failure::" + this.traceId + "@" + this.stream.TraceType, this.stream.StreamIdentity.ToString(), message.MessageSequenceNumber, this.transactionContext.Id.ToString()); } }