public virtual async Task <bool> InitiateCleanClose(bool initiatedByServer, AmqpError error)
{
if (_isClosed)
{
return(false);
}
Thread.MemoryBarrier();
_isClosed = true;
if (LogAdapter.IsDebugEnabled)
{
LogAdapter.LogDebug(LogSource, "Clean close initiated. By server? " + initiatedByServer);
}
if (initiatedByServer)
{
await SendCloseConfirmation().ConfigureAwait(false);
}
else
{
await SendStartClose().ConfigureAwait(false);
}
DrainPending(error);
if (error != null)
{
await FireErrorEvent(error);
}
return(true);
}
internal void DrainPendingIfNeeded(AmqpError error)
{
if (this._confirmationKeeper != null)
{
this._confirmationKeeper.DrainDueToFailure(error);
}
}
public override async Task <bool> InitiateCleanClose(bool initiatedByServer, AmqpError error)
{
if (!initiatedByServer)
{
_conn.NotifyClosedByUser();
while (!_commandOutbox.IsEmpty && !_socketHolder.IsClosed)
{
// give it some time to finish writing to the socket (if it's open)
Thread.Sleep(250);
}
}
else
{
if (_conn.NotifyClosedByServer() == RecoveryAction.WillReconnect)
{
CancelPendingCommands(error);
return(false);
}
}
_conn.CloseAllChannels(initiatedByServer, error);
await base.InitiateCleanClose(initiatedByServer, error).ConfigureAwait(false);
CancelPendingCommands(error);
_socketHolder.Close();
OnSocketClosed();
return(true);
}
public void GetPartitionPropertiesAsyncAppliesTheRetryPolicyForAmqpErrors(EventHubsRetryOptions retryOptions)
{
var eventHubName = "myName";
var partitionId = "Barney";
var tokenValue = "123ABC";
var retryPolicy = new BasicRetryPolicy(retryOptions);
var retriableException = AmqpError.CreateExceptionForError(new Error {
Condition = AmqpError.ServerBusyError
}, "dummy");
var mockConverter = new Mock <AmqpMessageConverter>();
var mockCredential = new Mock <EventHubTokenCredential>(Mock.Of <TokenCredential>());
var mockScope = new Mock <AmqpConnectionScope>();
using var cancellationSource = new CancellationTokenSource();
mockCredential
.Setup(credential => credential.GetTokenAsync(It.IsAny <TokenRequestContext>(), It.Is <CancellationToken>(value => value == cancellationSource.Token)))
.Returns(new ValueTask <AccessToken>(new AccessToken(tokenValue, DateTimeOffset.MaxValue)));
mockConverter
.Setup(converter => converter.CreatePartitionPropertiesRequest(It.Is <string>(value => value == eventHubName), It.Is <string>(value => value == partitionId), It.Is <string>(value => value == tokenValue)))
.Returns(default(AmqpMessage));
mockScope
.Setup(scope => scope.OpenManagementLinkAsync(It.IsAny <TimeSpan>(), It.IsAny <CancellationToken>()))
.Throws(retriableException);
var client = new InjectableMockClient("my.eventhub.com", eventHubName, mockCredential.Object, new EventHubConnectionOptions(), mockScope.Object, mockConverter.Object);
Assert.That(async() => await client.GetPartitionPropertiesAsync(partitionId, retryPolicy, cancellationSource.Token), Throws.InstanceOf(retriableException.GetType()));
mockScope.Verify(scope => scope.OpenManagementLinkAsync(It.IsAny <TimeSpan>(), It.IsAny <CancellationToken>()), Times.Exactly(1 + retryOptions.MaximumRetries));
}
public void CreateExceptionForErrorWithCondition(AmqpSymbol condition,
Type exceptionType,
EventHubsException.FailureReason?reason)
{
var description = "This is a test description.";
var resourceName = "TestHub";
var error = new Error
{
Condition = condition,
Description = description
};
Exception exception = AmqpError.CreateExceptionForError(error, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf(exceptionType), "The exception should be the proper type");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
if (exception is EventHubsException)
{
Assert.That(((EventHubsException)exception).Reason, Is.EqualTo(reason), "The proper failure reason should be specified");
Assert.That(((EventHubsException)exception).EventHubName, Is.EqualTo(resourceName), "The exception should report the proper resource");
}
}
public void SendEnumerableAppliesTheRetryPolicyForAmqpErrors(EventHubsRetryOptions retryOptions)
{
var partitionId = "testMe";
var retriableException = AmqpError.CreateExceptionForError(new Error {
Value = AmqpError.ServerBusyError
}, "dummy");
var retryPolicy = new BasicRetryPolicy(retryOptions);
var producer = new Mock <AmqpProducer>("aHub", partitionId, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy)
{
CallBase = true
};
producer
.Protected()
.Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync",
ItExpr.IsAny <string>(),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>())
.Throws(retriableException);
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
Assert.That(async() => await producer.Object.SendAsync(new[] { new EventData(new byte[] { 0x65 }) }, new SendEventOptions(), cancellationSource.Token), Throws.InstanceOf(retriableException.GetType()));
producer
.Protected()
.Verify("CreateLinkAndEnsureProducerStateAsync", Times.Exactly(1 + retryOptions.MaximumRetries),
ItExpr.Is <string>(value => value == partitionId),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>());
}
public void SendBatchAppliesTheRetryPolicyForAmqpErrors(EventHubsRetryOptions retryOptions)
{
var partitionKey = "testMe";
var options = new CreateBatchOptions {
PartitionKey = partitionKey
};
var retriableException = AmqpError.CreateExceptionForError(new Error {
Value = AmqpError.ServerBusyError
}, "dummy");
var retryPolicy = new BasicRetryPolicy(retryOptions);
var batch = new EventDataBatch(Mock.Of <TransportEventBatch>(), "ns", "eh", options.ToSendOptions());
var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy)
{
CallBase = true
};
producer
.Protected()
.Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync",
ItExpr.IsAny <string>(),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>())
.Throws(retriableException);
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
Assert.That(async() => await producer.Object.SendAsync(batch, cancellationSource.Token), Throws.InstanceOf(retriableException.GetType()));
producer
.Protected()
.Verify("CreateLinkAndEnsureProducerStateAsync", Times.Exactly(1 + retryOptions.MaximumRetries),
ItExpr.Is <string>(value => value == null),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>());
}
public void ReceiveAsyncAppliesTheRetryPolicyForAmqpErrors(EventHubsRetryOptions retryOptions)
{
var eventHub = "eventHubName";
var consumerGroup = "$DEFAULT";
var partition = "3";
var identifier = "cusTOM-1D";
var eventPosition = EventPosition.FromOffset(123);
var trackLastEnqueued = false;
var invalidateOnSteal = true;
var ownerLevel = 123L;
var tokenValue = "123ABC";
var retryPolicy = new BasicRetryPolicy(retryOptions);
var retriableException = AmqpError.CreateExceptionForError(new Error {
Condition = AmqpError.ServerBusyError
}, "dummy");
var mockConverter = new Mock <AmqpMessageConverter>();
var mockCredential = new Mock <TokenCredential>();
var mockScope = new Mock <AmqpConnectionScope>();
using var cancellationSource = new CancellationTokenSource();
mockCredential
.Setup(credential => credential.GetTokenAsync(It.IsAny <TokenRequestContext>(), It.Is <CancellationToken>(value => value == cancellationSource.Token)))
.Returns(new ValueTask <AccessToken>(new AccessToken(tokenValue, DateTimeOffset.MaxValue)));
mockScope
.Setup(scope => scope.OpenConsumerLinkAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <EventPosition>(),
It.IsAny <TimeSpan>(),
It.IsAny <TimeSpan>(),
It.IsAny <uint>(),
It.IsAny <long?>(),
It.IsAny <long?>(),
It.IsAny <bool>(),
It.IsAny <string>(),
It.IsAny <CancellationToken>()))
.Throws(retriableException);
var consumer = new AmqpConsumer(eventHub, consumerGroup, partition, identifier, eventPosition, trackLastEnqueued, invalidateOnSteal, ownerLevel, null, null, mockScope.Object, Mock.Of <AmqpMessageConverter>(), retryPolicy);
Assert.That(async() => await consumer.ReceiveAsync(100, null, cancellationSource.Token), Throws.InstanceOf(retriableException.GetType()));
mockScope
.Verify(scope => scope.OpenConsumerLinkAsync(
It.Is <string>(value => value == consumerGroup),
It.Is <string>(value => value == partition),
It.Is <EventPosition>(value => value == eventPosition),
It.IsAny <TimeSpan>(),
It.IsAny <TimeSpan>(),
It.IsAny <uint>(),
It.IsAny <long?>(),
It.Is <long?>(value => value == ownerLevel),
It.Is <bool>(value => value == trackLastEnqueued),
It.Is <string>(value => value == identifier),
It.IsAny <CancellationToken>()),
Times.Exactly(1 + retryOptions.MaximumRetries));
}
public void CreateExceptionForResponseWithNoResponse()
{
var exception = AmqpError.CreateExceptionForResponse(null, null);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsException>(), "The exception should be a generic Event Hubs exception");
Assert.That(exception.Message, Is.EqualTo(Resources.UnknownCommunicationException), "The exception message should indicate an unknown failure");
Assert.That(((EventHubsException)exception).IsTransient, Is.True, "The exception should be considered transient");
}
private void FireErrorEvent(AmqpError error)
{
var ev = this.OnError;
if (ev != null && error != null)
{
ev(error);
}
}
private void CancelPendingCommands(AmqpError error)
{
CommandToSend cmdToSend;
while (_commandOutbox.TryDequeue(out cmdToSend))
{
cmdToSend.RunReplyAction(0, 0, error).IntentionallyNotAwaited();
}
}
/// <summary>
/// Determines if a given AMQP message response is an error and, if so, throws the
/// appropriate corresponding exception type.
/// </summary>
///
/// <param name="response">The AMQP response message to consider.</param>
/// <param name="eventHubName">The name of the Event Hub associated with the request.</param>
///
private static void ThrowIfErrorResponse(AmqpMessage response,
string eventHubName)
{
var statusCode = default(int);
if ((response?.ApplicationProperties?.Map.TryGetValue(AmqpResponse.StatusCode, out statusCode) != true) ||
(!AmqpResponse.IsSuccessStatus((AmqpResponseStatusCode)statusCode)))
{
throw AmqpError.CreateExceptionForResponse(response, eventHubName);
}
}
private void CancelPendingCommands(AmqpError error)
{
CommandToSend cmdToSend;
while (_commandOutbox.TryDequeue(out cmdToSend))
{
#pragma warning disable 4014
cmdToSend.RunReplyAction(0, 0, error);
#pragma warning restore 4014
}
}
public void ThrowIfErrorResponseThrowsOnFailure(AmqpResponseStatusCode statusCode)
{
var description = "This is a test description.";
var resourceName = "TestHub";
using var response = AmqpMessage.Create();
response.ApplicationProperties = new ApplicationProperties();
response.ApplicationProperties.Map[AmqpResponse.StatusCode] = (int)statusCode;
response.ApplicationProperties.Map[AmqpResponse.StatusDescription] = description;
Assert.That(() => AmqpError.ThrowIfErrorResponse(response, resourceName), Throws.Exception);
}
// A disconnect may be expected coz we send a connection close, etc..
// or it may be something abruptal
internal void HandleDisconnect()
{
if (_isClosed)
{
return; // we have initiated the close
}
// otherwise
_lastError = new AmqpError {
ClassId = 0, MethodId = 0, ReplyCode = 0, ReplyText = "disconnected"
};
DrainPending(_lastError);
}
internal void CloseAllChannels(bool initiatedByServer, AmqpError error)
{
foreach (var channel in _channels)
{
if (channel == null)
{
continue;
}
#pragma warning disable 4014
channel._io.InitiateCleanClose(initiatedByServer, error);
#pragma warning restore 4014
}
}
public void CreateExceptionForResponseWithUnknownCondition()
{
var description = "NOT_KNOWN";
var resourceName = "TestHub";
using var response = AmqpMessage.Create();
response.ApplicationProperties = new ApplicationProperties();
response.ApplicationProperties.Map[AmqpResponse.ErrorCondition] = new AmqpSymbol("Invalid");
response.ApplicationProperties.Map[AmqpResponse.StatusDescription] = description;
var exception = AmqpError.CreateExceptionForResponse(response, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsException>(), "The exception should match");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
}
public void TranslateServiceExceptionTranslatesOperationCanceledWithEmbeddedAmqpException()
{
var eventHub = "someHub";
var exception = new OperationCanceledException("oops", AmqpError.CreateExceptionForError(new Error {
Condition = AmqpError.ServerBusyError
}, eventHub));
var translated = exception.TranslateServiceException(eventHub);
Assert.That(translated, Is.Not.Null, "An exception should have been returned.");
var eventHubsException = translated as EventHubsException;
Assert.That(eventHubsException, Is.Not.Null, "The exception type should be appropriate for the `Server Busy` scenario.");
Assert.That(eventHubsException.Reason, Is.EqualTo(EventHubsException.FailureReason.ServiceBusy), "The exception reason should indicate `Server Busy`.");
Assert.That(eventHubsException.EventHubName, Is.EqualTo(eventHub), "The Event Hub name should match.");
}
public void CreateExceptionForResponseWithTimeoutConditionAndPattern()
{
var description = GetNotFoundExpression().ToString().Replace("*.", "some value");
var resourceName = "TestHub";
using var response = AmqpMessage.Create();
response.ApplicationProperties = new ApplicationProperties();
response.ApplicationProperties.Map[AmqpResponse.ErrorCondition] = AmqpErrorCode.NotFound;
response.ApplicationProperties.Map[AmqpResponse.StatusDescription] = description;
var exception = AmqpError.CreateExceptionForResponse(response, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsResourceNotFoundException>(), "The exception should match");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
}
internal void CloseAllChannels(bool initiatedByServer, AmqpError error)
{
LogAdapter.LogDebug("Connection", "Closing all channels");
foreach (var channel in _channels)
{
if (channel == null)
{
continue;
}
#pragma warning disable 4014
channel._io.InitiateCleanClose(initiatedByServer, error);
#pragma warning restore 4014
}
}
public void CreateExceptionForResponseWithUnmappedStatus()
{
var description = "Some description";
var resourceName = "TestHub";
using var response = AmqpMessage.Create();
response.ApplicationProperties = new ApplicationProperties();
response.ApplicationProperties.Map[AmqpResponse.StatusCode] = Int32.MaxValue;
response.ApplicationProperties.Map[AmqpResponse.StatusDescription] = description;
var exception = AmqpError.CreateExceptionForResponse(response, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsException>(), "The exception should match");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
}
public void CreateExceptionForResponseWithTimeoutConditionAndStatusDescription()
{
var description = $"This has { GetNotFoundStatusText() } embedded in it";
var resourceName = "TestHub";
using var response = AmqpMessage.Create();
response.ApplicationProperties = new ApplicationProperties();
response.ApplicationProperties.Map[AmqpResponse.ErrorCondition] = AmqpErrorCode.NotFound;
response.ApplicationProperties.Map[AmqpResponse.StatusDescription] = description;
var exception = AmqpError.CreateExceptionForResponse(response, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsResourceNotFoundException>(), "The exception should match");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
}
public void CreateExceptionForErrorWithUnknownTimeout()
{
var description = "NOT_KNOWN";
var resourceName = "TestHub";
var error = new Error
{
Condition = AmqpErrorCode.NotFound,
Description = description
};
Exception exception = AmqpError.CreateExceptionForError(error, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsCommunicationException>(), "The exception should match");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
}
public void CreateExceptionForErrorWithTimeoutStatusDescription()
{
var description = $"This has { GetNotFoundStatusText() } embedded in it";
var resourceName = "TestHub";
var error = new Error
{
Condition = AmqpErrorCode.NotFound,
Description = description
};
var exception = AmqpError.CreateExceptionForError(error, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsResourceNotFoundException>(), "The exception should match");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
}
public void CreateExceptionForResponseWithUnknownTimeout()
{
var description = "NOT_KNOWN";
var resourceName = "TestHub";
using var response = AmqpMessage.Create();
response.ApplicationProperties = new ApplicationProperties();
response.ApplicationProperties.Map[AmqpResponse.ErrorCondition] = AmqpErrorCode.NotFound;
response.ApplicationProperties.Map[AmqpResponse.StatusDescription] = description;
Exception exception = AmqpError.CreateExceptionForResponse(response, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsException>(), "The exception should match");
Assert.That(((EventHubsException)exception).Reason, Is.EqualTo(EventHubsException.FailureReason.ServiceCommunicationProblem), "The exception reason should match.");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
}
public void CreateExceptionForErrorWithTimeoutPattern()
{
var description = GetNotFoundExpression().ToString().Replace("*.", "some value");
var resourceName = "TestHub";
var error = new Error
{
Condition = AmqpErrorCode.NotFound,
Description = description
};
var exception = AmqpError.CreateExceptionForError(error, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsResourceNotFoundException>(), "The exception should match");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
}
public void CreateExceptionForErrorWithUnknownCondition()
{
var description = "NOT_KNOWN";
var resourceName = "TestHub";
var error = new Error
{
Condition = new AmqpSymbol("Invalid"),
Description = description
};
var exception = AmqpError.CreateExceptionForError(error, resourceName);
Assert.That(exception, Is.Not.Null, "An exception should have been created");
Assert.That(exception, Is.TypeOf <EventHubsException>(), "The exception should match");
Assert.That(exception.Message, Is.SupersetOf(description), "The exception message should contain the description");
}
private async Task FireErrorEvent(AmqpError error)
{
Func <AmqpError, Task>[] copy = null;
lock (ErrorCallbacks)
{
if (ErrorCallbacks == null || ErrorCallbacks.Count == 0)
{
return;
}
copy = ErrorCallbacks.ToArray();
}
foreach (var errorCallback in copy)
{
await errorCallback(error);
}
}
// To be use in case of exceptions on our end. Close everything asap
internal virtual async Task InitiateAbruptClose(Exception reason)
{
if (_isClosed)
{
return;
}
Thread.MemoryBarrier();
_isClosed = true;
var syntheticError = new AmqpError {
ReplyText = reason.Message
};
DrainPending(syntheticError);
await FireErrorEvent(syntheticError);
this.Dispose();
}
protected virtual void DrainPending(AmqpError error)
{
// releases every task awaiting
CommandToSend sent;
while (_awaitingReplyQueue.TryDequeue(out sent))
{
if (error != null && sent.ClassId == error.ClassId && sent.MethodId == error.MethodId)
{
// if we find the "offending" command, then it gets a better error message
sent.RunReplyAction(0, 0, error);
}
else
{
// any other task dies with a generic error.
sent.RunReplyAction(0, 0, null);
}
}
}