public async Task BufferedConfiguration()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample04_BufferedConfiguration
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var options = new EventHubBufferedProducerClientOptions
{
MaximumWaitTime = TimeSpan.FromSeconds(1),
MaximumConcurrentSends = 5,
MaximumConcurrentSendsPerPartition = 1,
MaximumEventBufferLengthPerPartition = 5000,
EnableIdempotentRetries = true
};
var producer = new EventHubBufferedProducerClient(connectionString, eventHubName, options);
// The failure handler is required and invoked after all allowable
// retries were applied.
producer.SendEventBatchFailedAsync += args =>
{
Debug.WriteLine($"Publishing failed for { args.EventBatch.Count } events. Error: '{ args.Exception.Message }'");
return(Task.CompletedTask);
};
// The success handler is optional.
producer.SendEventBatchSucceededAsync += args =>
{
Debug.WriteLine($"{ args.EventBatch.Count } events were published to partition: '{ args.PartitionId }.");
return(Task.CompletedTask);
};
try
{
for (var index = 0; index < 5; ++index)
{
var eventData = new EventData($"Event #{ index }");
await producer.EnqueueEventAsync(eventData);
}
}
finally
{
// Closing the producer will flush any
// enqueued events that have not been published.
await producer.CloseAsync();
}
#endregion
}
private async Task PerformEnqueue(EventHubBufferedProducerClient producer,
CancellationToken cancellationToken)
{
var events = EventGenerator.CreateEvents(_testConfiguration.MaximumEventListSize,
_testConfiguration.EventEnqueueListSize,
_testConfiguration.LargeMessageRandomFactorPercent,
_testConfiguration.PublishingBodyMinBytes,
_testConfiguration.PublishingBodyRegularMaxBytes);
try
{
await producer.EnqueueEventsAsync(events, cancellationToken).ConfigureAwait(false);
_metrics.Client.GetMetric(_metrics.EventsEnqueued).TrackValue(_testConfiguration.EventEnqueueListSize);
}
catch (TaskCanceledException)
{
// Run is completed.
}
catch (Exception ex)
{
var eventProperties = new Dictionary <String, String>();
// Track that the exception took place during the enqueuing of an event
eventProperties.Add("Process", "Enqueue");
_metrics.Client.TrackException(ex, eventProperties);
}
}
public void CannotUnregisterEventHandlersWhenNotSet()
{
var mockProducer = new Mock <EventHubProducerClient>();
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
Assert.That(() => bufferedProducer.SendEventBatchSucceededAsync -= args => Task.CompletedTask, Throws.InstanceOf <ArgumentException>(), "The success handler should not allow removing an unset handler.");
Assert.That(() => bufferedProducer.SendEventBatchFailedAsync -= args => Task.CompletedTask, Throws.InstanceOf <ArgumentException>(), "The failure handler should not allow removing an unset handler.");
}
public void NullEventHandlersAreNotAllowed()
{
var mockProducer = new Mock <EventHubProducerClient>();
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
Assert.That(() => bufferedProducer.SendEventBatchSucceededAsync += null, Throws.InstanceOf <ArgumentNullException>(), "The success handler should not allow null.");
Assert.That(() => bufferedProducer.SendEventBatchFailedAsync += null, Throws.InstanceOf <ArgumentNullException>(), "The failure handler should not allow null.");
}
public async Task PartitionIdBuffered()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample04_PartitionIdBuffered
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var producer = new EventHubBufferedProducerClient(connectionString, eventHubName);
// The failure handler is required and invoked after all allowable
// retries were applied.
producer.SendEventBatchFailedAsync += args =>
{
Debug.WriteLine($"Publishing failed for { args.EventBatch.Count } events. Error: '{ args.Exception.Message }'");
return(Task.CompletedTask);
};
// The success handler is optional.
producer.SendEventBatchSucceededAsync += args =>
{
Debug.WriteLine($"{ args.EventBatch.Count } events were published to partition: '{ args.PartitionId }.");
return(Task.CompletedTask);
};
try
{
string firstPartition = (await producer.GetPartitionIdsAsync()).First();
var enqueueOptions = new EnqueueEventOptions
{
PartitionId = firstPartition
};
for (var index = 0; index < 5; ++index)
{
var eventData = new EventData($"Event #{ index }");
await producer.EnqueueEventAsync(eventData, enqueueOptions);
}
}
finally
{
// Closing the producer will flush any
// enqueued events that have not been published.
await producer.CloseAsync();
}
#endregion
}
/// <summary>
/// Performs the tasks needed to initialize and set up the environment for the test scenario.
/// This setup will take place once for each instance, running after the global setup has
/// completed.
/// </summary>
///
public override async Task SetupAsync()
{
await base.SetupAsync();
Scope = await EventHubScope.CreateAsync(Options.PartitionCount).ConfigureAwait(false);
var producerOptions = new EventHubBufferedProducerClientOptions
{
MaximumWaitTime = (Options.MaximumWaitTimeMilliseconds.HasValue) ? TimeSpan.FromMilliseconds(Options.MaximumWaitTimeMilliseconds.Value) : null,
MaximumEventBufferLengthPerPartition = Options.MaximumBufferLength,
MaximumConcurrentSendsPerPartition = Options.MaximumConcurrentSendsPerPartition
};
if (Options.MaximumConcurrentSends.HasValue)
{
producerOptions.MaximumConcurrentSends = Options.MaximumConcurrentSends.Value;
}
_producer = new EventHubBufferedProducerClient(TestEnvironment.EventHubsConnectionString, Scope.EventHubName, producerOptions);
// Create the handlers that call into the performance test infrastructure to
// report when errors and events are observed.
_producer.SendEventBatchFailedAsync += args =>
{
// Do not flag cancellation as a failure; it is expected for in-flight batches when
// the test run is cleaning up.
if (!typeof(OperationCanceledException).IsAssignableFrom(args.Exception.GetType()))
{
ErrorRaised(args.Exception);
}
return(Task.CompletedTask);
};
_producer.SendEventBatchSucceededAsync += args =>
{
for (var index = 0; index < args.EventBatch.Count; ++index)
{
EventRaised();
}
return(Task.CompletedTask);
};
// Read the available partitions and buffer a single events to establish the
// connection and link and start reading the buffers.
Partitions = await _producer.GetPartitionIdsAsync().ConfigureAwait(false);
await _producer.EnqueueEventsAsync(EventGenerator.CreateEvents(1)).ConfigureAwait(false);
// Start the background publishing task.
_backgroundCancellationSource = new CancellationTokenSource();
_backgroundBufferingTask = EnqueueEvents(_backgroundCancellationSource.Token);
}
public void OnlyOneEventHandlerIsAllowed()
{
var mockProducer = new Mock <EventHubProducerClient>();
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
bufferedProducer.SendEventBatchSucceededAsync += args => Task.CompletedTask;
bufferedProducer.SendEventBatchFailedAsync += args => Task.CompletedTask;
Assert.That(() => bufferedProducer.SendEventBatchSucceededAsync += args => Task.CompletedTask, Throws.InstanceOf <NotSupportedException>(), "The success handler should only allow one handler registration.");
Assert.That(() => bufferedProducer.SendEventBatchFailedAsync += args => Task.CompletedTask, Throws.InstanceOf <NotSupportedException>(), "The failure handler should only allow one handler registration.");
}
public async Task GetEventHubPropertiesAsyncIsDelegated()
{
using var cancellationSource = new CancellationTokenSource();
var mockProducer = new Mock <EventHubProducerClient>();
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
await bufferedProducer.GetEventHubPropertiesAsync(cancellationSource.Token);
mockProducer.Verify(producer => producer.GetEventHubPropertiesAsync(cancellationSource.Token), Times.Once);
}
public async Task CloseAsyncClosesTheProducer()
{
using var cancellationSource = new CancellationTokenSource();
var mockProducer = new Mock <EventHubProducerClient>("fakeNS", "fakeHub", Mock.Of <TokenCredential>(), new EventHubProducerClientOptions {
Identifier = "abc123"
});
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
await bufferedProducer.CloseAsync(false, cancellationSource.Token);
mockProducer.Verify(producer => producer.CloseAsync(CancellationToken.None), Times.Once);
}
public void RegisteredEventHandlersCanBeUnregistered()
{
var mockProducer = new Mock <EventHubProducerClient>();
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
Func <SendEventBatchSucceededEventArgs, Task> successHandler = args => Task.CompletedTask;
Func <SendEventBatchFailedEventArgs, Task> failHandler = args => Task.CompletedTask;
bufferedProducer.SendEventBatchSucceededAsync += successHandler;
bufferedProducer.SendEventBatchFailedAsync += failHandler;
Assert.That(() => bufferedProducer.SendEventBatchSucceededAsync -= successHandler, Throws.Nothing, "The success handler should allow removing the registered delegate.");
Assert.That(() => bufferedProducer.SendEventBatchFailedAsync -= failHandler, Throws.Nothing, "The failure handler should allow removing the registered delegate.");
}
public async Task GetPartitionIdsAsyncIsDelegated()
{
using var cancellationSource = new CancellationTokenSource();
var mockProducer = new Mock <EventHubProducerClient>();
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
mockProducer
.Setup(producer => producer.GetPartitionIdsAsync(It.IsAny <CancellationToken>()))
.ReturnsAsync(new[] { "0 " });
await bufferedProducer.GetPartitionIdsAsync(cancellationSource.Token);
mockProducer.Verify(producer => producer.GetPartitionIdsAsync(cancellationSource.Token), Times.Once);
}
public void ConnectionConstructorSetsDelegatedProperties()
{
var expectedIdentifier = "Test-Identifier";
var expectedNamespace = "testns.namespace.com";
var expectedEventHub = "testHub";
var options = new EventHubBufferedProducerClientOptions {
Identifier = expectedIdentifier
};
var connection = new EventHubConnection(expectedNamespace, expectedEventHub, Mock.Of <TokenCredential>());
var producer = new EventHubBufferedProducerClient(connection, options);
Assert.That(producer.Identifier, Is.EqualTo(expectedIdentifier), "The identifier should have been initialized.");
Assert.That(producer.FullyQualifiedNamespace, Is.EqualTo(expectedNamespace), "The fully qualified namespace should have been initialized.");
Assert.That(producer.EventHubName, Is.EqualTo(expectedEventHub), "The event hub name should have been initialized.");
}
public void SasCredentialConstructorSetsDelegatedProperties()
{
var expectedIdentifier = "Test-Identifier";
var expectedNamespace = "testns.namespace.com";
var expectedEventHub = "testHub";
var credential = new AzureSasCredential(new SharedAccessSignature("sb://this.is.Fake/blah", "key", "value").Value);
var options = new EventHubBufferedProducerClientOptions {
Identifier = expectedIdentifier
};
var producer = new EventHubBufferedProducerClient(expectedNamespace, expectedEventHub, credential, options);
Assert.That(producer.Identifier, Is.EqualTo(expectedIdentifier), "The identifier should have been initialized.");
Assert.That(producer.FullyQualifiedNamespace, Is.EqualTo(expectedNamespace), "The fully qualified namespace should have been initialized.");
Assert.That(producer.EventHubName, Is.EqualTo(expectedEventHub), "The event hub name should have been initialized.");
}
public void ConnectionStringAndEventHubConstructorSetsDelegatedProperties()
{
var expectedIdentifier = "Test-Identifier";
var expectedNamespace = "testns.namespace.com";
var expectedEventHub = "testHub";
var connectionString = $"Endpoint=sb://{ expectedNamespace };SharedAccessKeyName=ABC;SharedAccessKey=123";
var options = new EventHubBufferedProducerClientOptions {
Identifier = expectedIdentifier
};
var producer = new EventHubBufferedProducerClient(connectionString, expectedEventHub, options);
Assert.That(producer.Identifier, Is.EqualTo(expectedIdentifier), "The identifier should have been initialized.");
Assert.That(producer.FullyQualifiedNamespace, Is.EqualTo(expectedNamespace), "The fully qualified namespace should have been initialized.");
Assert.That(producer.EventHubName, Is.EqualTo(expectedEventHub), "The event hub name should have been initialized.");
}
public void CannotUnRegisterHandlersWhenPublishingIsActive()
{
var mockProducer = new Mock <EventHubProducerClient>();
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
Func <SendEventBatchSucceededEventArgs, Task> successHandler = args => Task.CompletedTask;
Func <SendEventBatchFailedEventArgs, Task> failHandler = args => Task.CompletedTask;
bufferedProducer.SendEventBatchSucceededAsync += successHandler;
bufferedProducer.SendEventBatchFailedAsync += failHandler;
bufferedProducer.IsPublishing = true;
Assert.That(() => bufferedProducer.SendEventBatchSucceededAsync -= successHandler, Throws.InstanceOf <NotSupportedException>(), "The success handler should not allow unregistering when publishing is active.");
Assert.That(() => bufferedProducer.SendEventBatchFailedAsync -= failHandler, Throws.InstanceOf <NotSupportedException>(), "The failure handler should not allow unregistering when publishing is active.");
}
public async Task GetPartitionPropertiesAsyncIsDelegated()
{
using var cancellationSource = new CancellationTokenSource();
var expectedPartition = "0";
var mockProducer = new Mock <EventHubProducerClient>();
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
mockProducer
.Setup(producer => producer.GetPartitionPropertiesAsync(
It.IsAny <string>(),
It.IsAny <CancellationToken>()))
.ReturnsAsync(new PartitionProperties("test", "1", true, 12345, 6789, 22, new DateTimeOffset(2015, 10, 27, 0, 0, 0, TimeSpan.Zero)));
await bufferedProducer.GetPartitionPropertiesAsync(expectedPartition, cancellationSource.Token);
mockProducer.Verify(producer => producer.GetPartitionPropertiesAsync(expectedPartition, cancellationSource.Token), Times.Once);
}
public async Task CloseAsyncUnregistersTheEventHandlers()
{
using var cancellationSource = new CancellationTokenSource();
var mockProducer = new Mock <EventHubProducerClient>("fakeNS", "fakeHub", Mock.Of <TokenCredential>(), new EventHubProducerClientOptions {
Identifier = "abc123"
});
var bufferedProducer = new EventHubBufferedProducerClient(mockProducer.Object);
Func <SendEventBatchSucceededEventArgs, Task> successHandler = args => Task.CompletedTask;
Func <SendEventBatchFailedEventArgs, Task> failHandler = args => Task.CompletedTask;
bufferedProducer.SendEventBatchSucceededAsync += successHandler;
bufferedProducer.SendEventBatchFailedAsync += failHandler;
await bufferedProducer.CloseAsync(false, cancellationSource.Token);
// The handlers should have been unregistered; attempting to do so again should cause
// an exception.
Assert.That(() => bufferedProducer.SendEventBatchSucceededAsync -= successHandler, Throws.InstanceOf <ArgumentException>(), "The success handler should have already been unregistered.");
Assert.That(() => bufferedProducer.SendEventBatchFailedAsync -= failHandler, Throws.InstanceOf <ArgumentException>(), "The failure handler should have already been unregistered.");
}
public async Task CustomMetadata()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample04_CustomMetadata
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var producer = new EventHubBufferedProducerClient(connectionString, eventHubName);
// The failure handler is required and invoked after all allowable
// retries were applied.
producer.SendEventBatchFailedAsync += args =>
{
Debug.WriteLine($"Publishing failed for { args.EventBatch.Count } events. Error: '{ args.Exception.Message }'");
return(Task.CompletedTask);
};
// The success handler is optional.
producer.SendEventBatchSucceededAsync += args =>
{
Debug.WriteLine($"{ args.EventBatch.Count } events were published to partition: '{ args.PartitionId }.");
return(Task.CompletedTask);
};
try
{
var eventData = new EventData("Hello, Event Hubs!")
{
MessageId = "H1",
ContentType = "application/json"
};
eventData.Properties.Add("EventType", "com.microsoft.samples.hello-event");
eventData.Properties.Add("priority", 1);
eventData.Properties.Add("score", 9.0);
await producer.EnqueueEventAsync(eventData);
eventData = new EventData("Goodbye, Event Hubs!")
{
MessageId = "G1",
ContentType = "application/json"
};
eventData.Properties.Add("EventType", "com.microsoft.samples.goodbye-event");
eventData.Properties.Add("priority", "17");
eventData.Properties.Add("blob", true);
await producer.EnqueueEventAsync(eventData);
}
finally
{
// Closing the producer will flush any
// enqueued events that have not been published.
await producer.CloseAsync();
}
#endregion
}
public async Task Start(CancellationToken cancellationToken)
{
var enqueueTasks = new List <Task>();
while (!cancellationToken.IsCancellationRequested)
{
using var backgroundCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
// Create the buffered producer client and register the success and failure handlers
var options = new EventHubBufferedProducerClientOptions
{
RetryOptions = new EventHubsRetryOptions
{
TryTimeout = _testConfiguration.SendTimeout,
MaximumRetries = 15
},
MaximumWaitTime = _testConfiguration.MaxWaitTime
};
var producer = new EventHubBufferedProducerClient(_connectionString, _eventHubName, options);
try
{
producer.SendEventBatchSucceededAsync += args =>
{
var numEvents = args.EventBatch.Count;
_metrics.Client.GetMetric(_metrics.SuccessfullySentFromQueue, "PartitionId").TrackValue(numEvents, args.PartitionId);
_metrics.Client.GetMetric(_metrics.BatchesPublished).TrackValue(1);
return(Task.CompletedTask);
};
producer.SendEventBatchFailedAsync += args =>
{
var numEvents = args.EventBatch.Count;
_metrics.Client.GetMetric(_metrics.EventsNotSentAfterEnqueue, "PartitionId").TrackValue(numEvents, args.PartitionId);
_metrics.Client.TrackException(args.Exception);
return(Task.CompletedTask);
};
// Start concurrent enqueuing tasks
if (_testConfiguration.ConcurrentSends > 1)
{
for (var index = 0; index < _testConfiguration.ConcurrentSends - 1; ++index)
{
enqueueTasks.Add(Task.Run(async() =>
{
while (!cancellationToken.IsCancellationRequested)
{
await PerformEnqueue(producer, cancellationToken).ConfigureAwait(false);
if ((_testConfiguration.ProducerPublishingDelay.HasValue) && (_testConfiguration.ProducerPublishingDelay.Value > TimeSpan.Zero))
{
await Task.Delay(_testConfiguration.ProducerPublishingDelay.Value, backgroundCancellationSource.Token).ConfigureAwait(false);
}
}
}));
}
}
// Perform one of the sends in the foreground, which will allow easier detection of a
// processor-level issue.
while (!cancellationToken.IsCancellationRequested)
{
try
{
await PerformEnqueue(producer, cancellationToken).ConfigureAwait(false);
if ((_testConfiguration.ProducerPublishingDelay.HasValue) && (_testConfiguration.ProducerPublishingDelay.Value > TimeSpan.Zero))
{
await Task.Delay(_testConfiguration.ProducerPublishingDelay.Value, cancellationToken).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
backgroundCancellationSource.Cancel();
await Task.WhenAll(enqueueTasks).ConfigureAwait(false);
}
}
}
catch (TaskCanceledException)
{
// No action needed, the cancellation token has been cancelled.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
// If this catch is hit, it means the producer has restarted, collect metrics.
_metrics.Client.GetMetric(_metrics.ProducerRestarted).TrackValue(1);
_metrics.Client.TrackException(ex);
}
finally
{
await producer.CloseAsync(false);
}
}
}
