public async Task ProducerCanSendSingleLargeEventInASet()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString, new EventHubClientOptions {
RetryOptions = new RetryOptions {
TryTimeout = TimeSpan.FromMinutes(5)
}
}))
await using (var producer = client.CreateProducer())
{
// Actual limit is 1046520 for a single event.
var eventSet = new[] { new EventData(new byte[100000]) };
Assert.That(async() => await producer.SendAsync(eventSet), Throws.Nothing);
}
}
}
public async Task ProducerCanSendASetOfEvents()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
var events = new[]
{
new EventData(Encoding.UTF8.GetBytes("This is a message")),
new EventData(Encoding.UTF8.GetBytes("This is another message")),
new EventData(Encoding.UTF8.GetBytes("So many messages"))
};
Assert.That(async() => await producer.SendAsync(events), Throws.Nothing);
}
}
}
public async Task ConsumerWithOptionsCanReceive()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var options = new EventHubConsumerOptions {
Identifier = "FakeIdentifier"
};
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
await using (var consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroup, partition, EventPosition.Latest, options))
{
Assert.That(async() => await consumer.ReceiveAsync(1, TimeSpan.Zero), Throws.Nothing);
}
}
}
}
public async Task ProducerCanSendEventsUsingAPartitionHashKey()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var events = Enumerable
.Range(0, 25)
.Select(index => new EventData(Encoding.UTF8.GetBytes(new String('X', index + 5))));
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
var batchOptions = new SendOptions {
PartitionKey = "some123key-!d"
};
Assert.That(async() => await producer.SendAsync(events, batchOptions), Throws.Nothing);
}
}
}
public async Task ProducerCanSendWhenPartitionIsNull()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var producerOptions = new EventHubProducerOptions {
PartitionId = null
};
await using (var producer = client.CreateProducer(producerOptions))
{
var events = new[] { new EventData(Encoding.UTF8.GetBytes("Will it work")) };
Assert.That(async() => await producer.SendAsync(events), Throws.Nothing);
}
}
}
}
public async Task ProducerCanSendAnEventBatch()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
using var batch = await producer.CreateBatchAsync();
batch.TryAdd(new EventData(Encoding.UTF8.GetBytes("This is a message")));
batch.TryAdd(new EventData(Encoding.UTF8.GetBytes("This is another message")));
batch.TryAdd(new EventData(Encoding.UTF8.GetBytes("So many messages")));
Assert.That(batch.Count, Is.EqualTo(3), "The batch should contain all 3 events.");
Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing);
}
}
}
public async Task ProducerCanSendZeroLengthSet()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
var events = new[]
{
new EventData(Array.Empty <byte>()),
new EventData(Array.Empty <byte>()),
new EventData(Array.Empty <byte>())
};
Assert.That(async() => await producer.SendAsync(events), Throws.Nothing);
}
}
}
public async Task ClientCanRetrievePartitionProperties()
{
var partitionCount = 4;
await using (var scope = await EventHubScope.CreateAsync(partitionCount))
{
var clientOptions = new EventHubClientOptions();
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var connectionProperties = ConnectionStringParser.Parse(connectionString);
var credential = new SharedAccessSignatureCredential
(
new SharedAccessSignature
(
clientOptions.TransportType,
connectionProperties.Endpoint.Host,
connectionProperties.EventHubPath,
connectionProperties.SharedAccessKeyName,
connectionProperties.SharedAccessKey,
TimeSpan.FromHours(4)
)
);
await using (var client = new EventHubClient(connectionProperties.Endpoint.Host, connectionProperties.EventHubPath, credential))
{
var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(20));
var properties = await client.GetPropertiesAsync();
var partition = properties.PartitionIds.First();
var partitionProperties = await client.GetPartitionPropertiesAsync(partition, cancellation.Token);
Assert.That(partitionProperties, Is.Not.Null, "A set of partition properties should have been returned.");
Assert.That(partitionProperties.Id, Is.EqualTo(partition), "The partition identifier should match.");
Assert.That(partitionProperties.EventHubPath, Is.EqualTo(connectionProperties.EventHubPath).Using((IEqualityComparer <string>)StringComparer.InvariantCultureIgnoreCase), "The Event Hub path should match.");
Assert.That(partitionProperties.BeginningSequenceNumber, Is.Not.EqualTo(default(Int64)), "The beginning sequence number should have been populated.");
Assert.That(partitionProperties.LastEnqueuedSequenceNumber, Is.Not.EqualTo(default(Int64)), "The last sequance number should have been populated.");
Assert.That(partitionProperties.LastEnqueuedOffset, Is.Not.Null.Or.Empty, "The last offset should have been populated.");
Assert.That(partitionProperties.PropertyRetrievalTimeUtc, Is.EqualTo(DateTime.UtcNow).Within(TimeSpan.FromSeconds(30)), "The property retrieval time should denote the current time.");
}
}
}
public async Task PartitionProcessorProcessEventsAsyncIsCalledWithNoEvents()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var receivedEventSets = new ConcurrentBag <IEnumerable <EventData> >();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onProcessEvents: (partitionContext, events, cancellationToken) =>
receivedEventSets.Add(events)
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventSets, Is.Not.Empty);
Assert.That(receivedEventSets.Any(set => (set == null || set.Any())), Is.False);
}
}
}
public async Task PartitionProcessorProcessEventsAsyncIsCalledWithNoEvents()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new EventHubConnection(connectionString))
{
var receivedEvents = new ConcurrentBag <EventData>();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumerClient.DefaultConsumerGroupName,
connection,
onProcessEvent: processorEvent =>
receivedEvents.Add(processorEvent.Data)
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEvents, Is.Not.Empty);
Assert.That(receivedEvents.Any(eventData => eventData != null), Is.False);
}
}
}
public async Task ClientPartitionIdsMatchPartitionProperties()
{
var partitionCount = 4;
await using (var scope = await EventHubScope.CreateAsync(partitionCount))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var properties = await client.GetPropertiesAsync();
var partitions = await client.GetPartitionIdsAsync();
Assert.That(properties, Is.Not.Null, "A set of properties should have been returned.");
Assert.That(properties.PartitionIds, Is.Not.Null, "A set of partition identifiers for the properties should have been returned.");
Assert.That(partitions, Is.Not.Null, "A set of partition identifiers should have been returned.");
Assert.That(partitions, Is.EquivalentTo(properties.PartitionIds), "The partition identifiers returned directly should match those returned with properties.");
}
}
}
public async Task ProducerCannotSendWhenProxyIsInvalid()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var clientOptions = new EventHubClientOptions
{
Proxy = new WebProxy("http://1.2.3.4:9999"),
TransportType = TransportType.AmqpWebSockets,
RetryOptions = new RetryOptions {
TryTimeout = TimeSpan.FromMinutes(2)
}
};
await using (var invalidProxyClient = new EventHubClient(connectionString, clientOptions))
await using (EventHubProducer invalidProxyProducer = invalidProxyClient.CreateProducer())
{
Assert.That(async() => await invalidProxyProducer.SendAsync(new EventData(new byte[1])), Throws.InstanceOf <WebSocketException>().Or.InstanceOf <TimeoutException>());
}
}
}
public async Task ProducerWithOptionsCanSend(TransportType transportType)
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var producerOptions = new EventHubProducerOptions {
RetryOptions = new RetryOptions {
MaximumRetries = 5
}
};
await using (var client = new EventHubClient(connectionString, new EventHubClientOptions {
TransportType = transportType
}))
await using (var producer = client.CreateProducer(producerOptions))
{
var events = new[] { new EventData(Encoding.UTF8.GetBytes("AWord")) };
Assert.That(async() => await producer.SendAsync(events), Throws.Nothing);
}
}
}
public async Task ProducerCannotSendBatchLargerThanMaximumSize()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
// Actual limit is 1046520 for a single event
var events = new[]
{
new EventData(new byte[1500000 / 3]),
new EventData(new byte[1500000 / 3]),
new EventData(new byte[1500000 / 3])
};
Assert.That(async() => await producer.SendAsync(events), Throws.TypeOf <TrackOne.MessageSizeExceededException>());
}
}
}
public async Task ProducerCanSendToASpecificPartition()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
var producerOptions = new EventHubProducerOptions {
PartitionId = partition
};
await using (var producer = client.CreateProducer(producerOptions))
{
var events = new[] { new EventData(Encoding.UTF8.GetBytes("AWord")) };
Assert.That(async() => await producer.SendAsync(events), Throws.Nothing);
}
}
}
}
public async Task ProducerCanSendLargeBatch()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
// Actual limit is 1046520 for a single event
var events = new[]
{
new EventData(new byte[1000000 / 3]),
new EventData(new byte[1000000 / 3]),
new EventData(new byte[1000000 / 3])
};
Assert.That(async() => await producer.SendAsync(events), Throws.Nothing);
}
}
}
public async Task ClientCanRetrievePartitionProperties(TransportType transportType)
{
var partitionCount = 4;
await using (EventHubScope scope = await EventHubScope.CreateAsync(partitionCount))
{
var clientOptions = new EventHubClientOptions();
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
ConnectionStringProperties connectionProperties = ConnectionStringParser.Parse(connectionString);
var credential = new SharedAccessSignatureCredential
(
new SharedAccessSignature
(
$"{ clientOptions.TransportType.GetUriScheme() }://{ connectionProperties.Endpoint.Host }/{ connectionProperties.EventHubName }".ToLowerInvariant(),
connectionProperties.SharedAccessKeyName,
connectionProperties.SharedAccessKey,
TimeSpan.FromHours(4)
)
);
await using (var client = new EventHubClient(connectionProperties.Endpoint.Host, connectionProperties.EventHubName, credential, new EventHubClientOptions {
TransportType = transportType
}))
{
var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(20));
EventHubProperties properties = await client.GetPropertiesAsync();
var partition = properties.PartitionIds.First();
PartitionProperties partitionProperties = await client.GetPartitionPropertiesAsync(partition, cancellation.Token);
Assert.That(partitionProperties, Is.Not.Null, "A set of partition properties should have been returned.");
Assert.That(partitionProperties.Id, Is.EqualTo(partition), "The partition identifier should match.");
Assert.That(partitionProperties.EventHubName, Is.EqualTo(connectionProperties.EventHubName).Using((IEqualityComparer <string>)StringComparer.InvariantCultureIgnoreCase), "The Event Hub path should match.");
Assert.That(partitionProperties.BeginningSequenceNumber, Is.Not.EqualTo(default(long)), "The beginning sequence number should have been populated.");
Assert.That(partitionProperties.LastEnqueuedSequenceNumber, Is.Not.EqualTo(default(long)), "The last sequence number should have been populated.");
Assert.That(partitionProperties.LastEnqueuedOffset, Is.Not.EqualTo(default(long)), "The last offset should have been populated.");
}
}
}
public async Task ClientCannotRetrieveMetadataWhenProxyIsInvalid()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var clientOptions = new EventHubClientOptions
{
Proxy = new WebProxy("http://1.2.3.4:9999"),
TransportType = TransportType.AmqpWebSockets
};
await using (var client = new EventHubClient(connectionString))
await using (var invalidProxyClient = new EventHubClient(connectionString, clientOptions))
{
var partition = (await client.GetPartitionIdsAsync()).First();
Assert.That(async() => await invalidProxyClient.GetPartitionIdsAsync(), Throws.InstanceOf <WebSocketException>());
Assert.That(async() => await invalidProxyClient.GetPropertiesAsync(), Throws.InstanceOf <WebSocketException>());
Assert.That(async() => await invalidProxyClient.GetPartitionPropertiesAsync(partition), Throws.InstanceOf <WebSocketException>());
}
}
}
public async Task ReceiverWithOptionsCanReceive()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var options = new EventReceiverOptions
{
ConsumerGroup = "$Default",
BeginReceivingAt = EventPosition.NewEventsOnly
};
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
await using (var receiver = client.CreateReceiver(partition, options))
{
Assert.That(async() => await receiver.ReceiveAsync(1, TimeSpan.Zero), Throws.Nothing);
}
}
}
}
public async Task ConnectionTransportCannotRetrieveMetadataWhenClosed()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var connection = new TestConnectionWithTransport(connectionString))
{
var partition = (await connection.GetPartitionIdsAsync()).First();
Assert.That(async() => await connection.GetPropertiesAsync(), Throws.Nothing);
Assert.That(async() => await connection.GetPartitionPropertiesAsync(partition), Throws.Nothing);
await connection.CloseAsync();
await Task.Delay(TimeSpan.FromSeconds(5));
Assert.That(async() => await connection.GetPartitionIdsAsync(), Throws.InstanceOf <EventHubsException>().And.Property(nameof(EventHubsException.Reason)).EqualTo(EventHubsException.FailureReason.ClientClosed));
Assert.That(async() => await connection.GetPropertiesAsync(), Throws.InstanceOf <EventHubsException>().And.Property(nameof(EventHubsException.Reason)).EqualTo(EventHubsException.FailureReason.ClientClosed));
Assert.That(async() => await connection.GetPartitionPropertiesAsync(partition), Throws.InstanceOf <EventHubsException>().And.Property(nameof(EventHubsException.Reason)).EqualTo(EventHubsException.FailureReason.ClientClosed));
}
}
}
public async Task PartitionReceiverCanRetrievePartitionProperties(EventHubsTransportType transportType)
{
var partitionCount = 1;
await using (EventHubScope scope = await EventHubScope.CreateAsync(partitionCount))
{
var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(20));
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var receiverOptions = new PartitionReceiverOptions {
ConnectionOptions = new EventHubConnectionOptions {
TransportType = transportType
}
};
var partitionId = default(string);
await using (var producer = new EventHubProducerClient(connectionString))
{
partitionId = (await producer.GetPartitionIdsAsync(cancellationSource.Token)).First();
}
await using (var receiver = new PartitionReceiver(EventHubConsumerClient.DefaultConsumerGroupName, partitionId, EventPosition.Earliest, connectionString, receiverOptions))
{
var partitionProperties = await receiver.GetPartitionPropertiesAsync(cancellationSource.Token);
Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled.");
Assert.That(partitionProperties, Is.Not.Null, "A set of partition properties should have been returned.");
Assert.That(partitionProperties.Id, Is.EqualTo(partitionId), "The partition identifier should match.");
Assert.That(partitionProperties.EventHubName, Is.EqualTo(scope.EventHubName).Using((IEqualityComparer <string>)StringComparer.InvariantCultureIgnoreCase), "The Event Hub path should match.");
Assert.That(partitionProperties.BeginningSequenceNumber, Is.Not.EqualTo(default(long)), "The beginning sequence number should have been populated.");
Assert.That(partitionProperties.LastEnqueuedSequenceNumber, Is.Not.EqualTo(default(long)), "The last sequence number should have been populated.");
Assert.That(partitionProperties.LastEnqueuedOffset, Is.Not.EqualTo(default(long)), "The last offset should have been populated.");
}
}
}
public async Task ProducerCanSendMultipleBatchesOfEventsUsingAPartitionHashHey()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var batchOptions = new SendOptions {
PartitionKey = "some123key-!d"
};
for (var index = 0; index < 5; ++index)
{
var events = Enumerable
.Range(0, 25)
.Select(index => new EventData(Encoding.UTF8.GetBytes(new String((char)(65 + index), index + 5))));
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
Assert.That(async() => await producer.SendAsync(events, batchOptions), Throws.Nothing, $"Batch { index } should not have thrown an exception.");
}
}
}
}
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
long?checkpointedSequenceNumber = default;
var partitionId = (await client.GetPartitionIdsAsync()).First();
await using (EventHubProducer producer = client.CreateProducer())
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
{
// Send a few dummy events. We are not expecting to receive these.
var dummyEventsCount = 30;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;
// Send the events we expect to receive.
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create a partition manager and add an ownership with a checkpoint in it.
var partitionManager = new InMemoryPartitionManager();
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
new PartitionOwnership(client.FullyQualifiedNamespace, client.EventHubName,
EventHubConsumer.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow)
});
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
partitionManager,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
public async Task ProducerSendsEventsWithTheSamePartitionHashKeyToTheSamePartition()
{
var partitions = 10;
var partitionKey = "some123key-!d";
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
var batches = 5;
var partitionIds = await client.GetPartitionIdsAsync();
var partitionsCount = 0;
var receivedEventsCount = 0;
var consumers = new List <EventHubConsumer>();
try
{
for (var index = 0; index < partitions; index++)
{
consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest));
// Initiate an operation to force the consumer to connect and set its position at the
// end of the event stream.
await consumers[index].ReceiveAsync(1, TimeSpan.Zero);
}
// Send the batches of events.
var batchOptions = new SendOptions {
PartitionKey = partitionKey
};
for (var index = 0; index < batches; index++)
{
await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes($"Just a few messages ({ index })")), batchOptions);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
foreach (var consumer in consumers)
{
var receivedEvents = new List <EventData>();
var index = 0;
while (++index < ReceiveRetryLimit)
{
receivedEvents.AddRange(await consumer.ReceiveAsync(batches + 10, TimeSpan.FromMilliseconds(25)));
}
if (receivedEvents.Count > 0)
{
partitionsCount++;
receivedEventsCount += receivedEvents.Count;
foreach (var receivedEvent in receivedEvents)
{
Assert.That(receivedEvent.PartitionKey, Is.EqualTo(partitionKey));
}
}
}
}
finally
{
foreach (var consumer in consumers)
{
consumer.Close();
}
}
Assert.That(partitionsCount, Is.EqualTo(1));
Assert.That(receivedEventsCount, Is.EqualTo(batches));
}
}
}
public async Task ProducerSendsEventsInTheSameBatchToTheSamePartition()
{
var partitions = 10;
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
var eventBatch = Enumerable
.Range(0, 30)
.Select(index => new EventData(Encoding.UTF8.GetBytes("I'm getting used to this amount of messages")))
.ToList();
var partitionIds = await client.GetPartitionIdsAsync();
var partitionsCount = 0;
var receivedEventsCount = 0;
var consumers = new List <EventHubConsumer>();
try
{
for (var index = 0; index < partitions; index++)
{
consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest));
// Initiate an operation to force the consumer to connect and set its position at the
// end of the event stream.
await consumers[index].ReceiveAsync(1, TimeSpan.Zero);
}
// Send the batch of events.
await producer.SendAsync(eventBatch);
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
foreach (var consumer in consumers)
{
var receivedEvents = new List <EventData>();
var index = 0;
while (++index < ReceiveRetryLimit)
{
receivedEvents.AddRange(await consumer.ReceiveAsync(eventBatch.Count + 10, TimeSpan.FromMilliseconds(25)));
}
if (receivedEvents.Count > 0)
{
partitionsCount++;
receivedEventsCount += receivedEvents.Count;
}
}
}
finally
{
foreach (var consumer in consumers)
{
consumer.Close();
}
}
Assert.That(partitionsCount, Is.EqualTo(1));
Assert.That(receivedEventsCount, Is.EqualTo(eventBatch.Count));
}
}
}
public async Task ProducerDoesNotSendToSpecificPartitionWhenPartitionIdIsNotSpecified(bool nullPartition)
{
var partitions = 10;
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var producerOptions = new EventHubProducerOptions {
};
if (nullPartition)
{
producerOptions.PartitionId = null;
}
await using (var producer = client.CreateProducer(producerOptions))
{
var batches = 30;
var partitionIds = await client.GetPartitionIdsAsync();
var partitionsCount = 0;
var consumers = new List <EventHubConsumer>();
try
{
for (var index = 0; index < partitions; index++)
{
consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest));
// Initiate an operation to force the consumer to connect and set its position at the
// end of the event stream.
await consumers[index].ReceiveAsync(1, TimeSpan.Zero);
}
// Send the batches of events.
for (var index = 0; index < batches; index++)
{
await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes("It's not healthy to send so many messages")));
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
foreach (var consumer in consumers)
{
var receivedEvents = new List <EventData>();
var index = 0;
while (++index < ReceiveRetryLimit)
{
receivedEvents.AddRange(await consumer.ReceiveAsync(batches + 10, TimeSpan.FromMilliseconds(25)));
}
if (receivedEvents.Count > 0)
{
partitionsCount++;
}
}
}
finally
{
await Task.WhenAll(consumers.Select(consumer => consumer.CloseAsync()));
}
Assert.That(partitionsCount, Is.GreaterThan(1));
}
}
}
}
public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
// Send some events.
EventData lastEvent;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
var partitionId = (await client.GetPartitionIdsAsync()).First();
await using (EventHubProducer producer = client.CreateProducer())
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
{
// Send a few events. We are only interested in the last one of them.
var dummyEventsCount = 10;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
lastEvent = receivedEvents.Last();
}
// Create a partition manager so we can retrieve the created checkpoint from it.
var partitionManager = new InMemoryPartitionManager();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
partitionManager,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Any())
{
partitionContext.UpdateCheckpointAsync(eventsList.Last());
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName);
Assert.That(ownershipEnumerable, Is.Not.Null);
Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));
PartitionOwnership ownership = ownershipEnumerable.Single();
Assert.That(ownership.Offset.HasValue, Is.True);
Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset));
Assert.That(ownership.SequenceNumber.HasValue, Is.True);
Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber));
}
}
}
public async Task EventProcessorCanReceiveFromSpecifiedInitialEventPosition()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
DateTimeOffset enqueuedTime;
await using (EventHubProducer producer = client.CreateProducer())
{
// Send a few dummy events. We are not expecting to receive these.
for (int i = 0; i < 30; i++)
{
await producer.SendAsync(dummyEvent);
}
// Wait a reasonable amount of time so the events are able to reach the service.
await Task.Delay(1000);
// Send the events we expect to receive.
enqueuedTime = DateTimeOffset.UtcNow;
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
options: new EventProcessorOptions {
InitialEventPosition = EventPosition.FromEnqueuedTime(enqueuedTime)
},
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
public async Task EventProcessorWaitsMaximumReceiveWaitTimeForEvents(int maximumWaitTimeInSecs)
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var timestamps = new ConcurrentDictionary <string, List <DateTimeOffset> >();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
options: new EventProcessorOptions {
MaximumReceiveWaitTime = TimeSpan.FromSeconds(maximumWaitTimeInSecs)
},
onInitialize: partitionContext =>
timestamps.TryAdd(partitionContext.PartitionId, new List <DateTimeOffset> {
DateTimeOffset.UtcNow
}),
onProcessEvents: (partitionContext, events, cancellationToken) =>
timestamps.AddOrUpdate
(
// The key already exists, so the 'addValue' factory will never be called.
partitionContext.PartitionId,
partitionId => null,
(partitionId, list) =>
{
list.Add(DateTimeOffset.UtcNow);
return(list);
}
)
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
foreach (KeyValuePair <string, List <DateTimeOffset> > kvp in timestamps)
{
var partitionId = kvp.Key;
List <DateTimeOffset> partitionTimestamps = kvp.Value;
Assert.That(partitionTimestamps.Count, Is.GreaterThan(1), $"{ partitionId }: more timestamp samples were expected.");
for (int index = 1; index < partitionTimestamps.Count; index++)
{
var elapsedTime = partitionTimestamps[index].Subtract(partitionTimestamps[index - 1]).TotalSeconds;
Assert.That(elapsedTime, Is.GreaterThan(maximumWaitTimeInSecs - 0.1), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too short.");
Assert.That(elapsedTime, Is.LessThan(maximumWaitTimeInSecs + 5), $"{ partitionId }: elapsed time between indexes { index - 1 } and { index } was too long.");
++index;
}
}
}
}
}
public async Task EventProcessorCannotReceiveMoreThanMaximumMessageCountMessagesAtATime(int maximumMessageCount)
{
var partitions = 2;
await using (EventHubScope scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var unexpectedMessageCount = -1;
// Send some events.
await using (EventHubProducer producer = client.CreateProducer())
{
var eventSet = Enumerable
.Range(0, 20 * maximumMessageCount)
.Select(index => new EventData(new byte[10]))
.ToList();
// Send one set per partition.
for (int i = 0; i < partitions; i++)
{
await producer.SendAsync(eventSet);
}
}
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
options: new EventProcessorOptions {
MaximumMessageCount = maximumMessageCount
},
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
// In case we find a message count greater than the allowed amount, we only store the first
// occurrence and ignore the subsequent ones.
if (eventsList.Count > maximumMessageCount)
{
Interlocked.CompareExchange(ref unexpectedMessageCount, eventsList.Count, -1);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(unexpectedMessageCount, Is.EqualTo(-1), $"A set of { unexpectedMessageCount } events was received.");
}
}
}
