public async void GivenCollectionOfMeasurements_WhenAddAsync_ThenAllEventsAreSentInASingleBatch_Test() { var mockEventDataBatch = EventHubsModelFactory.EventDataBatch( 10000, new List <EventData>(), new CreateBatchOptions() { PartitionKey = "partition123", }, (data) => true); _eventHubService.CreateEventDataBatchAsync(Arg.Any <string>()).Returns(mockEventDataBatch); var measurements = Enumerable.Range(0, 100).Select(i => { var mockMeasurement = Substitute.For <IMeasurement>(); mockMeasurement.DeviceId.Returns($"deviceId_{i}"); return(mockMeasurement); }); await _measurementCollector.AddAsync(measurements, default); await _eventHubService.Received(1).CreateEventDataBatchAsync("123"); await _eventHubService.Received(1) .SendAsync( Arg.Is <EventDataBatch>(data => data.Count == 100), default); }
public async void GivenCollectionOfMeasurements_WhenAddAsync_AndAEventIsToBigToSend_ThenEventsIsSkipped_Test() { var eventDataBatch = EventHubsModelFactory.EventDataBatch( 10000, new List <EventData>(), new CreateBatchOptions() { PartitionKey = "partition123", }, (data) => { var measurement = data.EventBody.ToObjectFromJson <Measurement>(null); return(measurement.DeviceId != "deviceId_5"); }); _eventHubService.CreateEventDataBatchAsync(Arg.Any <string>()).Returns(eventDataBatch); var measurements = Enumerable.Range(0, 10).Select(i => { var mockMeasurement = Substitute.For <IMeasurement>(); mockMeasurement.DeviceId.Returns($"deviceId_{i}"); return(mockMeasurement); }); await _measurementCollector.AddAsync(measurements, default); await _eventHubService.Received(2).CreateEventDataBatchAsync("123"); await _eventHubService.Received(1) .SendAsync( Arg.Is <EventDataBatch>(data => data.Count == 9), default); }
public void PartitionContextDefaultsLastEnqueuedEventProperties() { var partition = "0"; var context = EventHubsModelFactory.PartitionContext(partition); Assert.That(context, Is.Not.Null, "The context should have been created."); Assert.That(context.PartitionId, Is.EqualTo(partition), "The context should have been set."); Assert.That(context.ReadLastEnqueuedEventProperties(), Is.EqualTo(new LastEnqueuedEventProperties()), "Reading last enqueued event properties should return a default instance."); }
public void PartitionContextInitializesProperties() { var fakeDate = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero); var partition = "0"; var properties = EventHubsModelFactory.LastEnqueuedEventProperties(465, 988, fakeDate, fakeDate); var context = EventHubsModelFactory.PartitionContext(partition, properties); Assert.That(context, Is.Not.Null, "The context should have been created."); Assert.That(context.PartitionId, Is.EqualTo(partition), "The context should have been set."); Assert.That(context.ReadLastEnqueuedEventProperties(), Is.EqualTo(properties), "The last enqueued event properties should have been set."); }
public void EventHubPropertiesInitializesProperties() { var name = "fakename"; var createdOn = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero); var partitions = new[] { "0", "2", "4", "8" }; var properties = EventHubsModelFactory.EventHubProperties(name, createdOn, partitions); Assert.That(properties, Is.Not.Null, "The properties should have been created."); Assert.That(properties.Name, Is.EqualTo(name), "The name should have been set."); Assert.That(properties.CreatedOn, Is.EqualTo(createdOn), "The creation date/time should have been set."); Assert.That(properties.PartitionIds, Is.EquivalentTo(partitions), "The partition identifiers should have been set."); }
public void EventDataBatchIsSafeToDispose() { var size = 1024; var store = new List <EventData> { new EventData(new BinaryData(Array.Empty <byte>())), new EventData(new BinaryData(Array.Empty <byte>())) }; var options = new CreateBatchOptions { MaximumSizeInBytes = 2048 }; var batch = EventHubsModelFactory.EventDataBatch(size, store, options, _ => false); Assert.That(() => batch.Dispose(), Throws.Nothing); }
public void PartitionPublishingPropertiesInitializesProperties() { var isIdempotentPublishingEnabled = true; var producerGroupId = 675; var ownerLevel = (short)12; var lastPublishedSequenceNumber = 4; var properties = EventHubsModelFactory.PartitionPublishingProperties(isIdempotentPublishingEnabled, producerGroupId, ownerLevel, lastPublishedSequenceNumber); Assert.That(properties, Is.Not.Null, "The properties should have been created."); Assert.That(properties.IsIdempotentPublishingEnabled, Is.EqualTo(isIdempotentPublishingEnabled), "The idempotent publishing flag should have been set."); Assert.That(properties.ProducerGroupId, Is.EqualTo(producerGroupId), "The producer group should have been set."); Assert.That(properties.OwnerLevel, Is.EqualTo(ownerLevel), "The owner level should have been set."); Assert.That(properties.LastPublishedSequenceNumber, Is.EqualTo(lastPublishedSequenceNumber), "The last sequence number should have been set."); }
public void LastEnqueuedEventPropertiesInitializesProperties() { var lastSequence = long.MaxValue - 100; var lastOffset = long.MaxValue - 10; var lastEnqueued = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero); var lastReceived = new DateTimeOffset(2012, 03, 04, 08, 0, 0, TimeSpan.Zero); var properties = EventHubsModelFactory.LastEnqueuedEventProperties(lastSequence, lastOffset, lastEnqueued, lastReceived); Assert.That(properties, Is.Not.Null, "The properties should have been created."); Assert.That(properties.SequenceNumber, Is.EqualTo(lastSequence), "The sequence number should have been set."); Assert.That(properties.Offset, Is.EqualTo(lastOffset), "The offset should have been set."); Assert.That(properties.EnqueuedTime, Is.EqualTo(lastEnqueued), "The enqueued date/time should have been set."); Assert.That(properties.LastReceivedTime, Is.EqualTo(lastReceived), "The last received date/time should have been set."); }
public void PartitionContextDefaultsLastEnqueuedEventProperties() { var fullyQualifiedNamespace = "fakeNamespace"; var eventHubName = "fakeHub"; var consumerGroup = "fakeConsumerGroup"; var partition = "0"; var context = EventHubsModelFactory.PartitionContext(fullyQualifiedNamespace, eventHubName, consumerGroup, partition); Assert.That(context, Is.Not.Null, "The context should have been created."); Assert.That(context.FullyQualifiedNamespace, Is.EqualTo(fullyQualifiedNamespace), "The namespace should have been set."); Assert.That(context.EventHubName, Is.EqualTo(eventHubName), "The event hub name should have been set."); Assert.That(context.ConsumerGroup, Is.EqualTo(consumerGroup), "The consumer group should have been set."); Assert.That(context.PartitionId, Is.EqualTo(partition), "The partition should have been set."); Assert.That(context.ReadLastEnqueuedEventProperties(), Is.EqualTo(new LastEnqueuedEventProperties()), "Reading last enqueued event properties should return a default instance."); }
public async Task ProducerCanPublishBatchesAfterAnException() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit); var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName); var options = new EventHubProducerClientOptions { EnableIdempotentPartitions = true }; await using var producer = new EventHubProducerClient(connectionString, options); var partition = (await producer.GetPartitionIdsAsync()).First(); var batchOptions = new CreateBatchOptions { PartitionId = partition }; // Publish a batch to validate that the initial publish works. using var firstBatch = await producer.CreateBatchAsync(batchOptions, cancellationSource.Token); firstBatch.TryAdd(EventGenerator.CreateEvents(1).First()); Assert.That(async() => await producer.SendAsync(firstBatch, cancellationSource.Token), Throws.Nothing, "The first publishing operation was not successful."); // Publish an event too large to succeed; this will force the producer to deal with an exception, which should // update idempotent state. var producerId = (await producer.GetPartitionPublishingPropertiesAsync(partition, cancellationSource.Token)).ProducerGroupId; using var badBatch = EventHubsModelFactory.EventDataBatch(firstBatch.MaximumSizeInBytes + 1000, new List <EventData>(new[] { new EventData(EventGenerator.CreateRandomBody(firstBatch.MaximumSizeInBytes + 1000)) }), new CreateBatchOptions { PartitionId = partition }); Assert.That(async() => await producer.SendAsync(badBatch, cancellationSource.Token), Throws.InstanceOf <EventHubsException>(), "The attempt to publish a too-large event should fail."); // Publish a second batch of events; this will prove that the producer recovered from the exception. using var secondBatch = await producer.CreateBatchAsync(batchOptions, cancellationSource.Token); secondBatch.TryAdd(EventGenerator.CreateEvents(1).First()); secondBatch.TryAdd(EventGenerator.CreateEvents(1).First()); Assert.That(async() => await producer.SendAsync(secondBatch, cancellationSource.Token), Throws.Nothing, "The second publishing operation was not successful."); var newProducerId = (await producer.GetPartitionPublishingPropertiesAsync(partition, cancellationSource.Token)).ProducerGroupId; Assert.That(newProducerId, Is.Not.Null, "The producer group identifier should have a value."); Assert.That(newProducerId, Is.Not.EqualTo(producerId), "The producer group identifier should have been updated after the exception."); } }
public void EventDataBatchInitializesProperties() { var size = 1024; var store = new List <EventData> { new EventData(new BinaryData(Array.Empty <byte>())), new EventData(new BinaryData(Array.Empty <byte>())) }; var options = new CreateBatchOptions { MaximumSizeInBytes = 2048 }; var batch = EventHubsModelFactory.EventDataBatch(size, store, options); Assert.That(batch, Is.Not.Null, "The batch should have been created."); Assert.That(batch.SizeInBytes, Is.EqualTo(size), "The batch size should have been set."); Assert.That(batch.MaximumSizeInBytes, Is.EqualTo(options.MaximumSizeInBytes), "The maximum batch size should have been set."); Assert.That(batch.Count, Is.EqualTo(store.Count), "The batch count should reflect the count of the backing store."); Assert.That(batch.AsReadOnlyCollection <EventData>(), Is.EquivalentTo(store), "The batch enumerable should reflect the events in the backing store."); }
public void Enqueue_TransmissionFlow() { // assert ManualResetEventSlim resetEvent = new ManualResetEventSlim(); Mock <IMemoryBuffer <EventData> > buffer = new Mock <IMemoryBuffer <EventData> >(); Mock <ITransmissionBuffer <EventData, EventDataBatch> > aggregator = new Mock <ITransmissionBuffer <EventData, EventDataBatch> >(); Mock <ITransmissionSender <EventDataBatch> > sender = new Mock <ITransmissionSender <EventDataBatch> >(); Mock <ITransmissionStorage <EventData> > storage = new Mock <ITransmissionStorage <EventData> >(); ConcurrentQueue <EventDataBatch> bufferQueue = new ConcurrentQueue <EventDataBatch>(); buffer .Setup(t => t.Enqueue(It.IsAny <EventData>())) .Callback((EventData d) => bufferQueue.Enqueue(EventHubsModelFactory.EventDataBatch(1, new[] { d }))); aggregator .Setup(t => t.Dequeue(It.IsAny <CancellationToken>())) .Returns(() => { int count = 0; List <EventDataBatch> results = new List <EventDataBatch>(); while (bufferQueue.TryDequeue(out EventDataBatch d) && count < 10) { results.Add(d); count++; } return(results.ToAsyncEnumerable()); }); sender .Setup(t => t.SendAsync(It.IsAny <IAsyncEnumerable <EventDataBatch> >(), It.IsAny <CancellationToken>())) .Callback(() => resetEvent.Set()); ITelemetryEventTransmitter transmitter = new EventHubTransmitter( buffer.Object, aggregator.Object, sender.Object, storage.Object); TelemetryEvent data = new TelemetryEvent(); // act transmitter.Enqueue(data); // arrange resetEvent.Wait(TimeSpan.FromSeconds(5)); sender.Verify(s => s.SendAsync( It.IsAny <IAsyncEnumerable <EventDataBatch> >(), It.IsAny <CancellationToken>()), Times.Once); }
public void PartitionContextInitializesProperties() { var fakeDate = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero); var fullyQualifiedNamespace = "fakeNamespace"; var eventHubName = "fakeHub"; var consumerGroup = "fakeConsumerGroup"; var partition = "0"; var properties = EventHubsModelFactory.LastEnqueuedEventProperties(465, 988, fakeDate, fakeDate); var context = EventHubsModelFactory.PartitionContext(fullyQualifiedNamespace, eventHubName, consumerGroup, partition, properties); Assert.That(context, Is.Not.Null, "The context should have been created."); Assert.That(context.FullyQualifiedNamespace, Is.EqualTo(fullyQualifiedNamespace), "The namespace should have been set."); Assert.That(context.EventHubName, Is.EqualTo(eventHubName), "The event hub name should have been set."); Assert.That(context.ConsumerGroup, Is.EqualTo(consumerGroup), "The consumer group should have been set."); Assert.That(context.PartitionId, Is.EqualTo(partition), "The partition should have been set."); Assert.That(context.ReadLastEnqueuedEventProperties(), Is.EqualTo(properties), "The last enqueued event properties should have been set."); }
public void EventDataBatchRespectsTheTryAddCallback() { var eventLimit = 3; var store = new List <EventData>(); var batch = EventHubsModelFactory.EventDataBatch(5, store, tryAddCallback: _ => store.Count < eventLimit); while (store.Count < eventLimit) { Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Test"))), Is.True, $"The batch contains { store.Count } events; adding another should be permitted."); } Assert.That(store.Count, Is.EqualTo(eventLimit), "The batch should be at its limit."); Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Too many"))), Is.False, "The batch is full; it should not be possible to add a new event."); Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Too many"))), Is.False, "The batch is full; a second attempt to add a new event should not succeed."); Assert.That(store.Count, Is.EqualTo(eventLimit), "The batch should be at its limit after the failed TryAdd attempts."); Assert.That(batch.AsReadOnlyCollection <EventData>(), Is.EquivalentTo(store), "The batch enumerable should reflect the events in the backing store."); }
public void PartitionPropertiesInitializesProperties() { var eventHubName = "eventHubName"; var partitionId = "0"; var isEmpty = false; var beginningSequenceNumber = 123; var lastSequenceNumber = 9999; var lastOffset = 767; var lastEnqueuedTime = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero); var properties = EventHubsModelFactory.PartitionProperties(eventHubName, partitionId, isEmpty, beginningSequenceNumber, lastSequenceNumber, lastOffset, lastEnqueuedTime); Assert.That(properties, Is.Not.Null, "The properties should have been created."); Assert.That(properties.EventHubName, Is.EqualTo(eventHubName), "The event hub name should have been set."); Assert.That(properties.Id, Is.EqualTo(partitionId), "The partition identifier should have been set."); Assert.That(properties.IsEmpty, Is.EqualTo(isEmpty), "The `is empty` flag should have been set."); Assert.That(properties.BeginningSequenceNumber, Is.EqualTo(beginningSequenceNumber), "The beginning sequence number should have been set."); Assert.That(properties.LastEnqueuedSequenceNumber, Is.EqualTo(lastSequenceNumber), "The last sequence number should have been set."); Assert.That(properties.LastEnqueuedOffset, Is.EqualTo(lastOffset), "The last offset should have been set."); Assert.That(properties.LastEnqueuedTime, Is.EqualTo(lastEnqueuedTime), "The last enqueue date/time should have been set."); }
public async void GivenCollectionOfMeasurements_WhenAddAsync_AndEventsCannotFitInSingleBatch_ThenEventsAreSentInAMultipeBatches_Test() { var count = 0; var simpleMockEventDataBatch = EventHubsModelFactory.EventDataBatch( 10000, new List <EventData>(), new CreateBatchOptions() { PartitionKey = "partition123", }); var splittingEventDataBatch = EventHubsModelFactory.EventDataBatch( 10000, new List <EventData>(), new CreateBatchOptions() { PartitionKey = "partition123", }, (data) => count++ != 5); // split at 5 measurement _eventHubService.CreateEventDataBatchAsync(Arg.Any <string>()) .Returns(splittingEventDataBatch, simpleMockEventDataBatch); var measurements = Enumerable.Range(0, 10).Select(i => { var mockMeasurement = Substitute.For <IMeasurement>(); mockMeasurement.DeviceId.Returns($"deviceId_{i}"); return(mockMeasurement); }); await _measurementCollector.AddAsync(measurements, default); await _eventHubService.Received(2).CreateEventDataBatchAsync("123"); await _eventHubService.Received(2) .SendAsync( Arg.Is <EventDataBatch>(data => data.Count == 5), default); }
public void EventDataBatchRespectsTheTryAddCallback() { var eventLimit = 3; var converter = new AmqpMessageConverter(); var store = new List <EventData>(); var messages = new List <AmqpMessage>(); var batch = EventHubsModelFactory.EventDataBatch(5, store, tryAddCallback: _ => store.Count < eventLimit); while (store.Count < eventLimit) { var eventData = new EventData(new BinaryData("Test")); Assert.That(() => batch.TryAdd(eventData), Is.True, $"The batch contains { store.Count } events; adding another should be permitted."); messages.Add(converter.CreateMessageFromEvent(eventData)); } Assert.That(store.Count, Is.EqualTo(eventLimit), "The batch should be at its limit."); Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Too many"))), Is.False, "The batch is full; it should not be possible to add a new event."); Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Too many"))), Is.False, "The batch is full; a second attempt to add a new event should not succeed."); Assert.That(store.Count, Is.EqualTo(eventLimit), "The batch should be at its limit after the failed TryAdd attempts."); Assert.That(batch.AsReadOnlyCollection <AmqpMessage>().Count, Is.EqualTo(eventLimit), "The messages produced by the batch should match the limit."); }
public void EventDataInitializesProperties() { var body = new BinaryData("Hello"); var properties = new Dictionary <string, object> { { "id", 12 } }; var systemProperties = new Dictionary <string, object> { { "custom", "sys-value" } }; var sequenceNumber = long.MaxValue - 512; var offset = long.MaxValue - 1024; var enqueueTime = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero); var partitionKey = "omghai!"; var eventData = EventHubsModelFactory.EventData(body, properties, systemProperties, partitionKey, sequenceNumber, offset, enqueueTime); Assert.That(eventData, Is.Not.Null, "The event should have been created."); Assert.That(eventData.EventBody.ToString(), Is.EqualTo(body.ToString()), "The event body should have been set."); Assert.That(eventData.Properties, Is.EquivalentTo(properties), "The properties should have been set."); Assert.That(eventData.SystemProperties, Is.EquivalentTo(systemProperties), "The system properties should have been set."); Assert.That(eventData.PartitionKey, Is.EqualTo(partitionKey), "The partition key should have been set."); Assert.That(eventData.SequenceNumber, Is.EqualTo(sequenceNumber), "The sequence number should have been set."); Assert.That(eventData.Offset, Is.EqualTo(offset), "The offset should have been set."); Assert.That(eventData.EnqueuedTime, Is.EqualTo(enqueueTime), "The sequence number should have been set."); }