public async void GivenCollectionOfMeasurements_WhenAddAsync_ThenAllEventsAreSentInASingleBatch_Test()
{
var mockEventDataBatch = EventHubsModelFactory.EventDataBatch(
10000,
new List <EventData>(),
new CreateBatchOptions()
{
PartitionKey = "partition123",
},
(data) => true);
_eventHubService.CreateEventDataBatchAsync(Arg.Any <string>()).Returns(mockEventDataBatch);
var measurements = Enumerable.Range(0, 100).Select(i =>
{
var mockMeasurement = Substitute.For <IMeasurement>();
mockMeasurement.DeviceId.Returns($"deviceId_{i}");
return(mockMeasurement);
});
await _measurementCollector.AddAsync(measurements, default);
await _eventHubService.Received(1).CreateEventDataBatchAsync("123");
await _eventHubService.Received(1)
.SendAsync(
Arg.Is <EventDataBatch>(data => data.Count == 100),
default);
}
public async void GivenCollectionOfMeasurements_WhenAddAsync_AndAEventIsToBigToSend_ThenEventsIsSkipped_Test()
{
var eventDataBatch = EventHubsModelFactory.EventDataBatch(
10000,
new List <EventData>(),
new CreateBatchOptions()
{
PartitionKey = "partition123",
},
(data) =>
{
var measurement = data.EventBody.ToObjectFromJson <Measurement>(null);
return(measurement.DeviceId != "deviceId_5");
});
_eventHubService.CreateEventDataBatchAsync(Arg.Any <string>()).Returns(eventDataBatch);
var measurements = Enumerable.Range(0, 10).Select(i =>
{
var mockMeasurement = Substitute.For <IMeasurement>();
mockMeasurement.DeviceId.Returns($"deviceId_{i}");
return(mockMeasurement);
});
await _measurementCollector.AddAsync(measurements, default);
await _eventHubService.Received(2).CreateEventDataBatchAsync("123");
await _eventHubService.Received(1)
.SendAsync(
Arg.Is <EventDataBatch>(data => data.Count == 9),
default);
}
public void PartitionContextDefaultsLastEnqueuedEventProperties()
{
var partition = "0";
var context = EventHubsModelFactory.PartitionContext(partition);
Assert.That(context, Is.Not.Null, "The context should have been created.");
Assert.That(context.PartitionId, Is.EqualTo(partition), "The context should have been set.");
Assert.That(context.ReadLastEnqueuedEventProperties(), Is.EqualTo(new LastEnqueuedEventProperties()), "Reading last enqueued event properties should return a default instance.");
}
public void PartitionContextInitializesProperties()
{
var fakeDate = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero);
var partition = "0";
var properties = EventHubsModelFactory.LastEnqueuedEventProperties(465, 988, fakeDate, fakeDate);
var context = EventHubsModelFactory.PartitionContext(partition, properties);
Assert.That(context, Is.Not.Null, "The context should have been created.");
Assert.That(context.PartitionId, Is.EqualTo(partition), "The context should have been set.");
Assert.That(context.ReadLastEnqueuedEventProperties(), Is.EqualTo(properties), "The last enqueued event properties should have been set.");
}
public void EventHubPropertiesInitializesProperties()
{
var name = "fakename";
var createdOn = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero);
var partitions = new[] { "0", "2", "4", "8" };
var properties = EventHubsModelFactory.EventHubProperties(name, createdOn, partitions);
Assert.That(properties, Is.Not.Null, "The properties should have been created.");
Assert.That(properties.Name, Is.EqualTo(name), "The name should have been set.");
Assert.That(properties.CreatedOn, Is.EqualTo(createdOn), "The creation date/time should have been set.");
Assert.That(properties.PartitionIds, Is.EquivalentTo(partitions), "The partition identifiers should have been set.");
}
public void EventDataBatchIsSafeToDispose()
{
var size = 1024;
var store = new List <EventData> {
new EventData(new BinaryData(Array.Empty <byte>())), new EventData(new BinaryData(Array.Empty <byte>()))
};
var options = new CreateBatchOptions {
MaximumSizeInBytes = 2048
};
var batch = EventHubsModelFactory.EventDataBatch(size, store, options, _ => false);
Assert.That(() => batch.Dispose(), Throws.Nothing);
}
public void PartitionPublishingPropertiesInitializesProperties()
{
var isIdempotentPublishingEnabled = true;
var producerGroupId = 675;
var ownerLevel = (short)12;
var lastPublishedSequenceNumber = 4;
var properties = EventHubsModelFactory.PartitionPublishingProperties(isIdempotentPublishingEnabled, producerGroupId, ownerLevel, lastPublishedSequenceNumber);
Assert.That(properties, Is.Not.Null, "The properties should have been created.");
Assert.That(properties.IsIdempotentPublishingEnabled, Is.EqualTo(isIdempotentPublishingEnabled), "The idempotent publishing flag should have been set.");
Assert.That(properties.ProducerGroupId, Is.EqualTo(producerGroupId), "The producer group should have been set.");
Assert.That(properties.OwnerLevel, Is.EqualTo(ownerLevel), "The owner level should have been set.");
Assert.That(properties.LastPublishedSequenceNumber, Is.EqualTo(lastPublishedSequenceNumber), "The last sequence number should have been set.");
}
public void LastEnqueuedEventPropertiesInitializesProperties()
{
var lastSequence = long.MaxValue - 100;
var lastOffset = long.MaxValue - 10;
var lastEnqueued = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero);
var lastReceived = new DateTimeOffset(2012, 03, 04, 08, 0, 0, TimeSpan.Zero);
var properties = EventHubsModelFactory.LastEnqueuedEventProperties(lastSequence, lastOffset, lastEnqueued, lastReceived);
Assert.That(properties, Is.Not.Null, "The properties should have been created.");
Assert.That(properties.SequenceNumber, Is.EqualTo(lastSequence), "The sequence number should have been set.");
Assert.That(properties.Offset, Is.EqualTo(lastOffset), "The offset should have been set.");
Assert.That(properties.EnqueuedTime, Is.EqualTo(lastEnqueued), "The enqueued date/time should have been set.");
Assert.That(properties.LastReceivedTime, Is.EqualTo(lastReceived), "The last received date/time should have been set.");
}
public void PartitionContextDefaultsLastEnqueuedEventProperties()
{
var fullyQualifiedNamespace = "fakeNamespace";
var eventHubName = "fakeHub";
var consumerGroup = "fakeConsumerGroup";
var partition = "0";
var context = EventHubsModelFactory.PartitionContext(fullyQualifiedNamespace, eventHubName, consumerGroup, partition);
Assert.That(context, Is.Not.Null, "The context should have been created.");
Assert.That(context.FullyQualifiedNamespace, Is.EqualTo(fullyQualifiedNamespace), "The namespace should have been set.");
Assert.That(context.EventHubName, Is.EqualTo(eventHubName), "The event hub name should have been set.");
Assert.That(context.ConsumerGroup, Is.EqualTo(consumerGroup), "The consumer group should have been set.");
Assert.That(context.PartitionId, Is.EqualTo(partition), "The partition should have been set.");
Assert.That(context.ReadLastEnqueuedEventProperties(), Is.EqualTo(new LastEnqueuedEventProperties()), "Reading last enqueued event properties should return a default instance.");
}
public async Task ProducerCanPublishBatchesAfterAnException()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);
var connectionString = EventHubsTestEnvironment.Instance.BuildConnectionStringForEventHub(scope.EventHubName);
var options = new EventHubProducerClientOptions {
EnableIdempotentPartitions = true
};
await using var producer = new EventHubProducerClient(connectionString, options);
var partition = (await producer.GetPartitionIdsAsync()).First();
var batchOptions = new CreateBatchOptions {
PartitionId = partition
};
// Publish a batch to validate that the initial publish works.
using var firstBatch = await producer.CreateBatchAsync(batchOptions, cancellationSource.Token);
firstBatch.TryAdd(EventGenerator.CreateEvents(1).First());
Assert.That(async() => await producer.SendAsync(firstBatch, cancellationSource.Token), Throws.Nothing, "The first publishing operation was not successful.");
// Publish an event too large to succeed; this will force the producer to deal with an exception, which should
// update idempotent state.
var producerId = (await producer.GetPartitionPublishingPropertiesAsync(partition, cancellationSource.Token)).ProducerGroupId;
using var badBatch = EventHubsModelFactory.EventDataBatch(firstBatch.MaximumSizeInBytes + 1000, new List <EventData>(new[] { new EventData(EventGenerator.CreateRandomBody(firstBatch.MaximumSizeInBytes + 1000)) }), new CreateBatchOptions { PartitionId = partition });
Assert.That(async() => await producer.SendAsync(badBatch, cancellationSource.Token), Throws.InstanceOf <EventHubsException>(), "The attempt to publish a too-large event should fail.");
// Publish a second batch of events; this will prove that the producer recovered from the exception.
using var secondBatch = await producer.CreateBatchAsync(batchOptions, cancellationSource.Token);
secondBatch.TryAdd(EventGenerator.CreateEvents(1).First());
secondBatch.TryAdd(EventGenerator.CreateEvents(1).First());
Assert.That(async() => await producer.SendAsync(secondBatch, cancellationSource.Token), Throws.Nothing, "The second publishing operation was not successful.");
var newProducerId = (await producer.GetPartitionPublishingPropertiesAsync(partition, cancellationSource.Token)).ProducerGroupId;
Assert.That(newProducerId, Is.Not.Null, "The producer group identifier should have a value.");
Assert.That(newProducerId, Is.Not.EqualTo(producerId), "The producer group identifier should have been updated after the exception.");
}
}
public void EventDataBatchInitializesProperties()
{
var size = 1024;
var store = new List <EventData> {
new EventData(new BinaryData(Array.Empty <byte>())), new EventData(new BinaryData(Array.Empty <byte>()))
};
var options = new CreateBatchOptions {
MaximumSizeInBytes = 2048
};
var batch = EventHubsModelFactory.EventDataBatch(size, store, options);
Assert.That(batch, Is.Not.Null, "The batch should have been created.");
Assert.That(batch.SizeInBytes, Is.EqualTo(size), "The batch size should have been set.");
Assert.That(batch.MaximumSizeInBytes, Is.EqualTo(options.MaximumSizeInBytes), "The maximum batch size should have been set.");
Assert.That(batch.Count, Is.EqualTo(store.Count), "The batch count should reflect the count of the backing store.");
Assert.That(batch.AsReadOnlyCollection <EventData>(), Is.EquivalentTo(store), "The batch enumerable should reflect the events in the backing store.");
}
public void Enqueue_TransmissionFlow()
{
// assert
ManualResetEventSlim resetEvent = new ManualResetEventSlim();
Mock <IMemoryBuffer <EventData> > buffer = new Mock <IMemoryBuffer <EventData> >();
Mock <ITransmissionBuffer <EventData, EventDataBatch> > aggregator = new Mock <ITransmissionBuffer <EventData, EventDataBatch> >();
Mock <ITransmissionSender <EventDataBatch> > sender = new Mock <ITransmissionSender <EventDataBatch> >();
Mock <ITransmissionStorage <EventData> > storage = new Mock <ITransmissionStorage <EventData> >();
ConcurrentQueue <EventDataBatch> bufferQueue = new ConcurrentQueue <EventDataBatch>();
buffer
.Setup(t => t.Enqueue(It.IsAny <EventData>()))
.Callback((EventData d) => bufferQueue.Enqueue(EventHubsModelFactory.EventDataBatch(1, new[] { d })));
aggregator
.Setup(t => t.Dequeue(It.IsAny <CancellationToken>()))
.Returns(() =>
{
int count = 0;
List <EventDataBatch> results = new List <EventDataBatch>();
while (bufferQueue.TryDequeue(out EventDataBatch d) && count < 10)
{
results.Add(d);
count++;
}
return(results.ToAsyncEnumerable());
});
sender
.Setup(t => t.SendAsync(It.IsAny <IAsyncEnumerable <EventDataBatch> >(), It.IsAny <CancellationToken>()))
.Callback(() => resetEvent.Set());
ITelemetryEventTransmitter transmitter = new EventHubTransmitter(
buffer.Object, aggregator.Object, sender.Object, storage.Object);
TelemetryEvent data = new TelemetryEvent();
// act
transmitter.Enqueue(data);
// arrange
resetEvent.Wait(TimeSpan.FromSeconds(5));
sender.Verify(s => s.SendAsync(
It.IsAny <IAsyncEnumerable <EventDataBatch> >(),
It.IsAny <CancellationToken>()), Times.Once);
}
public void PartitionContextInitializesProperties()
{
var fakeDate = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero);
var fullyQualifiedNamespace = "fakeNamespace";
var eventHubName = "fakeHub";
var consumerGroup = "fakeConsumerGroup";
var partition = "0";
var properties = EventHubsModelFactory.LastEnqueuedEventProperties(465, 988, fakeDate, fakeDate);
var context = EventHubsModelFactory.PartitionContext(fullyQualifiedNamespace, eventHubName, consumerGroup, partition, properties);
Assert.That(context, Is.Not.Null, "The context should have been created.");
Assert.That(context.FullyQualifiedNamespace, Is.EqualTo(fullyQualifiedNamespace), "The namespace should have been set.");
Assert.That(context.EventHubName, Is.EqualTo(eventHubName), "The event hub name should have been set.");
Assert.That(context.ConsumerGroup, Is.EqualTo(consumerGroup), "The consumer group should have been set.");
Assert.That(context.PartitionId, Is.EqualTo(partition), "The partition should have been set.");
Assert.That(context.ReadLastEnqueuedEventProperties(), Is.EqualTo(properties), "The last enqueued event properties should have been set.");
}
public void EventDataBatchRespectsTheTryAddCallback()
{
var eventLimit = 3;
var store = new List <EventData>();
var batch = EventHubsModelFactory.EventDataBatch(5, store, tryAddCallback: _ => store.Count < eventLimit);
while (store.Count < eventLimit)
{
Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Test"))), Is.True, $"The batch contains { store.Count } events; adding another should be permitted.");
}
Assert.That(store.Count, Is.EqualTo(eventLimit), "The batch should be at its limit.");
Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Too many"))), Is.False, "The batch is full; it should not be possible to add a new event.");
Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Too many"))), Is.False, "The batch is full; a second attempt to add a new event should not succeed.");
Assert.That(store.Count, Is.EqualTo(eventLimit), "The batch should be at its limit after the failed TryAdd attempts.");
Assert.That(batch.AsReadOnlyCollection <EventData>(), Is.EquivalentTo(store), "The batch enumerable should reflect the events in the backing store.");
}
public void PartitionPropertiesInitializesProperties()
{
var eventHubName = "eventHubName";
var partitionId = "0";
var isEmpty = false;
var beginningSequenceNumber = 123;
var lastSequenceNumber = 9999;
var lastOffset = 767;
var lastEnqueuedTime = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero);
var properties = EventHubsModelFactory.PartitionProperties(eventHubName, partitionId, isEmpty, beginningSequenceNumber, lastSequenceNumber, lastOffset, lastEnqueuedTime);
Assert.That(properties, Is.Not.Null, "The properties should have been created.");
Assert.That(properties.EventHubName, Is.EqualTo(eventHubName), "The event hub name should have been set.");
Assert.That(properties.Id, Is.EqualTo(partitionId), "The partition identifier should have been set.");
Assert.That(properties.IsEmpty, Is.EqualTo(isEmpty), "The `is empty` flag should have been set.");
Assert.That(properties.BeginningSequenceNumber, Is.EqualTo(beginningSequenceNumber), "The beginning sequence number should have been set.");
Assert.That(properties.LastEnqueuedSequenceNumber, Is.EqualTo(lastSequenceNumber), "The last sequence number should have been set.");
Assert.That(properties.LastEnqueuedOffset, Is.EqualTo(lastOffset), "The last offset should have been set.");
Assert.That(properties.LastEnqueuedTime, Is.EqualTo(lastEnqueuedTime), "The last enqueue date/time should have been set.");
}
public async void GivenCollectionOfMeasurements_WhenAddAsync_AndEventsCannotFitInSingleBatch_ThenEventsAreSentInAMultipeBatches_Test()
{
var count = 0;
var simpleMockEventDataBatch = EventHubsModelFactory.EventDataBatch(
10000,
new List <EventData>(),
new CreateBatchOptions()
{
PartitionKey = "partition123",
});
var splittingEventDataBatch = EventHubsModelFactory.EventDataBatch(
10000,
new List <EventData>(),
new CreateBatchOptions()
{
PartitionKey = "partition123",
},
(data) => count++ != 5); // split at 5 measurement
_eventHubService.CreateEventDataBatchAsync(Arg.Any <string>())
.Returns(splittingEventDataBatch, simpleMockEventDataBatch);
var measurements = Enumerable.Range(0, 10).Select(i =>
{
var mockMeasurement = Substitute.For <IMeasurement>();
mockMeasurement.DeviceId.Returns($"deviceId_{i}");
return(mockMeasurement);
});
await _measurementCollector.AddAsync(measurements, default);
await _eventHubService.Received(2).CreateEventDataBatchAsync("123");
await _eventHubService.Received(2)
.SendAsync(
Arg.Is <EventDataBatch>(data => data.Count == 5),
default);
}
public void EventDataBatchRespectsTheTryAddCallback()
{
var eventLimit = 3;
var converter = new AmqpMessageConverter();
var store = new List <EventData>();
var messages = new List <AmqpMessage>();
var batch = EventHubsModelFactory.EventDataBatch(5, store, tryAddCallback: _ => store.Count < eventLimit);
while (store.Count < eventLimit)
{
var eventData = new EventData(new BinaryData("Test"));
Assert.That(() => batch.TryAdd(eventData), Is.True, $"The batch contains { store.Count } events; adding another should be permitted.");
messages.Add(converter.CreateMessageFromEvent(eventData));
}
Assert.That(store.Count, Is.EqualTo(eventLimit), "The batch should be at its limit.");
Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Too many"))), Is.False, "The batch is full; it should not be possible to add a new event.");
Assert.That(() => batch.TryAdd(new EventData(new BinaryData("Too many"))), Is.False, "The batch is full; a second attempt to add a new event should not succeed.");
Assert.That(store.Count, Is.EqualTo(eventLimit), "The batch should be at its limit after the failed TryAdd attempts.");
Assert.That(batch.AsReadOnlyCollection <AmqpMessage>().Count, Is.EqualTo(eventLimit), "The messages produced by the batch should match the limit.");
}
public void EventDataInitializesProperties()
{
var body = new BinaryData("Hello");
var properties = new Dictionary <string, object> {
{ "id", 12 }
};
var systemProperties = new Dictionary <string, object> {
{ "custom", "sys-value" }
};
var sequenceNumber = long.MaxValue - 512;
var offset = long.MaxValue - 1024;
var enqueueTime = new DateTimeOffset(2015, 10, 27, 12, 0, 0, TimeSpan.Zero);
var partitionKey = "omghai!";
var eventData = EventHubsModelFactory.EventData(body, properties, systemProperties, partitionKey, sequenceNumber, offset, enqueueTime);
Assert.That(eventData, Is.Not.Null, "The event should have been created.");
Assert.That(eventData.EventBody.ToString(), Is.EqualTo(body.ToString()), "The event body should have been set.");
Assert.That(eventData.Properties, Is.EquivalentTo(properties), "The properties should have been set.");
Assert.That(eventData.SystemProperties, Is.EquivalentTo(systemProperties), "The system properties should have been set.");
Assert.That(eventData.PartitionKey, Is.EqualTo(partitionKey), "The partition key should have been set.");
Assert.That(eventData.SequenceNumber, Is.EqualTo(sequenceNumber), "The sequence number should have been set.");
Assert.That(eventData.Offset, Is.EqualTo(offset), "The offset should have been set.");
Assert.That(eventData.EnqueuedTime, Is.EqualTo(enqueueTime), "The sequence number should have been set.");
}
