public void ClaimOwnershipLogsErrors()
{
var partitionOwnership = new List <EventProcessorPartitionOwnership>
{
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = FullyQualifiedNamespace,
EventHubName = EventHubName,
ConsumerGroup = ConsumerGroup,
OwnerIdentifier = OwnershipIdentifier,
PartitionId = PartitionId,
LastModifiedTime = DateTime.UtcNow
}
};
var expectedException = new DllNotFoundException("BOOM!");
var mockLog = new Mock <BlobEventStoreEventSource>();
var mockContainerClient = new MockBlobContainerClient().AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/1", client => client.UploadBlobException = expectedException);
var target = new BlobCheckpointStoreInternal(mockContainerClient);
target.Logger = mockLog.Object;
Assert.That(async() => await target.ClaimOwnershipAsync(partitionOwnership, CancellationToken.None), Throws.Exception.EqualTo(expectedException));
mockLog.Verify(m => m.ClaimOwnershipError(PartitionId, FullyQualifiedNamespace, EventHubName, ConsumerGroup, OwnershipIdentifier, expectedException.Message));
}
public async Task BlobStorageManagerCanClaimOwnership(string version)
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownershipList = new List <EventProcessorPartitionOwnership>
{
// Null version and non-null version hit different paths of the code, calling different methods that connect
// to the Storage service.
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId",
Version = version
}
};
Assert.That(async() => await checkpointStore.ClaimOwnershipAsync(ownershipList, default), Throws.Nothing);
}
}
public void ClaimOwnershipForMissingPartitionLogsClaimOwnershipError()
{
var partitionOwnership = new List <EventProcessorPartitionOwnership>
{
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = FullyQualifiedNamespace,
EventHubName = EventHubName,
ConsumerGroup = ConsumerGroup,
OwnerIdentifier = OwnershipIdentifier,
PartitionId = PartitionId,
LastModifiedTime = DateTime.UtcNow,
Version = MatchingEtag
}
};
var mockBlobContainerClient = new MockBlobContainerClient().AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/1", _ => { });
var target = new BlobCheckpointStoreInternal(mockBlobContainerClient);
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
Assert.That(async() => await target.ClaimOwnershipAsync(partitionOwnership, CancellationToken.None), Throws.InstanceOf <RequestFailedException>());
mockLog.Verify(m => m.ClaimOwnershipError(PartitionId, FullyQualifiedNamespace, EventHubName, ConsumerGroup, OwnershipIdentifier, It.Is <string>(e => e.Contains(BlobErrorCode.BlobNotFound.ToString()))));
}
public async Task ClaimOwnershipForExistingPartitionWithWrongEtagLogsOwnershipNotClaimable()
{
var blobInfo = BlobsModelFactory.BlobInfo(new ETag($@"""{WrongEtag}"""), DateTime.UtcNow);
var partitionOwnership = new List <EventProcessorPartitionOwnership>
{
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = FullyQualifiedNamespace,
EventHubName = EventHubName,
ConsumerGroup = ConsumerGroup,
OwnerIdentifier = OwnershipIdentifier,
PartitionId = PartitionId,
LastModifiedTime = DateTime.UtcNow,
Version = MatchingEtag
}
};
var mockContainerClient = new MockBlobContainerClient().AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/1", client => client.BlobInfo = blobInfo);
var target = new BlobCheckpointStoreInternal(mockContainerClient);
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
var result = await target.ClaimOwnershipAsync(partitionOwnership, CancellationToken.None);
mockLog.Verify(m => m.OwnershipNotClaimable(PartitionId, FullyQualifiedNamespace, EventHubName, ConsumerGroup, OwnershipIdentifier, It.Is <string>(e => e.Contains(BlobErrorCode.ConditionNotMet.ToString()))));
}
public async Task ListOwnershipLogsStartAndComplete()
{
var blobList = new List <BlobItem>
{
BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}",
false,
BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)),
"snapshot",
new Dictionary <string, string> {
{ BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() }
})
};
var target = new BlobCheckpointStoreInternal(new MockBlobContainerClient()
{
Blobs = blobList
});
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
await target.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup, CancellationToken.None);
mockLog.Verify(m => m.ListOwnershipStart(FullyQualifiedNamespace, EventHubName, ConsumerGroup));
mockLog.Verify(m => m.ListOwnershipComplete(FullyQualifiedNamespace, EventHubName, ConsumerGroup, blobList.Count));
}
public async Task UpdateCheckpointLogsStartAndCompleteWhenTheBlobExists()
{
var blobInfo = BlobsModelFactory.BlobInfo(new ETag($@"""{MatchingEtag}"""), DateTime.UtcNow);
var blobList = new List <BlobItem>
{
BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}",
false,
BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)),
"snapshot",
new Dictionary <string, string> {
{ BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() }
})
};
var mockContainerClient = new MockBlobContainerClient()
{
Blobs = blobList
};
mockContainerClient.AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", client =>
{
client.BlobInfo = blobInfo;
client.UploadBlobException = new Exception("Upload should not be called");
});
var target = new BlobCheckpointStoreInternal(mockContainerClient);
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
await target.UpdateCheckpointAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup, PartitionId, 0, default, CancellationToken.None);
public async Task OwnershipClaimSetsLastModifiedTimeAndVersion()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownershipList = new List <EventProcessorPartitionOwnership>();
var ownership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId"
};
ownershipList.Add(ownership);
await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
Assert.That(ownership.LastModifiedTime, Is.Not.EqualTo(default(DateTimeOffset)));
Assert.That(ownership.LastModifiedTime, Is.GreaterThan(DateTimeOffset.UtcNow.Subtract(TimeSpan.FromSeconds(5))));
Assert.That(ownership.Version, Is.Not.Null);
}
}
public async Task OwnershipClaimFailsWhenVersionExistsAndOwnershipDoesNotExist()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var eTaggyOwnership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId",
Version = "ETag"
};
await checkpointStore.ClaimOwnershipAsync(new[] { eTaggyOwnership }, default);
var storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(storedOwnershipList, Is.Not.Null.And.Empty);
}
}
public async Task CheckReturnedEtagContainsSingleQuotes()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
// A regular expression used to capture strings enclosed in double quotes.
Regex s_doubleQuotesExpression = new Regex("\"(.*)\"", RegexOptions.Compiled);
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownershipList = new List <EventProcessorPartitionOwnership>();
var ownership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId"
};
ownershipList.Add(ownership);
var claimedOwnership = await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
var storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default);
var claimedOwnershipMatch = s_doubleQuotesExpression.Match(claimedOwnership.First().Version);
var storedOwnershipListMatch = s_doubleQuotesExpression.Match(storedOwnershipList.First().Version);
Assert.That(claimedOwnershipMatch.Success, Is.False);
Assert.That(storedOwnershipListMatch.Success, Is.False);
}
}
public async Task FirstOwnershipClaimSucceeds()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownershipList = new List <EventProcessorPartitionOwnership>();
var ownership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId"
};
ownershipList.Add(ownership);
await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
var storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(storedOwnershipList, Is.Not.Null);
Assert.That(storedOwnershipList.Count, Is.EqualTo(1));
Assert.That(storedOwnershipList.Single().IsEquivalentTo(ownership), Is.True);
}
}
public async Task ClaimOwnershipLogsStartAndComplete()
{
var partitionOwnership = new List <EventProcessorPartitionOwnership>
{
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = FullyQualifiedNamespace,
EventHubName = EventHubName,
ConsumerGroup = ConsumerGroup,
OwnerIdentifier = OwnershipIdentifier,
PartitionId = PartitionId,
LastModifiedTime = DateTime.UtcNow
}
};
var mockBlobContainerClient = new MockBlobContainerClient().AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/1", _ => { });
var target = new BlobCheckpointStoreInternal(mockBlobContainerClient);
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
var result = await target.ClaimOwnershipAsync(partitionOwnership, CancellationToken.None);
mockLog.Verify(m => m.ClaimOwnershipStart(PartitionId, FullyQualifiedNamespace, EventHubName, ConsumerGroup, OwnershipIdentifier));
mockLog.Verify(m => m.ClaimOwnershipComplete(PartitionId, FullyQualifiedNamespace, EventHubName, ConsumerGroup, OwnershipIdentifier));
}
public EventHubListener(
string functionId,
ITriggeredFunctionExecutor executor,
EventProcessorHost eventProcessorHost,
bool singleDispatch,
IEventHubConsumerClient consumerClient,
BlobCheckpointStoreInternal checkpointStore,
EventHubOptions options,
ILoggerFactory loggerFactory)
{
_loggerFactory = loggerFactory;
_executor = executor;
_eventProcessorHost = eventProcessorHost;
_singleDispatch = singleDispatch;
_checkpointStore = checkpointStore;
_options = options;
_logger = _loggerFactory.CreateLogger <EventHubListener>();
_scaleMonitor = new Lazy <EventHubsScaleMonitor>(
() => new EventHubsScaleMonitor(
functionId,
consumerClient,
checkpointStore,
_loggerFactory.CreateLogger <EventHubsScaleMonitor>()));
_details = $"'namespace='{eventProcessorHost?.FullyQualifiedNamespace}', eventHub='{eventProcessorHost?.EventHubName}', " +
$"consumerGroup='{eventProcessorHost?.ConsumerGroup}', functionId='{functionId}', singleDispatch='{singleDispatch}'";
}
public async Task BlobStorageManagerCanUpdateCheckpoint()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownershipList = new List <EventProcessorPartitionOwnership>
{
// Make sure the ownership exists beforehand so we hit all storage SDK calls in the checkpoint store.
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId"
}
};
await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
Assert.That(async() => await checkpointStore.UpdateCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId", 10, 20, default), Throws.Nothing);
}
}
public async Task StartProcessingAsync(
IEventProcessorFactory processorFactory,
BlobCheckpointStoreInternal checkpointStore,
CancellationToken cancellationToken)
{
_processorFactory = processorFactory;
_checkpointStore = checkpointStore;
await StartProcessingAsync(cancellationToken).ConfigureAwait(false);
}
public async Task BlobStorageManagerCanGetACheckpoint()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
Assert.That(async() => await checkpointStore.GetCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partition", default), Throws.Nothing);
}
}
public void ListOwnershipLogsErrorOnException()
{
var ex = new RequestFailedException(0, "foo", BlobErrorCode.ContainerNotFound.ToString(), null);
var target = new BlobCheckpointStoreInternal(new MockBlobContainerClient(getBlobsAsyncException: ex));
var mockLog = new Mock <BlobEventStoreEventSource>();
target.Logger = mockLog.Object;
Assert.That(async() => await target.ListOwnershipAsync(FullyQualifiedNamespace, EventHubName, ConsumerGroup, CancellationToken.None), Throws.InstanceOf <RequestFailedException>());
mockLog.Verify(m => m.ListOwnershipError(FullyQualifiedNamespace, EventHubName, ConsumerGroup, ex.Message));
}
public async Task CheckpointUpdateFailsWhenContainerDoesNotExist()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, $"test-container-{Guid.NewGuid()}");
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
Assert.That(async() => await checkpointStore.UpdateCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId", 10, 20, default), Throws.InstanceOf <RequestFailedException>());
}
}
public async Task GetCheckpointReturnsNullIfTheContainerDoesNotExist()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, $"test-container-{Guid.NewGuid()}");
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var checkpoints = await checkpointStore.GetCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partition", default);
Assert.That(checkpoints, Is.Null);
}
}
public async Task ListOwnershipAsyncReturnsEmptyIEnumerableWhenThereAreNoOwnership()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownership = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(ownership, Is.Not.Null.And.Empty);
}
}
public EventHubsScaleMonitor(
string functionId,
IEventHubConsumerClient client,
BlobCheckpointStoreInternal checkpointStore,
ILogger logger)
{
_functionId = functionId;
_logger = logger;
_checkpointStore = checkpointStore;
_nextPartitionLogTime = DateTime.UtcNow;
_nextPartitionWarningTime = DateTime.UtcNow;
_client = client;
Descriptor = new ScaleMonitorDescriptor($"{_functionId}-EventHubTrigger-{_client.EventHubName}-{_client.ConsumerGroup}".ToLowerInvariant());
}
public Task <ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
ParameterInfo parameter = context.Parameter;
EventHubTriggerAttribute attribute = parameter.GetCustomAttribute <EventHubTriggerAttribute>(inherit: false);
if (attribute == null)
{
return(Task.FromResult <ITriggerBinding>(null));
}
Func <ListenerFactoryContext, bool, Task <IListener> > createListener =
(factoryContext, singleDispatch) =>
{
var options = _options.Value;
var checkpointStore = new BlobCheckpointStoreInternal(
_clientFactory.GetCheckpointStoreClient(),
factoryContext.Descriptor.Id,
_loggerFactory.CreateLogger <BlobCheckpointStoreInternal>());
IListener listener = new EventHubListener(
factoryContext.Descriptor.Id,
factoryContext.Executor,
_clientFactory.GetEventProcessorHost(attribute.EventHubName, attribute.Connection, attribute.ConsumerGroup, singleDispatch),
singleDispatch,
_clientFactory.GetEventHubConsumerClient(attribute.EventHubName, attribute.Connection, attribute.ConsumerGroup),
checkpointStore,
options,
_loggerFactory);
return(Task.FromResult(listener));
};
#pragma warning disable 618
ITriggerBinding binding = BindingFactory.GetTriggerBinding(new EventHubTriggerBindingStrategy(), parameter, _converterManager, createListener);
#pragma warning restore 618
ITriggerBinding eventHubTriggerBindingWrapper = new EventHubTriggerBindingWrapper(binding);
return(Task.FromResult(eventHubTriggerBindingWrapper));
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
await checkpointStore.UpdateCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId1", 10, 20, default);
await checkpointStore.UpdateCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId2", 10, 20, default);
var storedCheckpoint1 = await checkpointStore.GetCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId1", default);
var storedCheckpoint2 = await checkpointStore.GetCheckpointAsync("namespace", "eventHubName", "consumerGroup", "partitionId2", default);
Assert.That(storedCheckpoint1, Is.Not.Null);
Assert.That(storedCheckpoint2, Is.Not.Null);
}
}
public async Task OwnershipClaimSucceedsWhenVersionIsValid()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var firstOwnership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId"
};
await checkpointStore.ClaimOwnershipAsync(new[] { firstOwnership }, default);
// The first ownership's version should have been set by the checkpoint store.
var secondOwnership = new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = "partitionId",
Version = firstOwnership.Version
};
await checkpointStore.ClaimOwnershipAsync(new[] { secondOwnership }, default);
var storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(storedOwnershipList, Is.Not.Null);
Assert.That(storedOwnershipList.Count, Is.EqualTo(1));
Assert.That(storedOwnershipList.Single().IsEquivalentTo(secondOwnership), Is.True);
}
}
public async Task ClaimOwnershipAsyncCanClaimMultipleOwnership()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownershipList = new List <EventProcessorPartitionOwnership>();
var ownershipCount = 5;
for (var i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = $"partitionId { i }"
});
}
await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
var storedOwnershipList = await checkpointStore.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup", default);
Assert.That(storedOwnershipList, Is.Not.Null);
Assert.That(storedOwnershipList.Count, Is.EqualTo(ownershipCount));
var index = 0;
foreach (EventProcessorPartitionOwnership ownership in storedOwnershipList.OrderBy(ownership => ownership.PartitionId))
{
Assert.That(ownership.IsEquivalentTo(ownershipList[index]), Is.True, $"Ownership of partition '{ ownership.PartitionId }' should be equivalent.");
++index;
}
}
}
public void GetCheckpointsAsync_LogsOnRequestErrors()
{
var testLoggerProvider = new TestLoggerProvider();
Mock <BlobContainerClient> containerClientMock = new Mock <BlobContainerClient>(MockBehavior.Strict);
containerClientMock.Setup(c => c.GetBlobClient(It.IsAny <string>()))
.Throws(new RequestFailedException("Uh oh"));
BlobCheckpointStoreInternal store = new BlobCheckpointStoreInternal(
containerClientMock.Object,
_functionId,
testLoggerProvider.CreateLogger("TestLogger")
);
Assert.ThrowsAsync <RequestFailedException>(async() => await store.GetCheckpointAsync(_namespace, _eventHubName, _consumerGroup, _partitionId, CancellationToken.None));
var warning = testLoggerProvider.GetAllLogMessages().Single(p => p.Level == LogLevel.Warning);
var expectedWarning = "Function 'EventHubsTriggerFunction': An exception occurred when retrieving a checkpoint for " +
"FullyQualifiedNamespace: 'TestNamespace'; EventHubName: 'TestEventHubName'; ConsumerGroup: 'TestConsumerGroup'; PartitionId: '0'.";
Assert.AreEqual(expectedWarning, warning.FormattedMessage);
testLoggerProvider.ClearAllLogMessages();
}
public async Task CheckpointUpdatesAnExistingBlob()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var checkpoint = new EventProcessorCheckpoint
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
PartitionId = "partitionId"
};
var mockEvent = new MockEventData(
eventBody: Array.Empty <byte>(),
offset: 10,
sequenceNumber: 20);
// Calling update should create the checkpoint.
await checkpointStore.UpdateCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, mockEvent.Offset, mockEvent.SequenceNumber, default);
var blobCount = 0;
var storedCheckpoint = await checkpointStore.GetCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, default);
await foreach (var blob in containerClient.GetBlobsAsync())
{
++blobCount;
if (blobCount > 1)
{
break;
}
}
Assert.That(blobCount, Is.EqualTo(1));
Assert.That(storedCheckpoint, Is.Not.Null);
Assert.That(storedCheckpoint.StartingPosition, Is.EqualTo(EventPosition.FromOffset(mockEvent.Offset, false)));
// Calling update again should update the existing checkpoint.
mockEvent = new MockEventData(
eventBody: Array.Empty <byte>(),
offset: 50,
sequenceNumber: 60);
await checkpointStore.UpdateCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, mockEvent.Offset, mockEvent.SequenceNumber, default);
blobCount = 0;
storedCheckpoint = await checkpointStore.GetCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, default);
await foreach (var blob in containerClient.GetBlobsAsync())
{
++blobCount;
if (blobCount > 1)
{
break;
}
}
Assert.That(blobCount, Is.EqualTo(1));
Assert.That(storedCheckpoint, Is.Not.Null);
Assert.That(storedCheckpoint.StartingPosition, Is.EqualTo(EventPosition.FromOffset(mockEvent.Offset, false)));
}
}
public async Task ClaimOwnershipAsyncReturnsOnlyTheSuccessfullyClaimedOwnership()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var checkpointStore = new BlobCheckpointStoreInternal(containerClient);
var ownershipList = new List <EventProcessorPartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = $"{i}"
});
}
await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
// The versions must have been set by the checkpoint store.
var versions = ownershipList.Select(ownership => ownership.Version).ToList();
ownershipList.Clear();
// Use a valid eTag when 'i' is odd. This way, we can expect 'ownershipCount / 2' successful
// claims (rounded down).
var expectedClaimedCount = ownershipCount / 2;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new EventProcessorPartitionOwnership
{
FullyQualifiedNamespace = "namespace",
EventHubName = "eventHubName",
ConsumerGroup = "consumerGroup",
OwnerIdentifier = "ownerIdentifier",
PartitionId = $"{i}",
Version = i % 2 == 1 ? versions[i] : null
});
}
var claimedOwnershipList = await checkpointStore.ClaimOwnershipAsync(ownershipList, default);
var expectedOwnership = ownershipList.Where(ownership => int.Parse(ownership.PartitionId) % 2 == 1);
Assert.That(claimedOwnershipList, Is.Not.Null);
Assert.That(claimedOwnershipList.Count, Is.EqualTo(expectedClaimedCount));
var index = 0;
foreach (EventProcessorPartitionOwnership ownership in claimedOwnershipList.OrderBy(ownership => ownership.PartitionId))
{
Assert.That(ownership.IsEquivalentTo(expectedOwnership.ElementAt(index)), Is.True, $"Ownership of partition '{ ownership.PartitionId }' should be equivalent.");
++index;
}
}
}
