public async Task CheckpointUpdateFailsWhenContainerDoesNotExist() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, $"test-container-{Guid.NewGuid()}"); var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy); var checkpoint = new EventProcessorCheckpoint { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", PartitionId = "partitionId" }; var mockEvent = new MockEventData( eventBody: Array.Empty <byte>(), offset: 10, sequenceNumber: 20); Assert.That(async() => await checkpointStore.UpdateCheckpointAsync(checkpoint, mockEvent, default), Throws.InstanceOf <RequestFailedException>()); } }
/// <summary> /// Updates the checkpoint using the given information for the associated partition and consumer group in the storage blob service. /// </summary> /// /// <param name="checkpoint">The checkpoint containing the information to be stored.</param> /// <param name="eventData">The event to use as the basis for the checkpoint's starting position.</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param> /// /// <returns>A task to be resolved on when the operation has completed.</returns> /// public override async Task UpdateCheckpointAsync(EventProcessorCheckpoint checkpoint, EventData eventData, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>(); var blobName = string.Format(CheckpointPrefix + checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace.ToLowerInvariant(), checkpoint.EventHubName.ToLowerInvariant(), checkpoint.ConsumerGroup.ToLowerInvariant()); var blobClient = ContainerClient.GetBlobClient(blobName); var metadata = new Dictionary <string, string>() { { BlobMetadataKey.Offset, eventData.Offset.ToString() }, { BlobMetadataKey.SequenceNumber, eventData.SequenceNumber.ToString() } }; Func <CancellationToken, Task> updateCheckpointAsync = async updateCheckpointToken => { using var blobContent = new MemoryStream(Array.Empty <byte>()); await blobClient.UploadAsync(blobContent, metadata : metadata, cancellationToken : updateCheckpointToken).ConfigureAwait(false); }; try { await ApplyRetryPolicy(updateCheckpointAsync, cancellationToken).ConfigureAwait(false); Logger.CheckpointUpdated(checkpoint.PartitionId); } catch (RequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.ContainerNotFound) { Logger.CheckpointUpdateError(checkpoint.PartitionId, ex.ToString()); throw new RequestFailedException(Resources.BlobsResourceDoesNotExist); } }
public void UpdateCheckpointLogsErrorsWhenTheBlobDoesNotExist() { var checkpoint = new EventProcessorCheckpoint { FullyQualifiedNamespace = FullyQualifiedNamespace, EventHubName = EventHubName, ConsumerGroup = ConsumerGroup, PartitionId = PartitionId, }; var expectedException = new DllNotFoundException("BOOM!"); var mockLog = new Mock <BlobEventStoreEventSource>(); var mockContainerClient = new MockBlobContainerClient().AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", client => { client.UploadBlobException = expectedException; }); var target = new BlobsCheckpointStore(mockContainerClient, DefaultRetryPolicy); target.Logger = mockLog.Object; Assert.That(async() => await target.UpdateCheckpointAsync(checkpoint, new EventData(Array.Empty <byte>()), CancellationToken.None), Throws.Exception.EqualTo(expectedException)); mockLog.Verify(log => log.UpdateCheckpointError(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, expectedException.Message)); }
// Any checkpoint returned by GetCheckpointAsync is treated as the authoritative // starting point for the partition; if the return value is null, then the // global DefaultStartingPosition specified by the options is used. protected async override Task <EventProcessorCheckpoint> GetCheckpointAsync( string partitionId, CancellationToken cancellationToken) { EventProcessorCheckpoint checkpoint = await base.GetCheckpointAsync(partitionId, cancellationToken); // If there was no checkpoint, set the starting point for reading from // this specific partition to 5 minutes ago. if (checkpoint == null) { var startingTime = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(5)); checkpoint = new EventProcessorCheckpoint { FullyQualifiedNamespace = this.FullyQualifiedNamespace, EventHubName = this.EventHubName, ConsumerGroup = this.ConsumerGroup, PartitionId = partitionId, StartingPosition = EventPosition.FromEnqueuedTime(startingTime) }; } return(checkpoint); }
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions() { var storageManager = new MockCheckPointStorage(); await storageManager.UpdateCheckpointAsync(new EventProcessorCheckpoint { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", PartitionId = "partitionId1", Offset = 10, SequenceNumber = 20 }); await storageManager.UpdateCheckpointAsync(new EventProcessorCheckpoint { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", PartitionId = "partitionId2", Offset = 10, SequenceNumber = 20 }); IEnumerable <EventProcessorCheckpoint> storedCheckpointsList = await storageManager.ListCheckpointsAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedCheckpointsList, Is.Not.Null); Assert.That(storedCheckpointsList.Count, Is.EqualTo(2)); EventProcessorCheckpoint storedCheckpoint1 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId1"); EventProcessorCheckpoint storedCheckpoint2 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId2"); Assert.That(storedCheckpoint1, Is.Not.Null); Assert.That(storedCheckpoint2, Is.Not.Null); }
/// <summary> /// Updates the checkpoint using the given information for the associated partition and consumer group in the storage blob service. /// </summary> /// /// <param name="checkpoint">The checkpoint containing the information to be stored.</param> /// <param name="eventData">The event to use as the basis for the checkpoint's starting position.</param> /// <param name="cancellationToken">A <see cref="CancellationToken" /> instance to signal the request to cancel the operation.</param> /// /// <returns>A task to be resolved on when the operation has completed.</returns> /// public override async Task UpdateCheckpointAsync(EventProcessorCheckpoint checkpoint, EventData eventData, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>(); UpdateCheckpointStart(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup); var blobName = string.Format(CultureInfo.InvariantCulture, CheckpointPrefix + checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace.ToLowerInvariant(), checkpoint.EventHubName.ToLowerInvariant(), checkpoint.ConsumerGroup.ToLowerInvariant()); var blobClient = ContainerClient.GetBlobClient(blobName); var metadata = new Dictionary <string, string>() { { BlobMetadataKey.Offset, eventData.Offset.ToString(CultureInfo.InvariantCulture) }, { BlobMetadataKey.SequenceNumber, eventData.SequenceNumber.ToString(CultureInfo.InvariantCulture) } }; try { try { // Assume the blob is present and attempt to set the metadata. await ApplyRetryPolicy(token => blobClient.SetMetadataAsync(metadata, cancellationToken: token), cancellationToken).ConfigureAwait(false); } catch (RequestFailedException ex) when((ex.ErrorCode == BlobErrorCode.BlobNotFound) || (ex.ErrorCode == BlobErrorCode.ContainerNotFound)) { // If the blob wasn't present, fall-back to trying to create a new one. await ApplyRetryPolicy(async token => { using var blobContent = new MemoryStream(Array.Empty <byte>()); await blobClient.UploadAsync(blobContent, metadata: metadata, cancellationToken: token).ConfigureAwait(false); }, cancellationToken).ConfigureAwait(false); } } catch (RequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.ContainerNotFound) { UpdateCheckpointError(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, ex); throw new RequestFailedException(BlobsResourceDoesNotExist); } catch (Exception ex) { UpdateCheckpointError(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, ex); throw; } finally { UpdateCheckpointComplete(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup); } }
public async Task UpdateCheckpointLogsStartAndCompleteWhenTheBlobExists() { var checkpoint = new EventProcessorCheckpoint { FullyQualifiedNamespace = FullyQualifiedNamespace, EventHubName = EventHubName, ConsumerGroup = ConsumerGroup, PartitionId = PartitionId, }; var blobInfo = BlobsModelFactory.BlobInfo(new ETag($@"""{MatchingEtag}"""), DateTime.UtcNow); var blobList = new List <BlobItem> { BlobsModelFactory.BlobItem($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/ownership/{Guid.NewGuid().ToString()}", false, BlobsModelFactory.BlobItemProperties(true, lastModified: DateTime.UtcNow, eTag: new ETag(MatchingEtag)), "snapshot", new Dictionary <string, string> { { BlobMetadataKey.OwnerIdentifier, Guid.NewGuid().ToString() } }) }; var mockContainerClient = new MockBlobContainerClient() { Blobs = blobList }; mockContainerClient.AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", client => { client.BlobInfo = blobInfo; client.UploadBlobException = new Exception("Upload should not be called"); }); var target = new BlobsCheckpointStore(mockContainerClient, DefaultRetryPolicy); var mockLog = new Mock <BlobEventStoreEventSource>(); target.Logger = mockLog.Object; await target.UpdateCheckpointAsync(checkpoint, new EventData(Array.Empty <byte>()), CancellationToken.None); mockLog.Verify(log => log.UpdateCheckpointStart(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup)); mockLog.Verify(log => log.UpdateCheckpointComplete(checkpoint.PartitionId, checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup)); }
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy); await checkpointStore.UpdateCheckpointAsync(new EventProcessorCheckpoint { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", PartitionId = "partitionId1", Offset = 10, SequenceNumber = 20 }, default); await checkpointStore.UpdateCheckpointAsync(new EventProcessorCheckpoint { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", PartitionId = "partitionId2", Offset = 10, SequenceNumber = 20 }, default); IEnumerable<EventProcessorCheckpoint> storedCheckpointsList = await checkpointStore.ListCheckpointsAsync("namespace", "eventHubName", "consumerGroup", default); Assert.That(storedCheckpointsList, Is.Not.Null); Assert.That(storedCheckpointsList.Count, Is.EqualTo(2)); EventProcessorCheckpoint storedCheckpoint1 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId1"); EventProcessorCheckpoint storedCheckpoint2 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId2"); Assert.That(storedCheckpoint1, Is.Not.Null); Assert.That(storedCheckpoint2, Is.Not.Null); } }
public void UpdateCheckpointForMissingContainerLogsCheckpointUpdateError() { var checkpoint = new EventProcessorCheckpoint { FullyQualifiedNamespace = FullyQualifiedNamespace, EventHubName = EventHubName, ConsumerGroup = ConsumerGroup, PartitionId = PartitionId }; var ex = new RequestFailedException(404, BlobErrorCode.ContainerNotFound.ToString(), BlobErrorCode.ContainerNotFound.ToString(), null); var mockBlobContainerClient = new MockBlobContainerClient().AddBlobClient($"{FullyQualifiedNamespace}/{EventHubName}/{ConsumerGroup}/checkpoint/1", client => client.UploadBlobException = ex); var target = new BlobsCheckpointStore(mockBlobContainerClient, DefaultRetryPolicy); var mockLog = new Mock <BlobEventStoreEventSource>(); target.Logger = mockLog.Object; Assert.That(async() => await target.UpdateCheckpointAsync(checkpoint, new EventData(Array.Empty <byte>()), CancellationToken.None), Throws.InstanceOf <RequestFailedException>()); mockLog.Verify(m => m.UpdateCheckpointError(PartitionId, FullyQualifiedNamespace, EventHubName, ConsumerGroup, ex.Message)); }
public async Task BlobStorageManagerCanUpdateCheckpoint() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var checkpointStore = new BlobsCheckpointStore(containerClient, DefaultRetryPolicy); var ownershipList = new List <EventProcessorPartitionOwnership> { // Make sure the ownership exists beforehand so we hit all storage SDK calls in the checkpoint store. new EventProcessorPartitionOwnership { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", OwnerIdentifier = "ownerIdentifier", PartitionId = "partitionId" } }; await checkpointStore.ClaimOwnershipAsync(ownershipList, default); var checkpoint = new EventProcessorCheckpoint { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", PartitionId = "partitionId" }; var mockEvent = new MockEventData( eventBody: Array.Empty <byte>(), offset: 10, sequenceNumber: 20); Assert.That(async() => await checkpointStore.UpdateCheckpointAsync(checkpoint, mockEvent, default), Throws.Nothing); } }
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions() { var storageManager = new InMemoryStorageManager(); var mockEvent = new MockEventData( eventBody: Array.Empty <byte>(), offset: 10, sequenceNumber: 20); await storageManager.UpdateCheckpointAsync(new EventProcessorCheckpoint { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", PartitionId = "partitionId1" }, mockEvent); await storageManager.UpdateCheckpointAsync(new EventProcessorCheckpoint { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", PartitionId = "partitionId2" }, mockEvent); IEnumerable <EventProcessorCheckpoint> storedCheckpointsList = await storageManager.ListCheckpointsAsync("namespace", "eventHubName", "consumerGroup"); Assert.That(storedCheckpointsList, Is.Not.Null); Assert.That(storedCheckpointsList.Count, Is.EqualTo(2)); EventProcessorCheckpoint storedCheckpoint1 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId1"); EventProcessorCheckpoint storedCheckpoint2 = storedCheckpointsList.First(checkpoint => checkpoint.PartitionId == "partitionId2"); Assert.That(storedCheckpoint1, Is.Not.Null); Assert.That(storedCheckpoint2, Is.Not.Null); }
public CheckpointData(EventProcessorCheckpoint checkpoint, EventData eventData) { Checkpoint = checkpoint; Event = eventData; }
public async Task CheckpointUpdatesAnExistingBlob() { await using (StorageScope storageScope = await StorageScope.CreateAsync()) { var storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString; var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName); var checkpointStore = new BlobCheckpointStoreInternal(containerClient); var checkpoint = new EventProcessorCheckpoint { FullyQualifiedNamespace = "namespace", EventHubName = "eventHubName", ConsumerGroup = "consumerGroup", PartitionId = "partitionId" }; var mockEvent = new MockEventData( eventBody: Array.Empty <byte>(), offset: 10, sequenceNumber: 20); // Calling update should create the checkpoint. await checkpointStore.UpdateCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, mockEvent.Offset, mockEvent.SequenceNumber, default); var blobCount = 0; var storedCheckpoint = await checkpointStore.GetCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, default); await foreach (var blob in containerClient.GetBlobsAsync()) { ++blobCount; if (blobCount > 1) { break; } } Assert.That(blobCount, Is.EqualTo(1)); Assert.That(storedCheckpoint, Is.Not.Null); Assert.That(storedCheckpoint.StartingPosition, Is.EqualTo(EventPosition.FromOffset(mockEvent.Offset, false))); // Calling update again should update the existing checkpoint. mockEvent = new MockEventData( eventBody: Array.Empty <byte>(), offset: 50, sequenceNumber: 60); await checkpointStore.UpdateCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, mockEvent.Offset, mockEvent.SequenceNumber, default); blobCount = 0; storedCheckpoint = await checkpointStore.GetCheckpointAsync(checkpoint.FullyQualifiedNamespace, checkpoint.EventHubName, checkpoint.ConsumerGroup, checkpoint.PartitionId, default); await foreach (var blob in containerClient.GetBlobsAsync()) { ++blobCount; if (blobCount > 1) { break; } } Assert.That(blobCount, Is.EqualTo(1)); Assert.That(storedCheckpoint, Is.Not.Null); Assert.That(storedCheckpoint.StartingPosition, Is.EqualTo(EventPosition.FromOffset(mockEvent.Offset, false))); } }