public static async Task<EventHubMonitor> CreateAsync( Configuration configuration, TimeSpan pauseBetweenParitions, TimeSpan pauseAfterAllPartitions ) { var nsm = CreateNamespaceManager(configuration, pauseBetweenParitions); // We create a partially applied function so that // we'll only need to pass the partition id later. Func<string, Task<PartitionDescription>> getEventHubPartitionAsync = partitionId => nsm.GetEventHubPartitionAsync(configuration.EventHubName, configuration.ConsumerGroupName, partitionId); var eventhub = await nsm.GetEventHubAsync(configuration.EventHubName).ConfigureAwait(false); // Construct a reference to the blob container. var storageAccount = CloudStorageAccount.Parse(configuration.CheckpointStorageAccount); var blobClient = storageAccount.CreateCloudBlobClient(); var checkpointContainer = blobClient.GetContainerReference(configuration.EventHubName); // The `PartitionCheckpointManager` manages the complexity // of getting the latest checkpoint for each parition. var checkpoints = new PartitionCheckpointManager( configuration.ConsumerGroupName, eventhub.PartitionIds, checkpointContainer ); return new EventHubMonitor( eventhub.PartitionIds, checkpoints.GetLastCheckpointAsync, getEventHubPartitionAsync, pauseBetweenParitions, pauseAfterAllPartitions); }
public static async Task <EventHubMonitor> CreateAsync( Configuration configuration, TimeSpan pauseBetweenParitions, TimeSpan pauseAfterAllPartitions ) { var nsm = CreateNamespaceManager(configuration, pauseBetweenParitions); // We create a partially applied function so that // we'll only need to pass the partition id later. Func <string, Task <PartitionDescription> > getEventHubPartitionAsync = partitionId => nsm.GetEventHubPartitionAsync(configuration.EventHubName, configuration.ConsumerGroupName, partitionId); var eventhub = await nsm.GetEventHubAsync(configuration.EventHubName).ConfigureAwait(false); // Construct a reference to the blob container. var storageAccount = CloudStorageAccount.Parse(configuration.CheckpointStorageAccount); var blobClient = storageAccount.CreateCloudBlobClient(); var checkpointContainer = blobClient.GetContainerReference(configuration.EventHubName); // The `PartitionCheckpointManager` manages the complexity // of getting the latest checkpoint for each parition. var checkpoints = new PartitionCheckpointManager( configuration.ConsumerGroupName, eventhub.PartitionIds, checkpointContainer ); return(new EventHubMonitor( eventhub.PartitionIds, checkpoints.GetLastCheckpointAsync, getEventHubPartitionAsync, pauseBetweenParitions, pauseAfterAllPartitions)); }
public async Task should_retrieve_the_checkpoint() { var wasRetrieved = false; Func<CloudBlockBlob, Task<PartitionCheckpoint>> getCheckpoint = _ => { wasRetrieved = true; return Task.FromResult(new PartitionCheckpoint()); }; var manager = new PartitionCheckpointManager( ConsumerGroupName, new[] { PartitionId }, GetCloudBlobContainerStub(), _ => Task.FromResult(new DateTimeOffset()), getCheckpoint); await manager.GetLastCheckpointAsync(PartitionId); Assert.True(wasRetrieved); }
public async Task should_check_the_last_modified_date_of_the_blob() { var dateWasChecked = false; Func<CloudBlockBlob, Task<DateTimeOffset>> getLastModified = _ => { dateWasChecked = true; return Task.FromResult(new DateTimeOffset()); }; var manager = new PartitionCheckpointManager( ConsumerGroupName, new[] { PartitionId }, GetCloudBlobContainerStub(), getLastModified, _ => Task.FromResult(new PartitionCheckpoint())); await manager.GetLastCheckpointAsync(PartitionId); Assert.True(dateWasChecked); }
public when_looking_up_the_blob_reference_for_a_partition() { var manager = new PartitionCheckpointManager( ConsumerGroupName, new[] { PartitionId }, GetCloudBlobContainerStub(), b => { _blobForLastModified = b; return Task.FromResult(new DateTimeOffset()); }, b => { _blobForRetrieval = b; return Task.FromResult(new PartitionCheckpoint()); }); manager.GetLastCheckpointAsync(PartitionId).Wait(); }
public when_accessing_multiple_partitions() { // create a set of differing "last modified dates" var lastModifiedDates = _partitionIds .Select((id, index) => new { id, index }) .ToDictionary( x => GetBlobName(x.id), x => new DateTimeOffset().AddDays(x.index)); // keep track of how many times we attempt to retrieve // the checkpoint for each partition _retrievalCounts = _partitionIds .ToDictionary( GetBlobName, id => 0); _manager = new PartitionCheckpointManager( ConsumerGroupName, _partitionIds, GetCloudBlobContainerStub(), blob => Task.FromResult(lastModifiedDates[blob.Name]), blob => { _retrievalCounts[blob.Name]++; return Task.FromResult(new PartitionCheckpoint()); }); }
public async Task should_throw() { var manager = new PartitionCheckpointManager( ConsumerGroupName, new string[] { }, GetCloudBlobContainerStub(), _ => Task.FromResult(new DateTimeOffset()), _ => Task.FromResult(new PartitionCheckpoint())); await Assert.ThrowsAnyAsync<ArgumentException>(async () => { await manager.GetLastCheckpointAsync("unknown"); }); }
public async Task should_return_a_cached_checkpoint() { var lastModified = new DateTimeOffset(); Func<CloudBlockBlob, Task<DateTimeOffset>> getLastModified = _ => Task.FromResult(lastModified); Func<CloudBlockBlob, Task<PartitionCheckpoint>> getCheckpoint = _ => Task.FromResult(new PartitionCheckpoint()); var manager = new PartitionCheckpointManager( ConsumerGroupName, new[] { PartitionId }, GetCloudBlobContainerStub(), getLastModified, getCheckpoint); var checkpoint1 = await manager.GetLastCheckpointAsync(PartitionId); var checkpoint2 = await manager.GetLastCheckpointAsync(PartitionId); Assert.Equal(checkpoint1, checkpoint2); }
public async Task should_not_retrieve_the_checkpoint() { var timesRetrieved = 0; var lastModified = new DateTimeOffset(); Func<CloudBlockBlob, Task<DateTimeOffset>> getLastModified = _ => Task.FromResult(lastModified); Func<CloudBlockBlob, Task<PartitionCheckpoint>> getCheckpoint = _ => { timesRetrieved++; return Task.FromResult(new PartitionCheckpoint()); }; var manager = new PartitionCheckpointManager( ConsumerGroupName, new[] { PartitionId }, GetCloudBlobContainerStub(), getLastModified, getCheckpoint); await manager.GetLastCheckpointAsync(PartitionId); // since the last modified stamp hasn't changed, // we should not attempt to retrieve the blob await manager.GetLastCheckpointAsync(PartitionId); Assert.Equal(1, timesRetrieved); }