public static async Task<EventHubMonitor> CreateAsync(
            Configuration configuration,
            TimeSpan pauseBetweenParitions,
            TimeSpan pauseAfterAllPartitions
            )
        {
            var nsm = CreateNamespaceManager(configuration, pauseBetweenParitions);

            // We create a partially applied function so that 
            // we'll only need to pass the partition id later.
            Func<string, Task<PartitionDescription>> getEventHubPartitionAsync =
                partitionId => nsm.GetEventHubPartitionAsync(configuration.EventHubName, configuration.ConsumerGroupName, partitionId);

            var eventhub = await nsm.GetEventHubAsync(configuration.EventHubName).ConfigureAwait(false);

            // Construct a reference to the blob container.
            var storageAccount = CloudStorageAccount.Parse(configuration.CheckpointStorageAccount);
            var blobClient = storageAccount.CreateCloudBlobClient();
            var checkpointContainer = blobClient.GetContainerReference(configuration.EventHubName);

            // The `PartitionCheckpointManager` manages the complexity 
            // of getting the latest checkpoint for each parition.
            var checkpoints = new PartitionCheckpointManager(
                configuration.ConsumerGroupName,
                eventhub.PartitionIds,
                checkpointContainer
                );

            return new EventHubMonitor(
                eventhub.PartitionIds,
                checkpoints.GetLastCheckpointAsync,
                getEventHubPartitionAsync,
                pauseBetweenParitions,
                pauseAfterAllPartitions);
        }
Exemplo n.º 2
0
        public static async Task <EventHubMonitor> CreateAsync(
            Configuration configuration,
            TimeSpan pauseBetweenParitions,
            TimeSpan pauseAfterAllPartitions
            )
        {
            var nsm = CreateNamespaceManager(configuration, pauseBetweenParitions);

            // We create a partially applied function so that
            // we'll only need to pass the partition id later.
            Func <string, Task <PartitionDescription> > getEventHubPartitionAsync =
                partitionId => nsm.GetEventHubPartitionAsync(configuration.EventHubName, configuration.ConsumerGroupName, partitionId);

            var eventhub = await nsm.GetEventHubAsync(configuration.EventHubName).ConfigureAwait(false);

            // Construct a reference to the blob container.
            var storageAccount      = CloudStorageAccount.Parse(configuration.CheckpointStorageAccount);
            var blobClient          = storageAccount.CreateCloudBlobClient();
            var checkpointContainer = blobClient.GetContainerReference(configuration.EventHubName);

            // The `PartitionCheckpointManager` manages the complexity
            // of getting the latest checkpoint for each parition.
            var checkpoints = new PartitionCheckpointManager(
                configuration.ConsumerGroupName,
                eventhub.PartitionIds,
                checkpointContainer
                );

            return(new EventHubMonitor(
                       eventhub.PartitionIds,
                       checkpoints.GetLastCheckpointAsync,
                       getEventHubPartitionAsync,
                       pauseBetweenParitions,
                       pauseAfterAllPartitions));
        }
        public async Task should_retrieve_the_checkpoint()
        {
            var wasRetrieved = false;

            Func<CloudBlockBlob, Task<PartitionCheckpoint>> getCheckpoint = _ =>
            {
                wasRetrieved = true;
                return Task.FromResult(new PartitionCheckpoint());
            };

            var manager = new PartitionCheckpointManager(
                ConsumerGroupName,
                new[] { PartitionId },
                GetCloudBlobContainerStub(),
                _ => Task.FromResult(new DateTimeOffset()),
                getCheckpoint);

            await manager.GetLastCheckpointAsync(PartitionId);

            Assert.True(wasRetrieved);
        }
        public async Task should_check_the_last_modified_date_of_the_blob()
        {
            var dateWasChecked = false;

            Func<CloudBlockBlob, Task<DateTimeOffset>> getLastModified = _ =>
            {
                dateWasChecked = true;
                return Task.FromResult(new DateTimeOffset());
            };

            var manager = new PartitionCheckpointManager(
                ConsumerGroupName,
                new[] { PartitionId },
                GetCloudBlobContainerStub(),
                getLastModified,
                _ => Task.FromResult(new PartitionCheckpoint()));

            await manager.GetLastCheckpointAsync(PartitionId);

            Assert.True(dateWasChecked);
        }
            public when_looking_up_the_blob_reference_for_a_partition()
            {
                var manager = new PartitionCheckpointManager(
                    ConsumerGroupName,
                    new[] { PartitionId },
                    GetCloudBlobContainerStub(),
                    b =>
                    {
                        _blobForLastModified = b;
                        return Task.FromResult(new DateTimeOffset());
                    },
                    b =>
                    {
                        _blobForRetrieval = b;
                        return Task.FromResult(new PartitionCheckpoint());
                    });

                manager.GetLastCheckpointAsync(PartitionId).Wait();
            }
            public when_accessing_multiple_partitions()
            {
                // create a set of differing  "last modified dates"
                var lastModifiedDates = _partitionIds
                    .Select((id, index) => new { id, index })
                    .ToDictionary(
                        x => GetBlobName(x.id),
                        x => new DateTimeOffset().AddDays(x.index));

                // keep track of how many times we attempt to retrieve
                // the checkpoint for each partition
                _retrievalCounts = _partitionIds
                    .ToDictionary(
                        GetBlobName,
                        id => 0);

                _manager = new PartitionCheckpointManager(
                    ConsumerGroupName,
                    _partitionIds,
                    GetCloudBlobContainerStub(),
                    blob => Task.FromResult(lastModifiedDates[blob.Name]),
                    blob =>
                    {
                        _retrievalCounts[blob.Name]++;
                        return Task.FromResult(new PartitionCheckpoint());
                    });
            }
            public async Task should_throw()
            {
                var manager = new PartitionCheckpointManager(
                    ConsumerGroupName,
                    new string[] { },
                    GetCloudBlobContainerStub(),
                    _ => Task.FromResult(new DateTimeOffset()),
                    _ => Task.FromResult(new PartitionCheckpoint()));

                await Assert.ThrowsAnyAsync<ArgumentException>(async () =>
                 {
                     await manager.GetLastCheckpointAsync("unknown");
                 });
            }
            public async Task should_return_a_cached_checkpoint()
            {
                var lastModified = new DateTimeOffset();
                Func<CloudBlockBlob, Task<DateTimeOffset>> getLastModified = _ => Task.FromResult(lastModified);
                Func<CloudBlockBlob, Task<PartitionCheckpoint>> getCheckpoint = _ => Task.FromResult(new PartitionCheckpoint());

                var manager = new PartitionCheckpointManager(
                    ConsumerGroupName,
                    new[] { PartitionId },
                    GetCloudBlobContainerStub(),
                    getLastModified,
                    getCheckpoint);

                var checkpoint1 = await manager.GetLastCheckpointAsync(PartitionId);
                var checkpoint2 = await manager.GetLastCheckpointAsync(PartitionId);

                Assert.Equal(checkpoint1, checkpoint2);
            }
            public async Task should_not_retrieve_the_checkpoint()
            {
                var timesRetrieved = 0;
                var lastModified = new DateTimeOffset();
                Func<CloudBlockBlob, Task<DateTimeOffset>> getLastModified = _ => Task.FromResult(lastModified);
                Func<CloudBlockBlob, Task<PartitionCheckpoint>> getCheckpoint = _ =>
                {
                    timesRetrieved++;
                    return Task.FromResult(new PartitionCheckpoint());
                };

                var manager = new PartitionCheckpointManager(
                    ConsumerGroupName,
                    new[] { PartitionId },
                    GetCloudBlobContainerStub(),
                    getLastModified,
                    getCheckpoint);

                await manager.GetLastCheckpointAsync(PartitionId);
                // since the last modified stamp hasn't changed, 
                // we should not attempt to retrieve the blob
                await manager.GetLastCheckpointAsync(PartitionId);

                Assert.Equal(1, timesRetrieved);
            }