public static async Task<EventHubMonitor> CreateAsync(
Configuration configuration,
TimeSpan pauseBetweenParitions,
TimeSpan pauseAfterAllPartitions
)
{
var nsm = CreateNamespaceManager(configuration, pauseBetweenParitions);
// We create a partially applied function so that
// we'll only need to pass the partition id later.
Func<string, Task<PartitionDescription>> getEventHubPartitionAsync =
partitionId => nsm.GetEventHubPartitionAsync(configuration.EventHubName, configuration.ConsumerGroupName, partitionId);
var eventhub = await nsm.GetEventHubAsync(configuration.EventHubName).ConfigureAwait(false);
// Construct a reference to the blob container.
var storageAccount = CloudStorageAccount.Parse(configuration.CheckpointStorageAccount);
var blobClient = storageAccount.CreateCloudBlobClient();
var checkpointContainer = blobClient.GetContainerReference(configuration.EventHubName);
// The `PartitionCheckpointManager` manages the complexity
// of getting the latest checkpoint for each parition.
var checkpoints = new PartitionCheckpointManager(
configuration.ConsumerGroupName,
eventhub.PartitionIds,
checkpointContainer
);
return new EventHubMonitor(
eventhub.PartitionIds,
checkpoints.GetLastCheckpointAsync,
getEventHubPartitionAsync,
pauseBetweenParitions,
pauseAfterAllPartitions);
}
public static async Task <EventHubMonitor> CreateAsync(
Configuration configuration,
TimeSpan pauseBetweenParitions,
TimeSpan pauseAfterAllPartitions
)
{
var nsm = CreateNamespaceManager(configuration, pauseBetweenParitions);
// We create a partially applied function so that
// we'll only need to pass the partition id later.
Func <string, Task <PartitionDescription> > getEventHubPartitionAsync =
partitionId => nsm.GetEventHubPartitionAsync(configuration.EventHubName, configuration.ConsumerGroupName, partitionId);
var eventhub = await nsm.GetEventHubAsync(configuration.EventHubName).ConfigureAwait(false);
// Construct a reference to the blob container.
var storageAccount = CloudStorageAccount.Parse(configuration.CheckpointStorageAccount);
var blobClient = storageAccount.CreateCloudBlobClient();
var checkpointContainer = blobClient.GetContainerReference(configuration.EventHubName);
// The `PartitionCheckpointManager` manages the complexity
// of getting the latest checkpoint for each parition.
var checkpoints = new PartitionCheckpointManager(
configuration.ConsumerGroupName,
eventhub.PartitionIds,
checkpointContainer
);
return(new EventHubMonitor(
eventhub.PartitionIds,
checkpoints.GetLastCheckpointAsync,
getEventHubPartitionAsync,
pauseBetweenParitions,
pauseAfterAllPartitions));
}
public async Task should_retrieve_the_checkpoint()
{
var wasRetrieved = false;
Func<CloudBlockBlob, Task<PartitionCheckpoint>> getCheckpoint = _ =>
{
wasRetrieved = true;
return Task.FromResult(new PartitionCheckpoint());
};
var manager = new PartitionCheckpointManager(
ConsumerGroupName,
new[] { PartitionId },
GetCloudBlobContainerStub(),
_ => Task.FromResult(new DateTimeOffset()),
getCheckpoint);
await manager.GetLastCheckpointAsync(PartitionId);
Assert.True(wasRetrieved);
}
public async Task should_check_the_last_modified_date_of_the_blob()
{
var dateWasChecked = false;
Func<CloudBlockBlob, Task<DateTimeOffset>> getLastModified = _ =>
{
dateWasChecked = true;
return Task.FromResult(new DateTimeOffset());
};
var manager = new PartitionCheckpointManager(
ConsumerGroupName,
new[] { PartitionId },
GetCloudBlobContainerStub(),
getLastModified,
_ => Task.FromResult(new PartitionCheckpoint()));
await manager.GetLastCheckpointAsync(PartitionId);
Assert.True(dateWasChecked);
}
public when_looking_up_the_blob_reference_for_a_partition()
{
var manager = new PartitionCheckpointManager(
ConsumerGroupName,
new[] { PartitionId },
GetCloudBlobContainerStub(),
b =>
{
_blobForLastModified = b;
return Task.FromResult(new DateTimeOffset());
},
b =>
{
_blobForRetrieval = b;
return Task.FromResult(new PartitionCheckpoint());
});
manager.GetLastCheckpointAsync(PartitionId).Wait();
}
public when_accessing_multiple_partitions()
{
// create a set of differing "last modified dates"
var lastModifiedDates = _partitionIds
.Select((id, index) => new { id, index })
.ToDictionary(
x => GetBlobName(x.id),
x => new DateTimeOffset().AddDays(x.index));
// keep track of how many times we attempt to retrieve
// the checkpoint for each partition
_retrievalCounts = _partitionIds
.ToDictionary(
GetBlobName,
id => 0);
_manager = new PartitionCheckpointManager(
ConsumerGroupName,
_partitionIds,
GetCloudBlobContainerStub(),
blob => Task.FromResult(lastModifiedDates[blob.Name]),
blob =>
{
_retrievalCounts[blob.Name]++;
return Task.FromResult(new PartitionCheckpoint());
});
}
public async Task should_throw()
{
var manager = new PartitionCheckpointManager(
ConsumerGroupName,
new string[] { },
GetCloudBlobContainerStub(),
_ => Task.FromResult(new DateTimeOffset()),
_ => Task.FromResult(new PartitionCheckpoint()));
await Assert.ThrowsAnyAsync<ArgumentException>(async () =>
{
await manager.GetLastCheckpointAsync("unknown");
});
}
public async Task should_return_a_cached_checkpoint()
{
var lastModified = new DateTimeOffset();
Func<CloudBlockBlob, Task<DateTimeOffset>> getLastModified = _ => Task.FromResult(lastModified);
Func<CloudBlockBlob, Task<PartitionCheckpoint>> getCheckpoint = _ => Task.FromResult(new PartitionCheckpoint());
var manager = new PartitionCheckpointManager(
ConsumerGroupName,
new[] { PartitionId },
GetCloudBlobContainerStub(),
getLastModified,
getCheckpoint);
var checkpoint1 = await manager.GetLastCheckpointAsync(PartitionId);
var checkpoint2 = await manager.GetLastCheckpointAsync(PartitionId);
Assert.Equal(checkpoint1, checkpoint2);
}
public async Task should_not_retrieve_the_checkpoint()
{
var timesRetrieved = 0;
var lastModified = new DateTimeOffset();
Func<CloudBlockBlob, Task<DateTimeOffset>> getLastModified = _ => Task.FromResult(lastModified);
Func<CloudBlockBlob, Task<PartitionCheckpoint>> getCheckpoint = _ =>
{
timesRetrieved++;
return Task.FromResult(new PartitionCheckpoint());
};
var manager = new PartitionCheckpointManager(
ConsumerGroupName,
new[] { PartitionId },
GetCloudBlobContainerStub(),
getLastModified,
getCheckpoint);
await manager.GetLastCheckpointAsync(PartitionId);
// since the last modified stamp hasn't changed,
// we should not attempt to retrieve the blob
await manager.GetLastCheckpointAsync(PartitionId);
Assert.Equal(1, timesRetrieved);
}