public async Task StartPump(CancellationToken cancellationToken = default)
{
if (!CloudStorageAccount.TryParse(
eventHubsDetails.CaptureStorageAccountConnectionString,
out CloudStorageAccount storageAccount))
{
return;
}
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
var captureContainer = cloudBlobClient.GetContainerReference(eventHubsDetails.CaptureContainerName);
var captureContainerUri = captureContainer.Uri.ToString() + "/";
string uriPrefix = captureContainerUri + startString;
string listBlobPrefix = FormatStorageString(partitionContext);
OperationContext operationContext = null;
BlobContinuationToken blobContinuationToken = null;
do
{
var blobResultSegment = await captureContainer.ListBlobsSegmentedAsync(
prefix : listBlobPrefix, useFlatBlobListing : true,
blobListingDetails : BlobListingDetails.None,
maxResults : null, currentToken : blobContinuationToken,
options : new BlobRequestOptions(), operationContext : operationContext,
cancellationToken : cancellationToken);
blobContinuationToken = blobResultSegment.ContinuationToken;
foreach (var listBlobItem in blobResultSegment.Results)
{
if (useStartFile && string.Compare(uriPrefix, listBlobItem.Uri.AbsoluteUri) > 0)
{
continue;
}
var blob = await cloudBlobClient.GetBlobReferenceFromServerAsync(
blobUri : listBlobItem.Uri, cancellationToken : cancellationToken);
using Stream stream = await blob.OpenReadAsync(
accessCondition : null,
options : new BlobRequestOptions(),
operationContext : operationContext,
cancellationToken : cancellationToken);
IEnumerable <EventData> eventHubMessages = stream.ReadAvroStreamToEventHubData(
partitionKey: this.partitionContext.PartitionId);
await eventProcessor.ProcessEventsAsync(
context : partitionContext,
messages : eventHubMessages);
}
} while (blobContinuationToken != null);
}
public async Task StartPump()
{
//start the pump on storage account or
CloudStorageAccount storageAccount;
if (CloudStorageAccount.TryParse(eventHubsDetails.StorageAccountConnectionString, out storageAccount))
{
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
var cloudBlobContainer = cloudBlobClient.GetContainerReference(eventHubsDetails.CaptureContainer);
string containerUri = cloudBlobContainer.Uri.ToString() + "/";
BlobContinuationToken blobContinuationToken = null;
do
{
BlobRequestOptions bro = new BlobRequestOptions();
string format = FormatStorageString(context.PartitionId);
var results = cloudBlobContainer.ListBlobsSegmentedAsync(format,
true, BlobListingDetails.None, null, blobContinuationToken, bro, null).Result;
// Get the value of the continuation token returned by the listing call.
blobContinuationToken = results.ContinuationToken;
foreach (IListBlobItem item in results.Results)
{
if (useStartFile)
{
if (string.Compare(containerUri + startString, item.Uri.ToString()) > 0)
{
//do something?
continue;
}
}
Console.WriteLine(item.Uri);
ICloudBlob blob = await cloudBlobClient.GetBlobReferenceFromServerAsync(item.Uri);
using (Stream stream = blob.OpenReadAsync(null, bro, null).Result)
{
await processor.ProcessEventsAsync(context, ReadFile(stream));
}
}
} while (blobContinuationToken != null); // Loop while the continuation token is not null.
}
/*
* //file local store
* foreach (var file in Directory.EnumerateFiles(filePath, searchPattern))
* {
* //load each file
* using (var inStream = File.OpenRead(file))
* {
* processor.ProcessEventsAsync(context, ReadFile(inStream));
* }
* }
*/
//return Task.CompletedTask;
}
