public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken,
int hoursWindow = DefaultScanHoursWindow)
{
List <IStorageBlob> blobs = new List <IStorageBlob>();
var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew
foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow))
{
bool isAdded = _scannedBlobNames.Add(blob.Name);
if (!isAdded)
{
continue;
}
// Need to clear out cache.
if (_scannedBlobNames.Count > 100 * 1000)
{
_scannedBlobNames.Clear();
}
var parsedBlobPaths = from entry in await _parser.ParseLogAsync(blob, cancellationToken)
where entry.IsBlobWrite
select entry.ToBlobPath();
foreach (BlobPath path in parsedBlobPaths.Where(p => p != null))
{
IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName);
blobs.Add(container.GetBlockBlobReference(path.BlobName));
}
}
return(blobs);
}
public async Task <IEnumerable <ICloudBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken,
int hoursWindow = DefaultScanHoursWindow)
{
List <ICloudBlob> blobs = new List <ICloudBlob>();
var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew
foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow))
{
bool isAdded = _scannedBlobNames.Add(blob.Name);
if (!isAdded)
{
continue;
}
// Need to clear out cache.
if (_scannedBlobNames.Count > 100 * 1000)
{
_scannedBlobNames.Clear();
}
IEnumerable <StorageAnalyticsLogEntry> entries = await _parser.ParseLogAsync(blob, cancellationToken);
IEnumerable <BlobPath> filteredBlobs = GetPathsForValidBlobWrites(entries);
foreach (BlobPath path in filteredBlobs)
{
var container = _blobClient.GetContainerReference(path.ContainerName);
blobs.Add(container.GetBlockBlobReference(path.BlobName));
}
}
return(blobs);
}