public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken, int hoursWindow = DefaultScanHoursWindow) { List <IStorageBlob> blobs = new List <IStorageBlob>(); var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow)) { bool isAdded = _scannedBlobNames.Add(blob.Name); if (!isAdded) { continue; } // Need to clear out cache. if (_scannedBlobNames.Count > 100 * 1000) { _scannedBlobNames.Clear(); } var parsedBlobPaths = from entry in await _parser.ParseLogAsync(blob, cancellationToken) where entry.IsBlobWrite select entry.ToBlobPath(); foreach (BlobPath path in parsedBlobPaths.Where(p => p != null)) { IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName); blobs.Add(container.GetBlockBlobReference(path.BlobName)); } } return(blobs); }
public async Task <IEnumerable <ICloudBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken, int hoursWindow = DefaultScanHoursWindow) { List <ICloudBlob> blobs = new List <ICloudBlob>(); var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow)) { bool isAdded = _scannedBlobNames.Add(blob.Name); if (!isAdded) { continue; } // Need to clear out cache. if (_scannedBlobNames.Count > 100 * 1000) { _scannedBlobNames.Clear(); } IEnumerable <StorageAnalyticsLogEntry> entries = await _parser.ParseLogAsync(blob, cancellationToken); IEnumerable <BlobPath> filteredBlobs = GetPathsForValidBlobWrites(entries); foreach (BlobPath path in filteredBlobs) { var container = _blobClient.GetContainerReference(path.ContainerName); blobs.Add(container.GetBlockBlobReference(path.BlobName)); } } return(blobs); }