/// <summary> /// Trims the cache of any expired items in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> representing an asynchronous operation. /// </returns> public override async Task TrimCacheAsync() { Uri uri = new Uri(this.CachedPath); string path = uri.GetLeftPart(UriPartial.Path).Substring(cloudCachedBlobContainer.Uri.ToString().Length + 1); string parent = path.Substring(0, path.LastIndexOf('/') - 9); BlobContinuationToken continuationToken = null; List <IListBlobItem> results = new List <IListBlobItem>(); // Loop through the all the files in a non blocking fashion. do { BlobResultSegment response = await cloudCachedBlobContainer.ListBlobsSegmentedAsync(parent, true, BlobListingDetails.Metadata, 5000, continuationToken, null, null); continuationToken = response.ContinuationToken; results.AddRange(response.Results); }while (continuationToken != null); // Now leap through and delete. foreach ( CloudBlockBlob blob in results.Where((blobItem, type) => blobItem is CloudBlockBlob) .Cast <CloudBlockBlob>() .OrderBy(b => b.Properties.LastModified?.UtcDateTime ?? new DateTime())) { if (blob.Properties.LastModified.HasValue && !this.IsExpired(blob.Properties.LastModified.Value.UtcDateTime)) { break; } // Remove from the cache and delete each CachedImage. CacheIndexer.Remove(blob.Name); await blob.DeleteAsync(); } }
/// <summary> /// Gets a value indicating whether the image is new or updated in an asynchronous manner. /// </summary> /// <returns> /// The <see cref="Task"/>. /// </returns> public override async Task <bool> IsNewOrUpdatedAsync() { cachedFilename = prefix + await CreateCachedFileNameAsync(); var media = UrlResolver.Current.Route(new UrlBuilder(FullPath)) as MediaData; container = media?.BinaryDataContainer?.Segments[1]; if (container == null) { // We're working with a static file here container = $"_{prefix}static"; } CachedPath = Path.Combine(absoluteCachePath, container, cachedFilename); virtualCachedFilePath = string.Join("/", virtualCachePath, container, cachedFilename); bool isUpdated = false; CachedImage cachedImage = CacheIndexer.Get(cachedFilename); if (cachedImage == null) { if (File.Exists(CachedPath)) { cachedImage = new CachedImage { Key = Path.GetFileNameWithoutExtension(cachedFilename), Path = CachedPath, CreationTimeUtc = File.GetCreationTimeUtc(CachedPath) }; CacheIndexer.Add(cachedImage); } } if (cachedImage == null) { // Nothing in the cache so we should return true. isUpdated = true; } else { // Check to see if the cached image is set to expire // or a new file with the same name has replaced our current image if (IsExpired(cachedImage.CreationTimeUtc) || IsUpdated(cachedImage.CreationTimeUtc)) { CacheIndexer.Remove(CachedPath); isUpdated = true; } else { // Set cachedImageCreationTimeUtc so we can sender Last-Modified or ETag header when using Response.TransmitFile() cachedImageCreationTimeUtc = cachedImage.CreationTimeUtc; } } return(isUpdated); }
/// <summary> /// Trims the cache of any expired items in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> representing an asynchronous operation. /// </returns> public override async Task TrimCacheAsync() { string path = this.GetFolderStructureForAmazon(this.CachedPath); string directory = path.Substring(0, path.LastIndexOf('/')); string parent = directory.Substring(0, directory.LastIndexOf('/') + 1); ListObjectsRequest request = new ListObjectsRequest { BucketName = this.awsBucketName, Prefix = parent, Delimiter = @"/" }; try { List <S3Object> results = new List <S3Object>(); do { ListObjectsResponse response = await this.amazonS3ClientCache.ListObjectsAsync(request); results.AddRange(response.S3Objects); // If response is truncated, set the marker to get the next // set of keys. if (response.IsTruncated) { request.Marker = response.NextMarker; } else { request = null; } }while (request != null); foreach (S3Object file in results.OrderBy(x => x.LastModified.ToUniversalTime())) { if (!this.IsExpired(file.LastModified.ToUniversalTime())) { break; } CacheIndexer.Remove(file.Key); await this.amazonS3ClientCache.DeleteObjectAsync(new DeleteObjectRequest { BucketName = this.awsBucketName, Key = file.Key }); } } catch (Exception ex) { // TODO: Remove try/catch. throw ex; } }
/// <summary> /// Trims the cache of any expired items in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> representing an asynchronous operation. /// </returns> public override Task TrimCacheAsync() { if (!this.TrimCache) { return(Task.FromResult(0)); } this.ScheduleCacheTrimmer(async token => { // Jump up to the parent branch to clean through the cache. string parent = string.Empty; if (this.FolderDepth > 0) { Uri uri = new Uri(this.CachedPath); string path = uri.GetLeftPart(UriPartial.Path).Substring(cloudCachedBlobContainer.Uri.ToString().Length + 1); parent = path.Substring(0, 2); } BlobContinuationToken continuationToken = null; List <IListBlobItem> results = new List <IListBlobItem>(); // Loop through the all the files in a non blocking fashion. do { BlobResultSegment response = await cloudCachedBlobContainer.ListBlobsSegmentedAsync(parent, true, BlobListingDetails.Metadata, 5000, continuationToken, null, null, token); continuationToken = response.ContinuationToken; results.AddRange(response.Results); }while (token.IsCancellationRequested == false && continuationToken != null); // Now leap through and delete. foreach ( CloudBlockBlob blob in results.Where((blobItem, type) => blobItem is CloudBlockBlob) .Cast <CloudBlockBlob>() .OrderBy(b => b.Properties.LastModified?.UtcDateTime ?? new DateTime())) { if (token.IsCancellationRequested || (blob.Properties.LastModified.HasValue && !this.IsExpired(blob.Properties.LastModified.Value.UtcDateTime))) { break; } // Remove from the cache and delete each CachedImage. CacheIndexer.Remove(blob.Name); await blob.DeleteAsync(token); } }); return(Task.FromResult(0)); }
/// <summary> /// Trims the cache of any expired items in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> representing an asynchronous operation. /// </returns> public override Task TrimCacheAsync() { if (!TrimCache) { return(Task.FromResult(0)); } ScheduleCacheTrimmer(async token => { BlobContinuationToken continuationToken = null; List <IListBlobItem> results = new List <IListBlobItem>(); // Loop through the all the files in a non blocking fashion. do { BlobResultSegment response = await rootContainer.ListBlobsSegmentedAsync(string.Empty, true, BlobListingDetails.Metadata, 5000, continuationToken, null, null, token); continuationToken = response.ContinuationToken; results.AddRange(response.Results); }while (token.IsCancellationRequested == false && continuationToken != null); // Now leap through and delete. foreach ( CloudBlockBlob blob in results.Where((blobItem, type) => blobItem is CloudBlockBlob) .Cast <CloudBlockBlob>() .OrderBy(b => b.Properties.LastModified?.UtcDateTime ?? new DateTime())) { if (token.IsCancellationRequested) { break; } if (blob.Properties.LastModified.HasValue && !IsExpired(blob.Properties.LastModified.Value.UtcDateTime)) { continue; } if (!blob.Name.Contains(prefix)) { continue; } // Remove from the cache and delete each CachedImage. CacheIndexer.Remove(blob.Name); await blob.DeleteAsync(token); } }); return(Task.FromResult(0)); }
/// <summary> /// Gets a value indicating whether the image is new or updated in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> returning the value. /// </returns> public override async Task <bool> IsNewOrUpdatedAsync() { string cachedFileName = await this.CreateCachedFileNameAsync(); // Collision rate of about 1 in 10000 for the folder structure. // That gives us massive scope to store millions of files. string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6)); this.CachedPath = Path.Combine(cloudCachedBlobContainer.Uri.ToString(), pathFromKey, cachedFileName).Replace(@"\", "/"); // Do we insert the cache container? This seems to break some setups. bool useCachedContainerInUrl = this.Settings.ContainsKey("UseCachedContainerInUrl") && this.Settings["UseCachedContainerInUrl"].ToLower() != "false"; if (useCachedContainerInUrl) { this.cachedRewritePath = Path.Combine(this.cachedCdnRoot, cloudCachedBlobContainer.Name, pathFromKey, cachedFileName) .Replace(@"\", "/"); } else { this.cachedRewritePath = Path.Combine(this.cachedCdnRoot, pathFromKey, cachedFileName) .Replace(@"\", "/"); } bool isUpdated = false; CachedImage cachedImage = CacheIndexer.Get(this.CachedPath); if (new Uri(this.CachedPath).IsFile) { FileInfo fileInfo = new FileInfo(this.CachedPath); if (fileInfo.Exists) { // Pull the latest info. fileInfo.Refresh(); cachedImage = new CachedImage { Key = Path.GetFileNameWithoutExtension(this.CachedPath), Path = this.CachedPath, CreationTimeUtc = fileInfo.CreationTimeUtc }; CacheIndexer.Add(cachedImage); } } if (cachedImage == null) { string blobPath = this.CachedPath.Substring(cloudCachedBlobContainer.Uri.ToString().Length + 1); CloudBlockBlob blockBlob = cloudCachedBlobContainer.GetBlockBlobReference(blobPath); if (await blockBlob.ExistsAsync()) { // Pull the latest info. await blockBlob.FetchAttributesAsync(); if (blockBlob.Properties.LastModified.HasValue) { cachedImage = new CachedImage { Key = Path.GetFileNameWithoutExtension(this.CachedPath), Path = this.CachedPath, CreationTimeUtc = blockBlob.Properties.LastModified.Value.UtcDateTime }; CacheIndexer.Add(cachedImage); } } } if (cachedImage == null) { // Nothing in the cache so we should return true. isUpdated = true; } else { // Check to see if the cached image is set to expire. if (this.IsExpired(cachedImage.CreationTimeUtc)) { CacheIndexer.Remove(this.CachedPath); isUpdated = true; } } return(isUpdated); }
/// <summary> /// Gets a value indicating whether the image is new or updated in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> returning the value. /// </returns> public override async Task <bool> IsNewOrUpdatedAsync() { // TODO: Before this check is performed it should be throttled. For example, only perform this check // if the last time it was checked is greater than 5 seconds. This would be much better for perf // if there is a high throughput of image requests. string cachedFileName = await this.CreateCachedFileNameAsync(); // Collision rate of about 1 in 10000 for the folder structure. // That gives us massive scope to store millions of files. string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6)); this.CachedPath = Path.Combine(this.cachedCdnRoot, this.imageProcessorCachePrefix, pathFromKey, cachedFileName) .Replace(@"\", "/"); // TODO: What is the S3 version of the following lines? The Above doesn't match what I expect //this.CachedPath = Path.Combine(this.cloudCachedBlobContainer.Uri.ToString(), pathFromKey, cachedFileName).Replace(@"\", "/"); //this.cachedRewritePath = Path.Combine(this.cachedCdnRoot, this.cloudCachedBlobContainer.Name, pathFromKey, cachedFileName).Replace(@"\", "/"); bool isUpdated = false; CachedImage cachedImage = CacheIndexer.Get(this.CachedPath); if (new Uri(this.CachedPath).IsFile) { FileInfo fileInfo = new FileInfo(this.CachedPath); if (fileInfo.Exists) { // Pull the latest info. fileInfo.Refresh(); cachedImage = new CachedImage { Key = Path.GetFileNameWithoutExtension(this.CachedPath), Path = this.CachedPath, CreationTimeUtc = fileInfo.CreationTimeUtc }; CacheIndexer.Add(cachedImage); } } if (cachedImage == null) { try { string path = this.GetFolderStructureForAmazon(this.CachedPath); string filename = Path.GetFileName(this.CachedPath); string key = this.GetKey(path, filename); GetObjectMetadataRequest objectMetaDataRequest = new GetObjectMetadataRequest { BucketName = this.awsBucketName, Key = key, }; GetObjectMetadataResponse response = await this.amazonS3ClientCache.GetObjectMetadataAsync(objectMetaDataRequest); if (response != null) { cachedImage = new CachedImage { Key = key, Path = this.CachedPath, CreationTimeUtc = response.LastModified.ToUniversalTime() }; CacheIndexer.Add(cachedImage); } } catch (AmazonS3Exception) { // Nothing in S3 so we should return true. isUpdated = true; } } if (cachedImage == null) { // Nothing in the cache so we should return true. isUpdated = true; } else { // Check to see if the cached image is set to expire. if (this.IsExpired(cachedImage.CreationTimeUtc)) { CacheIndexer.Remove(this.CachedPath); isUpdated = true; } } return(isUpdated); }
/// <summary> /// Gets a value indicating whether the image is new or updated in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> returning the value. /// </returns> public override async Task <bool> IsNewOrUpdatedAsync() { // TODO: Before this check is performed it should be throttled. For example, only perform this check // if the last time it was checked is greater than 5 seconds. This would be much better for perf // if there is a high throughput of image requests. string cachedFileName = await this.CreateCachedFileNameAsync().ConfigureAwait(false); this.CachedPath = CachedImageHelper.GetCachedPath(cloudCachedBlobContainer.Uri.ToString(), cachedFileName, true, this.FolderDepth); this.cachedRewritePath = CachedImageHelper.GetCachedPath(useCachedContainerInUrl ? Path.Combine(cachedCdnRoot, cloudCachedBlobContainer.Name) : cachedCdnRoot, cachedFileName, true, this.FolderDepth); bool isUpdated = false; CachedImage cachedImage = CacheIndexer.Get(this.CachedPath); if (new Uri(this.CachedPath).IsFile&& File.Exists(this.CachedPath)) { cachedImage = new CachedImage { Key = Path.GetFileNameWithoutExtension(this.CachedPath), Path = this.CachedPath, CreationTimeUtc = File.GetCreationTimeUtc(this.CachedPath) }; CacheIndexer.Add(cachedImage, this.ImageCacheMaxMinutes); } if (cachedImage is null) { string blobPath = this.CachedPath.Substring(cloudCachedBlobContainer.Uri.ToString().Length + 1); CloudBlockBlob blockBlob = cloudCachedBlobContainer.GetBlockBlobReference(blobPath); if (await blockBlob.ExistsAsync().ConfigureAwait(false)) { // Pull the latest info. await blockBlob.FetchAttributesAsync().ConfigureAwait(false); if (blockBlob.Properties.LastModified.HasValue) { cachedImage = new CachedImage { Key = Path.GetFileNameWithoutExtension(this.CachedPath), Path = this.CachedPath, CreationTimeUtc = blockBlob.Properties.LastModified.Value.UtcDateTime }; CacheIndexer.Add(cachedImage, this.ImageCacheMaxMinutes); } } } if (cachedImage is null) { // Nothing in the cache so we should return true. isUpdated = true; } else { // Check to see if the cached image is set to expire // or a new file with the same name has replaced our current image if (this.IsExpired(cachedImage.CreationTimeUtc) || await this.IsUpdatedAsync(cachedImage.CreationTimeUtc).ConfigureAwait(false)) { CacheIndexer.Remove(this.CachedPath); isUpdated = true; } } return(isUpdated); }
/// <summary> /// Gets a value indicating whether the image is new or updated in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> returning the value. /// </returns> public override async Task <bool> IsNewOrUpdatedAsync() { string cachedFilename = prefix + await CreateCachedFileNameAsync(); var media = UrlResolver.Current.Route(new UrlBuilder(FullPath)) as MediaData; string containerName = media?.BinaryDataContainer?.Segments[1]; if (containerName == null) { // We're working with a static file here containerName = $"_{prefix}static"; } blobPath = $"{containerName}/{cachedFilename}"; CachedPath = $"{rootContainer.Uri.ToString()}/{containerName}/{cachedFilename}"; bool isUpdated = false; CachedImage cachedImage = CacheIndexer.Get(CachedPath); if (cachedImage == null) { CloudBlockBlob blockBlob = rootContainer.GetBlockBlobReference(blobPath); //string t = GetSaSForBlob(blockBlob, SharedAccessBlobPermissions.Read); if (await blockBlob.ExistsAsync()) { // Pull the latest info. await blockBlob.FetchAttributesAsync(); if (blockBlob.Properties.LastModified.HasValue) { cachedImage = new CachedImage { Key = Path.GetFileNameWithoutExtension(CachedPath), Path = CachedPath, CreationTimeUtc = blockBlob.Properties.LastModified.Value.UtcDateTime }; CacheIndexer.Add(cachedImage, ImageCacheMaxMinutes); } } } if (cachedImage == null) { // Nothing in the cache so we should return true. isUpdated = true; } else { // Check to see if the cached image is set to expire // or a new file with the same name has replaced our current image if (IsExpired(cachedImage.CreationTimeUtc) || await IsUpdatedAsync(cachedImage.CreationTimeUtc)) { CacheIndexer.Remove(CachedPath); isUpdated = true; } } return(isUpdated); }
/// <summary> /// Trims the cache of any expired items in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> representing an asynchronous operation. /// </returns> public override Task TrimCacheAsync() { if (!TrimCache) { return(Task.FromResult(0)); } ScheduleCacheTrimmer(token => { string rootDirectory = Path.GetDirectoryName(CachedPath); if (rootDirectory != null) { // Jump up to the parent branch to clean through the cache. // UNC folders can throw exceptions if the file doesn't exist. IEnumerable <string> directories = Directory.EnumerateDirectories(absoluteCachePath).Reverse(); foreach (string directory in directories) { if (!Directory.Exists(directory)) { continue; } if (token.IsCancellationRequested) { break; } IEnumerable <FileInfo> files = Directory.EnumerateFiles(directory) .Select(f => new FileInfo(f)) .Where(f => f.Name.StartsWith(prefix)) .OrderBy(f => f.CreationTimeUtc); foreach (FileInfo fileInfo in files) { if (token.IsCancellationRequested) { break; } try { if (!IsExpired(fileInfo.CreationTimeUtc)) { continue; } // Remove from the cache and delete each CachedImage. CacheIndexer.Remove(fileInfo.Name); fileInfo.Delete(); } catch (Exception ex) { // Log it but skip to the next file. logger.Error($"Unable to clean cached file: {fileInfo.FullName}", ex); } } // If the directory is empty of files delete it to remove the FCN. RecursivelyDeleteEmptyDirectories(directory, absoluteCachePath, token); } } return(Task.FromResult(0)); }); return(Task.FromResult(0)); }
/// <summary> /// Gets a value indicating whether the image is new or updated in an asynchronous manner. /// </summary> /// <returns> /// The asynchronous <see cref="Task"/> returning the value. /// </returns> public override async Task <bool> IsNewOrUpdatedAsync() { string cachedFileName = await this.CreateCachedFileNameAsync(); // Collision rate of about 1 in 10000 for the folder structure. // That gives us massive scope to store millions of files. string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6)); this.CachedPath = Path.Combine(this.cachedCdnRoot, this.imageProcessorCachePrefix, pathFromKey, cachedFileName) .Replace(@"\", "/"); bool isUpdated = false; CachedImage cachedImage = CacheIndexer.Get(this.CachedPath); if (new Uri(this.CachedPath).IsFile) { FileInfo fileInfo = new FileInfo(this.CachedPath); if (fileInfo.Exists) { // Pull the latest info. fileInfo.Refresh(); cachedImage = new CachedImage { Key = Path.GetFileNameWithoutExtension(this.CachedPath), Path = this.CachedPath, CreationTimeUtc = fileInfo.CreationTimeUtc }; CacheIndexer.Add(cachedImage); } } if (cachedImage == null) { try { string path = this.GetFolderStructureForAmazon(this.CachedPath); string filename = Path.GetFileName(this.CachedPath); string key = this.GetKey(path, filename); GetObjectMetadataRequest objectMetaDataRequest = new GetObjectMetadataRequest { BucketName = this.awsBucketName, Key = key, }; GetObjectMetadataResponse response = await this.amazonS3ClientCache.GetObjectMetadataAsync(objectMetaDataRequest); if (response != null) { cachedImage = new CachedImage { Key = key, Path = this.CachedPath, CreationTimeUtc = response.LastModified.ToUniversalTime() }; CacheIndexer.Add(cachedImage); } } catch (AmazonS3Exception) { // Nothing in S3 so we should return true. isUpdated = true; } } if (cachedImage == null) { // Nothing in the cache so we should return true. isUpdated = true; } else { // Check to see if the cached image is set to expire. if (this.IsExpired(cachedImage.CreationTimeUtc)) { CacheIndexer.Remove(this.CachedPath); isUpdated = true; } } return(isUpdated); }