/// <summary>
/// Trims the cache of any expired items in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> representing an asynchronous operation.
/// </returns>
public override async Task TrimCacheAsync()
{
Uri uri = new Uri(this.CachedPath);
string path = uri.GetLeftPart(UriPartial.Path).Substring(cloudCachedBlobContainer.Uri.ToString().Length + 1);
string parent = path.Substring(0, path.LastIndexOf('/') - 9);
BlobContinuationToken continuationToken = null;
List <IListBlobItem> results = new List <IListBlobItem>();
// Loop through the all the files in a non blocking fashion.
do
{
BlobResultSegment response = await cloudCachedBlobContainer.ListBlobsSegmentedAsync(parent, true, BlobListingDetails.Metadata, 5000, continuationToken, null, null);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}while (continuationToken != null);
// Now leap through and delete.
foreach (
CloudBlockBlob blob in
results.Where((blobItem, type) => blobItem is CloudBlockBlob)
.Cast <CloudBlockBlob>()
.OrderBy(b => b.Properties.LastModified?.UtcDateTime ?? new DateTime()))
{
if (blob.Properties.LastModified.HasValue && !this.IsExpired(blob.Properties.LastModified.Value.UtcDateTime))
{
break;
}
// Remove from the cache and delete each CachedImage.
CacheIndexer.Remove(blob.Name);
await blob.DeleteAsync();
}
}
/// <summary>
/// Gets a value indicating whether the image is new or updated in an asynchronous manner.
/// </summary>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
public override async Task <bool> IsNewOrUpdatedAsync()
{
cachedFilename = prefix + await CreateCachedFileNameAsync();
var media = UrlResolver.Current.Route(new UrlBuilder(FullPath)) as MediaData;
container = media?.BinaryDataContainer?.Segments[1];
if (container == null)
{
// We're working with a static file here
container = $"_{prefix}static";
}
CachedPath = Path.Combine(absoluteCachePath, container, cachedFilename);
virtualCachedFilePath = string.Join("/", virtualCachePath, container, cachedFilename);
bool isUpdated = false;
CachedImage cachedImage = CacheIndexer.Get(cachedFilename);
if (cachedImage == null)
{
if (File.Exists(CachedPath))
{
cachedImage = new CachedImage
{
Key = Path.GetFileNameWithoutExtension(cachedFilename),
Path = CachedPath,
CreationTimeUtc = File.GetCreationTimeUtc(CachedPath)
};
CacheIndexer.Add(cachedImage);
}
}
if (cachedImage == null)
{
// Nothing in the cache so we should return true.
isUpdated = true;
}
else
{
// Check to see if the cached image is set to expire
// or a new file with the same name has replaced our current image
if (IsExpired(cachedImage.CreationTimeUtc) || IsUpdated(cachedImage.CreationTimeUtc))
{
CacheIndexer.Remove(CachedPath);
isUpdated = true;
}
else
{
// Set cachedImageCreationTimeUtc so we can sender Last-Modified or ETag header when using Response.TransmitFile()
cachedImageCreationTimeUtc = cachedImage.CreationTimeUtc;
}
}
return(isUpdated);
}
/// <summary>
/// Trims the cache of any expired items in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> representing an asynchronous operation.
/// </returns>
public override async Task TrimCacheAsync()
{
string path = this.GetFolderStructureForAmazon(this.CachedPath);
string directory = path.Substring(0, path.LastIndexOf('/'));
string parent = directory.Substring(0, directory.LastIndexOf('/') + 1);
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = this.awsBucketName,
Prefix = parent,
Delimiter = @"/"
};
try
{
List <S3Object> results = new List <S3Object>();
do
{
ListObjectsResponse response = await this.amazonS3ClientCache.ListObjectsAsync(request);
results.AddRange(response.S3Objects);
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
request = null;
}
}while (request != null);
foreach (S3Object file in results.OrderBy(x => x.LastModified.ToUniversalTime()))
{
if (!this.IsExpired(file.LastModified.ToUniversalTime()))
{
break;
}
CacheIndexer.Remove(file.Key);
await this.amazonS3ClientCache.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = this.awsBucketName,
Key = file.Key
});
}
}
catch (Exception ex)
{
// TODO: Remove try/catch.
throw ex;
}
}
/// <summary>
/// Trims the cache of any expired items in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> representing an asynchronous operation.
/// </returns>
public override Task TrimCacheAsync()
{
if (!this.TrimCache)
{
return(Task.FromResult(0));
}
this.ScheduleCacheTrimmer(async token =>
{
// Jump up to the parent branch to clean through the cache.
string parent = string.Empty;
if (this.FolderDepth > 0)
{
Uri uri = new Uri(this.CachedPath);
string path = uri.GetLeftPart(UriPartial.Path).Substring(cloudCachedBlobContainer.Uri.ToString().Length + 1);
parent = path.Substring(0, 2);
}
BlobContinuationToken continuationToken = null;
List <IListBlobItem> results = new List <IListBlobItem>();
// Loop through the all the files in a non blocking fashion.
do
{
BlobResultSegment response = await cloudCachedBlobContainer.ListBlobsSegmentedAsync(parent, true, BlobListingDetails.Metadata, 5000, continuationToken, null, null, token);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}while (token.IsCancellationRequested == false && continuationToken != null);
// Now leap through and delete.
foreach (
CloudBlockBlob blob in
results.Where((blobItem, type) => blobItem is CloudBlockBlob)
.Cast <CloudBlockBlob>()
.OrderBy(b => b.Properties.LastModified?.UtcDateTime ?? new DateTime()))
{
if (token.IsCancellationRequested || (blob.Properties.LastModified.HasValue && !this.IsExpired(blob.Properties.LastModified.Value.UtcDateTime)))
{
break;
}
// Remove from the cache and delete each CachedImage.
CacheIndexer.Remove(blob.Name);
await blob.DeleteAsync(token);
}
});
return(Task.FromResult(0));
}
/// <summary>
/// Trims the cache of any expired items in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> representing an asynchronous operation.
/// </returns>
public override Task TrimCacheAsync()
{
if (!TrimCache)
{
return(Task.FromResult(0));
}
ScheduleCacheTrimmer(async token =>
{
BlobContinuationToken continuationToken = null;
List <IListBlobItem> results = new List <IListBlobItem>();
// Loop through the all the files in a non blocking fashion.
do
{
BlobResultSegment response = await rootContainer.ListBlobsSegmentedAsync(string.Empty, true, BlobListingDetails.Metadata, 5000, continuationToken, null, null, token);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}while (token.IsCancellationRequested == false && continuationToken != null);
// Now leap through and delete.
foreach (
CloudBlockBlob blob in
results.Where((blobItem, type) => blobItem is CloudBlockBlob)
.Cast <CloudBlockBlob>()
.OrderBy(b => b.Properties.LastModified?.UtcDateTime ?? new DateTime()))
{
if (token.IsCancellationRequested)
{
break;
}
if (blob.Properties.LastModified.HasValue && !IsExpired(blob.Properties.LastModified.Value.UtcDateTime))
{
continue;
}
if (!blob.Name.Contains(prefix))
{
continue;
}
// Remove from the cache and delete each CachedImage.
CacheIndexer.Remove(blob.Name);
await blob.DeleteAsync(token);
}
});
return(Task.FromResult(0));
}
/// <summary>
/// Gets a value indicating whether the image is new or updated in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> returning the value.
/// </returns>
public override async Task <bool> IsNewOrUpdatedAsync()
{
string cachedFileName = await this.CreateCachedFileNameAsync();
// Collision rate of about 1 in 10000 for the folder structure.
// That gives us massive scope to store millions of files.
string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6));
this.CachedPath = Path.Combine(cloudCachedBlobContainer.Uri.ToString(), pathFromKey, cachedFileName).Replace(@"\", "/");
// Do we insert the cache container? This seems to break some setups.
bool useCachedContainerInUrl = this.Settings.ContainsKey("UseCachedContainerInUrl") &&
this.Settings["UseCachedContainerInUrl"].ToLower() != "false";
if (useCachedContainerInUrl)
{
this.cachedRewritePath =
Path.Combine(this.cachedCdnRoot, cloudCachedBlobContainer.Name, pathFromKey, cachedFileName)
.Replace(@"\", "/");
}
else
{
this.cachedRewritePath = Path.Combine(this.cachedCdnRoot, pathFromKey, cachedFileName)
.Replace(@"\", "/");
}
bool isUpdated = false;
CachedImage cachedImage = CacheIndexer.Get(this.CachedPath);
if (new Uri(this.CachedPath).IsFile)
{
FileInfo fileInfo = new FileInfo(this.CachedPath);
if (fileInfo.Exists)
{
// Pull the latest info.
fileInfo.Refresh();
cachedImage = new CachedImage
{
Key = Path.GetFileNameWithoutExtension(this.CachedPath),
Path = this.CachedPath,
CreationTimeUtc = fileInfo.CreationTimeUtc
};
CacheIndexer.Add(cachedImage);
}
}
if (cachedImage == null)
{
string blobPath = this.CachedPath.Substring(cloudCachedBlobContainer.Uri.ToString().Length + 1);
CloudBlockBlob blockBlob = cloudCachedBlobContainer.GetBlockBlobReference(blobPath);
if (await blockBlob.ExistsAsync())
{
// Pull the latest info.
await blockBlob.FetchAttributesAsync();
if (blockBlob.Properties.LastModified.HasValue)
{
cachedImage = new CachedImage
{
Key = Path.GetFileNameWithoutExtension(this.CachedPath),
Path = this.CachedPath,
CreationTimeUtc = blockBlob.Properties.LastModified.Value.UtcDateTime
};
CacheIndexer.Add(cachedImage);
}
}
}
if (cachedImage == null)
{
// Nothing in the cache so we should return true.
isUpdated = true;
}
else
{
// Check to see if the cached image is set to expire.
if (this.IsExpired(cachedImage.CreationTimeUtc))
{
CacheIndexer.Remove(this.CachedPath);
isUpdated = true;
}
}
return(isUpdated);
}
/// <summary>
/// Gets a value indicating whether the image is new or updated in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> returning the value.
/// </returns>
public override async Task <bool> IsNewOrUpdatedAsync()
{
// TODO: Before this check is performed it should be throttled. For example, only perform this check
// if the last time it was checked is greater than 5 seconds. This would be much better for perf
// if there is a high throughput of image requests.
string cachedFileName = await this.CreateCachedFileNameAsync();
// Collision rate of about 1 in 10000 for the folder structure.
// That gives us massive scope to store millions of files.
string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6));
this.CachedPath = Path.Combine(this.cachedCdnRoot, this.imageProcessorCachePrefix, pathFromKey, cachedFileName)
.Replace(@"\", "/");
// TODO: What is the S3 version of the following lines? The Above doesn't match what I expect
//this.CachedPath = Path.Combine(this.cloudCachedBlobContainer.Uri.ToString(), pathFromKey, cachedFileName).Replace(@"\", "/");
//this.cachedRewritePath = Path.Combine(this.cachedCdnRoot, this.cloudCachedBlobContainer.Name, pathFromKey, cachedFileName).Replace(@"\", "/");
bool isUpdated = false;
CachedImage cachedImage = CacheIndexer.Get(this.CachedPath);
if (new Uri(this.CachedPath).IsFile)
{
FileInfo fileInfo = new FileInfo(this.CachedPath);
if (fileInfo.Exists)
{
// Pull the latest info.
fileInfo.Refresh();
cachedImage = new CachedImage
{
Key = Path.GetFileNameWithoutExtension(this.CachedPath),
Path = this.CachedPath,
CreationTimeUtc = fileInfo.CreationTimeUtc
};
CacheIndexer.Add(cachedImage);
}
}
if (cachedImage == null)
{
try
{
string path = this.GetFolderStructureForAmazon(this.CachedPath);
string filename = Path.GetFileName(this.CachedPath);
string key = this.GetKey(path, filename);
GetObjectMetadataRequest objectMetaDataRequest = new GetObjectMetadataRequest
{
BucketName = this.awsBucketName,
Key = key,
};
GetObjectMetadataResponse response = await this.amazonS3ClientCache.GetObjectMetadataAsync(objectMetaDataRequest);
if (response != null)
{
cachedImage = new CachedImage
{
Key = key,
Path = this.CachedPath,
CreationTimeUtc = response.LastModified.ToUniversalTime()
};
CacheIndexer.Add(cachedImage);
}
}
catch (AmazonS3Exception)
{
// Nothing in S3 so we should return true.
isUpdated = true;
}
}
if (cachedImage == null)
{
// Nothing in the cache so we should return true.
isUpdated = true;
}
else
{
// Check to see if the cached image is set to expire.
if (this.IsExpired(cachedImage.CreationTimeUtc))
{
CacheIndexer.Remove(this.CachedPath);
isUpdated = true;
}
}
return(isUpdated);
}
/// <summary>
/// Gets a value indicating whether the image is new or updated in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> returning the value.
/// </returns>
public override async Task <bool> IsNewOrUpdatedAsync()
{
// TODO: Before this check is performed it should be throttled. For example, only perform this check
// if the last time it was checked is greater than 5 seconds. This would be much better for perf
// if there is a high throughput of image requests.
string cachedFileName = await this.CreateCachedFileNameAsync().ConfigureAwait(false);
this.CachedPath = CachedImageHelper.GetCachedPath(cloudCachedBlobContainer.Uri.ToString(), cachedFileName, true, this.FolderDepth);
this.cachedRewritePath = CachedImageHelper.GetCachedPath(useCachedContainerInUrl
? Path.Combine(cachedCdnRoot, cloudCachedBlobContainer.Name)
: cachedCdnRoot, cachedFileName, true, this.FolderDepth);
bool isUpdated = false;
CachedImage cachedImage = CacheIndexer.Get(this.CachedPath);
if (new Uri(this.CachedPath).IsFile&& File.Exists(this.CachedPath))
{
cachedImage = new CachedImage
{
Key = Path.GetFileNameWithoutExtension(this.CachedPath),
Path = this.CachedPath,
CreationTimeUtc = File.GetCreationTimeUtc(this.CachedPath)
};
CacheIndexer.Add(cachedImage, this.ImageCacheMaxMinutes);
}
if (cachedImage is null)
{
string blobPath = this.CachedPath.Substring(cloudCachedBlobContainer.Uri.ToString().Length + 1);
CloudBlockBlob blockBlob = cloudCachedBlobContainer.GetBlockBlobReference(blobPath);
if (await blockBlob.ExistsAsync().ConfigureAwait(false))
{
// Pull the latest info.
await blockBlob.FetchAttributesAsync().ConfigureAwait(false);
if (blockBlob.Properties.LastModified.HasValue)
{
cachedImage = new CachedImage
{
Key = Path.GetFileNameWithoutExtension(this.CachedPath),
Path = this.CachedPath,
CreationTimeUtc = blockBlob.Properties.LastModified.Value.UtcDateTime
};
CacheIndexer.Add(cachedImage, this.ImageCacheMaxMinutes);
}
}
}
if (cachedImage is null)
{
// Nothing in the cache so we should return true.
isUpdated = true;
}
else
{
// Check to see if the cached image is set to expire
// or a new file with the same name has replaced our current image
if (this.IsExpired(cachedImage.CreationTimeUtc) || await this.IsUpdatedAsync(cachedImage.CreationTimeUtc).ConfigureAwait(false))
{
CacheIndexer.Remove(this.CachedPath);
isUpdated = true;
}
}
return(isUpdated);
}
/// <summary>
/// Gets a value indicating whether the image is new or updated in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> returning the value.
/// </returns>
public override async Task <bool> IsNewOrUpdatedAsync()
{
string cachedFilename = prefix + await CreateCachedFileNameAsync();
var media = UrlResolver.Current.Route(new UrlBuilder(FullPath)) as MediaData;
string containerName = media?.BinaryDataContainer?.Segments[1];
if (containerName == null)
{
// We're working with a static file here
containerName = $"_{prefix}static";
}
blobPath = $"{containerName}/{cachedFilename}";
CachedPath = $"{rootContainer.Uri.ToString()}/{containerName}/{cachedFilename}";
bool isUpdated = false;
CachedImage cachedImage = CacheIndexer.Get(CachedPath);
if (cachedImage == null)
{
CloudBlockBlob blockBlob = rootContainer.GetBlockBlobReference(blobPath);
//string t = GetSaSForBlob(blockBlob, SharedAccessBlobPermissions.Read);
if (await blockBlob.ExistsAsync())
{
// Pull the latest info.
await blockBlob.FetchAttributesAsync();
if (blockBlob.Properties.LastModified.HasValue)
{
cachedImage = new CachedImage
{
Key = Path.GetFileNameWithoutExtension(CachedPath),
Path = CachedPath,
CreationTimeUtc = blockBlob.Properties.LastModified.Value.UtcDateTime
};
CacheIndexer.Add(cachedImage, ImageCacheMaxMinutes);
}
}
}
if (cachedImage == null)
{
// Nothing in the cache so we should return true.
isUpdated = true;
}
else
{
// Check to see if the cached image is set to expire
// or a new file with the same name has replaced our current image
if (IsExpired(cachedImage.CreationTimeUtc) || await IsUpdatedAsync(cachedImage.CreationTimeUtc))
{
CacheIndexer.Remove(CachedPath);
isUpdated = true;
}
}
return(isUpdated);
}
/// <summary>
/// Trims the cache of any expired items in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> representing an asynchronous operation.
/// </returns>
public override Task TrimCacheAsync()
{
if (!TrimCache)
{
return(Task.FromResult(0));
}
ScheduleCacheTrimmer(token =>
{
string rootDirectory = Path.GetDirectoryName(CachedPath);
if (rootDirectory != null)
{
// Jump up to the parent branch to clean through the cache.
// UNC folders can throw exceptions if the file doesn't exist.
IEnumerable <string> directories = Directory.EnumerateDirectories(absoluteCachePath).Reverse();
foreach (string directory in directories)
{
if (!Directory.Exists(directory))
{
continue;
}
if (token.IsCancellationRequested)
{
break;
}
IEnumerable <FileInfo> files = Directory.EnumerateFiles(directory)
.Select(f => new FileInfo(f))
.Where(f => f.Name.StartsWith(prefix))
.OrderBy(f => f.CreationTimeUtc);
foreach (FileInfo fileInfo in files)
{
if (token.IsCancellationRequested)
{
break;
}
try
{
if (!IsExpired(fileInfo.CreationTimeUtc))
{
continue;
}
// Remove from the cache and delete each CachedImage.
CacheIndexer.Remove(fileInfo.Name);
fileInfo.Delete();
}
catch (Exception ex)
{
// Log it but skip to the next file.
logger.Error($"Unable to clean cached file: {fileInfo.FullName}", ex);
}
}
// If the directory is empty of files delete it to remove the FCN.
RecursivelyDeleteEmptyDirectories(directory, absoluteCachePath, token);
}
}
return(Task.FromResult(0));
});
return(Task.FromResult(0));
}
/// <summary>
/// Gets a value indicating whether the image is new or updated in an asynchronous manner.
/// </summary>
/// <returns>
/// The asynchronous <see cref="Task"/> returning the value.
/// </returns>
public override async Task <bool> IsNewOrUpdatedAsync()
{
string cachedFileName = await this.CreateCachedFileNameAsync();
// Collision rate of about 1 in 10000 for the folder structure.
// That gives us massive scope to store millions of files.
string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6));
this.CachedPath = Path.Combine(this.cachedCdnRoot, this.imageProcessorCachePrefix, pathFromKey, cachedFileName)
.Replace(@"\", "/");
bool isUpdated = false;
CachedImage cachedImage = CacheIndexer.Get(this.CachedPath);
if (new Uri(this.CachedPath).IsFile)
{
FileInfo fileInfo = new FileInfo(this.CachedPath);
if (fileInfo.Exists)
{
// Pull the latest info.
fileInfo.Refresh();
cachedImage = new CachedImage
{
Key = Path.GetFileNameWithoutExtension(this.CachedPath),
Path = this.CachedPath,
CreationTimeUtc = fileInfo.CreationTimeUtc
};
CacheIndexer.Add(cachedImage);
}
}
if (cachedImage == null)
{
try
{
string path = this.GetFolderStructureForAmazon(this.CachedPath);
string filename = Path.GetFileName(this.CachedPath);
string key = this.GetKey(path, filename);
GetObjectMetadataRequest objectMetaDataRequest = new GetObjectMetadataRequest
{
BucketName = this.awsBucketName,
Key = key,
};
GetObjectMetadataResponse response = await this.amazonS3ClientCache.GetObjectMetadataAsync(objectMetaDataRequest);
if (response != null)
{
cachedImage = new CachedImage
{
Key = key,
Path = this.CachedPath,
CreationTimeUtc = response.LastModified.ToUniversalTime()
};
CacheIndexer.Add(cachedImage);
}
}
catch (AmazonS3Exception)
{
// Nothing in S3 so we should return true.
isUpdated = true;
}
}
if (cachedImage == null)
{
// Nothing in the cache so we should return true.
isUpdated = true;
}
else
{
// Check to see if the cached image is set to expire.
if (this.IsExpired(cachedImage.CreationTimeUtc))
{
CacheIndexer.Remove(this.CachedPath);
isUpdated = true;
}
}
return(isUpdated);
}
