private void EnsureBlobExists()
{
if (m_blob == null)
{
m_blob = m_blobContainer.CreateBlockBlob(m_consumerId + "/" + m_streamName);
}
}
private async Task <HttpContent> StoreContentAsync(string rowKey, HttpContent originalContent, CancellationToken cancellationToken)
{
HttpContent storedContent = null;
if (originalContent != null)
{
ICloudBlockBlob blob = _blobContainer.GetBlockBlobReference(rowKey);
Stream originalStream = await originalContent.ReadAsStreamAsync();
Stream destinationStream = await blob.OpenWriteAsync(cancellationToken);
if (_useCompression)
{
destinationStream = new GZipStream(destinationStream, CompressionLevel.Optimal);
}
using (destinationStream)
{
await originalStream.CopyToAsync(destinationStream, BufferSize, cancellationToken);
}
storedContent = await GetStoredHttpContentAsync(rowKey, originalContent.Headers, _useCompression, cancellationToken);
}
return(storedContent);
}
private async Task <string> GetNuspecAsync(
ICloudBlockBlob sourceBlob,
string packageId,
CancellationToken cancellationToken)
{
using (var stream = await sourceBlob.GetStreamAsync(cancellationToken))
{
return(GetNuspec(stream, packageId));
}
}
public static async Task <Lease> ReleaseAsync(ICloudBlockBlob blob, Lease lease)
{
Require.NotNull(blob, "blob");
Require.NotNull(lease, "lease");
Ensure.True(lease.m_leaseId.IsNotNullOrEmpty(), "Lease was not acquired");
await blob.ReleaseLeaseAsync(lease.m_leaseId);
return(NotAcquired);
}
public async Task AcquireLeaseAsync()
{
if (this.blob != null)
{
throw new InvalidOperationException("The lease has already been acquired.");
}
this.blob = this.blobLeaseHelper.GetLeaseBlob(this.leaseObjectName);
this.leaseId = await this.blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null, this.cancellationToken);
this.acquiredTimestamp = this.timestampCreator.Now();
this.lastRenewedTimestamp = this.acquiredTimestamp;
}
private async Task <HttpContent> GetStoredHttpContentAsync(string rowKey, IEnumerable <KeyValuePair <string, IEnumerable <string> > > headers, bool isCompressed, CancellationToken cancellationToken)
{
ICloudBlockBlob blob = _blobContainer.GetBlockBlobReference(rowKey);
Stream stream = await blob.OpenReadAsync(cancellationToken);
if (isCompressed)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
var httpContent = new StreamContent(stream);
httpContent.Headers.AddRange(headers);
return(httpContent);
}
public PendingNotificationsChaser(
IPendingNotifications pendingNotifications,
INotificationHub notificationHub,
IPollingJob pollingJob,
ICloudBlockBlob chaserExclusiveAccessBlobLock)
{
Require.NotNull(pendingNotifications, "pendingNotifications");
Require.NotNull(notificationHub, "notificationHub");
Require.NotNull(pollingJob, "pollingJob");
Require.NotNull(chaserExclusiveAccessBlobLock, "chaserExclusiveAccessBlobLock");
m_pendingNotifications = pendingNotifications;
m_notificationHub = notificationHub;
m_pollingJob = pollingJob;
m_chaserExclusiveAccessBlobLock = chaserExclusiveAccessBlobLock;
}
public static async Task <Lease> AcquireAsync(ICloudBlockBlob blob, TimeSpan leasePeriod)
{
Require.NotNull(blob, "blob");
try
{
var leaseId = await blob.AcquireLeaseAsync(leasePeriod);
return(new Lease(leaseId));
}
catch (LeaseAlreadyAcquiredException)
{
}
return(NotAcquired);
}
public async Task <bool> AreSynchronized(ICloudBlockBlob sourceBlockBlob, ICloudBlockBlob destinationBlockBlob)
{
if (await destinationBlockBlob.ExistsAsync(CancellationToken.None))
{
if (await sourceBlockBlob.ExistsAsync(CancellationToken.None))
{
var sourceBlobMetadata = await sourceBlockBlob.GetMetadataAsync(CancellationToken.None);
var destinationBlobMetadata = await destinationBlockBlob.GetMetadataAsync(CancellationToken.None);
if (sourceBlobMetadata == null || destinationBlobMetadata == null)
{
return(false);
}
var sourceBlobHasSha512Hash = sourceBlobMetadata.TryGetValue(Sha512HashAlgorithmId, out var sourceBlobSha512Hash);
var destinationBlobHasSha512Hash = destinationBlobMetadata.TryGetValue(Sha512HashAlgorithmId, out var destinationBlobSha512Hash);
if (!sourceBlobHasSha512Hash)
{
Trace.TraceWarning(string.Format("The source blob ({0}) doesn't have the SHA512 hash.", sourceBlockBlob.Uri.ToString()));
}
if (!destinationBlobHasSha512Hash)
{
Trace.TraceWarning(string.Format("The destination blob ({0}) doesn't have the SHA512 hash.", destinationBlockBlob.Uri.ToString()));
}
if (sourceBlobHasSha512Hash && destinationBlobHasSha512Hash)
{
if (sourceBlobSha512Hash == destinationBlobSha512Hash)
{
Trace.WriteLine(string.Format("The source blob ({0}) and destination blob ({1}) have the same SHA512 hash and are synchronized.",
sourceBlockBlob.Uri.ToString(), destinationBlockBlob.Uri.ToString()));
return(true);
}
// The SHA512 hash between the source and destination blob should be always same.
Trace.TraceWarning(string.Format("The source blob ({0}) and destination blob ({1}) have the different SHA512 hash and are not synchronized. " +
"The source blob hash is {2} while the destination blob hash is {3}",
sourceBlockBlob.Uri.ToString(), destinationBlockBlob.Uri.ToString(), sourceBlobSha512Hash, destinationBlobSha512Hash));
}
return(false);
}
return(true);
}
return(!(await sourceBlockBlob.ExistsAsync(CancellationToken.None)));
}
private async Task OrientAndSetMetadataAsync(MagickImage image, ICloudBlockBlob output, CancellationToken cancellationToken, ILogger logger, Stopwatch sw)
{
if (image.Orientation != OrientationType.Undefined && image.Orientation != OrientationType.TopLeft)
{
image.AutoOrient();
using (var outputStream = await output.OpenWriteAsync(cancellationToken))
{
image.Write(outputStream);
SetMetadata(image, output);
await Task.Factory.FromAsync(outputStream.BeginCommit(null, null), outputStream.EndCommit);
}
logger.Info("Orient: " + sw.ElapsedMilliseconds);
}
else
{
await SetAndSaveMetadata(output, image);
}
}
private async Task <MagickImage> OpenImageAsync(ICloudBlockBlob input, CancellationToken cancellationToken, ILogger logger, Stopwatch sw)
{
// Magick.NET doesn't read from the blob stream correctly, so we download the entire blob first to a buffer and read from there.
// The buffer only exists for a short period in this scope, lessening memory pressure.
byte[] buffer;
using (var inputStream = await input.OpenReadAsync(cancellationToken))
{
logger.Info("OpenStream: " + sw.ElapsedMilliseconds);
buffer = new byte[inputStream.Length];
using (var ms = new MemoryStream(buffer))
{
await inputStream.CopyToAsync(ms, buffer.Length, cancellationToken);
}
logger.Info("ReadData: " + sw.ElapsedMilliseconds);
}
return(new MagickImage(buffer));
}
public async Task <bool> TrySaveItemAsync(T item)
{
ICloudBlockBlob blockBlob = _container.GetBlockBlobReference(GetName(item));
await blockBlob.UploadFromByteArrayAsync(_getBytes(item), 0, _getBytes(item).Length);
foreach (var metaItem in GetMetaData(item).Where(pair => !string.IsNullOrWhiteSpace(pair.Value.Value)))
{
blockBlob.Metadata[$"meta_{metaItem.Key}"] = metaItem.Value.Value;
}
try
{
await blockBlob.SetMetadataAsync();
}
catch (Exception ex)
{
return(false);
}
return(true);
}
public async Task ProcessPackageAsync(CatalogIndexEntry packageEntry, ICloudBlockBlob blob)
{
await blob.FetchAttributesAsync(CancellationToken.None);
// Skip the package if it has a Content MD5 hash
if (blob.ContentMD5 != null)
{
_telemetryService.TrackPackageAlreadyHasHash(packageEntry.Id, packageEntry.Version);
return;
}
// Download the blob and calculate its hash. We use HttpClient to download blobs as Azure Blob Sotrage SDK
// occassionally hangs. See: https://github.com/Azure/azure-storage-net/issues/470
string hash;
using (var hashAlgorithm = MD5.Create())
using (var packageStream = await _httpClient.GetStreamAsync(blob.Uri))
{
var hashBytes = hashAlgorithm.ComputeHash(packageStream);
hash = Convert.ToBase64String(hashBytes);
}
blob.ContentMD5 = hash;
var condition = AccessCondition.GenerateIfMatchCondition(blob.ETag);
await blob.SetPropertiesAsync(
condition,
options : null,
operationContext : null);
_telemetryService.TrackPackageHashFixed(packageEntry.Id, packageEntry.Version);
_logger.LogWarning(
"Updated package {PackageId} {PackageVersion}, set hash to '{Hash}' using ETag {ETag}",
packageEntry.Id,
packageEntry.Version,
hash,
blob.ETag);
}
public async Task ProcessPackageAsync(CatalogIndexEntry packageEntry, ICloudBlockBlob blob)
{
await blob.FetchAttributesAsync(CancellationToken.None);
if (blob.ContentMD5 == null)
{
_telemetryService.TrackPackageMissingHash(packageEntry.Id, packageEntry.Version);
_logger.LogError(
"Package {PackageId} {PackageVersion} has a null Content MD5 hash!",
packageEntry.Id,
packageEntry.Version);
return;
}
// Download the blob and calculate its hash. We use HttpClient to download blobs as Azure Blob Sotrage SDK
// occassionally hangs. See: https://github.com/Azure/azure-storage-net/issues/470
string hash;
using (var hashAlgorithm = MD5.Create())
using (var packageStream = await _httpClient.GetStreamAsync(blob.Uri))
{
var hashBytes = hashAlgorithm.ComputeHash(packageStream);
hash = Convert.ToBase64String(hashBytes);
}
if (blob.ContentMD5 != hash)
{
_telemetryService.TrackPackageHasIncorrectHash(packageEntry.Id, packageEntry.Version);
_logger.LogError(
"Package {PackageId} {PackageVersion} has an incorrect Content MD5 hash! Expected: '{ExpectedHash}', actual: '{ActualHash}'",
packageEntry.Id,
packageEntry.Version,
hash,
blob.ContentMD5);
}
}
public async Task CreateThumbnailSetAsync(
CreateThumbnailsMessage message,
ICloudBlockBlob input,
ICloudStorageAccount cloudStorageAccount,
ILogger logger,
int dequeueCount,
CancellationToken cancellationToken)
{
try
{
var startProcessingTime = DateTime.UtcNow;
var result = await this.thumbnailProcessor.CreateThumbnailSetAsync(
message,
input,
cloudStorageAccount,
logger,
cancellationToken);
var completeProcessingTime = DateTime.UtcNow;
if (result != null)
{
await this.setFileProcessingComplete.ExecuteAsync(
message.FileId,
dequeueCount,
startProcessingTime,
completeProcessingTime,
result.RenderWidth,
result.RenderHeight);
}
}
catch (Exception t)
{
logger.Error(t);
throw;
}
}
private static async Task SetAndSaveMetadata(ICloudBlockBlob input, MagickImage image)
{
SetMetadata(image, input);
await input.SetMetadataAsync();
}
private static void SetMetadata(MagickImage image, ICloudBlockBlob blockBlob)
{
blockBlob.Metadata[WidthHeaderKey] = image.Width.ToString();
blockBlob.Metadata[HeightHeaderKey] = image.Height.ToString();
}
private void EnsureBlobExists()
{
if (m_blob == null)
{
m_blob = m_blobContainer.CreateBlockBlob(m_consumerId + "/" + m_streamName);
}
}
private static Dictionary <string, string> GetProperties(FeedPackageDetails packageItem, ICloudBlockBlob blob)
{
return(GetProperties(
packageItem.PackageId.ToLowerInvariant(),
packageItem.PackageNormalizedVersion.ToLowerInvariant(),
blob));
}
private static Dictionary <string, string> GetProperties(string packageId, string packageNormalizedVersion, ICloudBlockBlob blob)
{
var properties = new Dictionary <string, string>()
{
{ TelemetryConstants.Id, packageId },
{ TelemetryConstants.Version, packageNormalizedVersion }
};
if (blob != null)
{
properties.Add(TelemetryConstants.Uri, blob.Uri.AbsoluteUri);
}
return(properties);
}
public static Task <string> AcquireLeaseAsync(this ICloudBlockBlob blob)
{
Require.NotNull(blob, "blob");
return(blob.AcquireLeaseAsync(null));
}
public ThumbnailSetItemData(ICloudBlockBlob blockBlob)
{
this.BlockBlob = blockBlob;
}
public async Task <CreateThumbnailSetResult> CreateThumbnailSetAsync(
CreateThumbnailsMessage message,
ICloudBlockBlob input,
ICloudStorageAccount cloudStorageAccount,
ILogger logger,
CancellationToken cancellationToken)
{
CreateThumbnailSetResult result = null;
var sw = new Stopwatch();
sw.Start();
logger.Info("StartFileProcessor: " + sw.ElapsedMilliseconds);
if (message.Items == null || message.Items.Count == 0)
{
return(null);
}
var cache = this.GetItemsCache(message, cloudStorageAccount);
logger.Info("GetBlockBlobs: " + sw.ElapsedMilliseconds);
await this.PopulateExists(message, cancellationToken, cache);
logger.Info("CheckExists: " + sw.ElapsedMilliseconds);
if (message.Overwrite == false && cache.Values.All(v => v.Exists))
{
return(null);
}
await input.FetchAttributesAsync(cancellationToken);
var cacheControl = input.Properties.CacheControl;
this.PopulateCacheControl(cacheControl, cache);
using (var image = await this.OpenImageAsync(input, cancellationToken, logger, sw))
{
logger.Info("OpenImage: " + sw.ElapsedMilliseconds);
var outputMimeType = image.FormatInfo.MimeType;
if (!SupportedOutputMimeTypes.Contains(outputMimeType))
{
outputMimeType = DefaultOutputMimeType;
image.Format = DefaultOutputFormat;
logger.Info("ConvertToJpeg: " + sw.ElapsedMilliseconds);
}
if (image.Format == MagickFormat.Jpg || image.Format == MagickFormat.Jpeg)
{
var colorProfile = image.GetColorProfile();
if (colorProfile == null)
{
image.AddProfile(ColorProfile.SRGB);
logger.Info("AddColorProfile: " + sw.ElapsedMilliseconds);
}
if (image.Quality > 85)
{
image.Quality = 85;
}
await this.OrientAndSetMetadataAsync(image, input, cancellationToken, logger, sw);
}
else
{
await SetAndSaveMetadata(input, image);
}
var jobData = new JobData(outputMimeType);
logger.Info("Processing: " + sw.ElapsedMilliseconds);
result = new CreateThumbnailSetResult(image.Width, image.Height);
await this.ProcessThumbnailItems(message.Items, cache, image, jobData, logger, cancellationToken);
logger.Info("ProcessingComplete: " + sw.ElapsedMilliseconds);
}
logger.Info("Disposed: " + sw.ElapsedMilliseconds);
return(result);
}
public static Task BreakLeaseAsync(this ICloudBlockBlob blob)
{
Require.NotNull(blob, "blob");
return(blob.BreakLeaseAsync(null));
}
