private async Task <Stream> GetPackageStreamAsync(
IAzureStorage sourceStorage,
string packageId,
string normalizedPackageVersion,
CancellationToken cancellationToken)
{
var packageFileName = PackageUtility.GetPackageFileName(packageId, normalizedPackageVersion);
var sourceUri = sourceStorage.ResolveUri(packageFileName);
var packageSourceBlob = await sourceStorage.GetCloudBlockBlobReferenceAsync(sourceUri);
return(await packageSourceBlob.GetStreamAsync(cancellationToken));
}
private async Task ProcessEmbeddedIconAsync(IStorage destinationStorage, CatalogCommitItem item, string iconFile, CancellationToken cancellationToken)
{
var packageFilename = PackageUtility.GetPackageFileName(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()).ToLowerInvariant();
var packageUri = _packageStorage.ResolveUri(packageFilename);
var packageBlobReference = await _packageStorage.GetCloudBlockBlobReferenceAsync(packageUri);
using (_telemetryService.TrackEmbeddedIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()))
{
Stream packageStream;
try
{
packageStream = await packageBlobReference.GetStreamAsync(cancellationToken);
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
_logger.LogWarning("Package blob not found at {PackageUrl}: {Exception}. Will assume package was deleted and skip",
packageUri.AbsoluteUri,
ex);
return;
}
catch (Exception ex)
{
// logging other exceptions here to have proper scope in log message
_logger.LogError("Exception while trying to access package blob {PackageUrl}: {Exception}",
packageUri.AbsoluteUri,
ex);
throw;
}
using (packageStream)
{
var targetStoragePath = GetTargetStorageIconPath(item);
var resultUrl = await _iconProcessor.CopyEmbeddedIconFromPackageAsync(
packageStream,
iconFile,
destinationStorage,
targetStoragePath,
cancellationToken,
item.PackageIdentity.Id,
item.PackageIdentity.Version.ToNormalizedString());
}
}
}
private async Task ProcessEmbeddedIconAsync(IStorage destinationStorage, CatalogCommitItem item, string iconFile, CancellationToken cancellationToken)
{
var packageFilename = PackageUtility.GetPackageFileName(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()).ToLowerInvariant();
var packageUri = _packageStorage.ResolveUri(packageFilename);
var packageBlobReference = await _packageStorage.GetCloudBlockBlobReferenceAsync(packageUri);
using (_telemetryService.TrackEmbeddedIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()))
using (var packageStream = await packageBlobReference.GetStreamAsync(cancellationToken))
{
var targetStoragePath = GetTargetStorageIconPath(item);
var resultUrl = await _iconProcessor.CopyEmbeddedIconFromPackageAsync(
packageStream,
iconFile,
destinationStorage,
targetStoragePath,
cancellationToken,
item.PackageIdentity.Id,
item.PackageIdentity.Version.ToNormalizedString());
}
}
private async Task <PackageCatalogItem> GetPackageViaStorageAsync(
FeedPackageDetails packageItem,
CancellationToken cancellationToken)
{
PackageCatalogItem item = null;
var packageId = packageItem.PackageId.ToLowerInvariant();
var packageNormalizedVersion = packageItem.PackageNormalizedVersion.ToLowerInvariant();
var packageFileName = PackageUtility.GetPackageFileName(packageId, packageNormalizedVersion);
var blobUri = _storage.ResolveUri(packageFileName);
var blob = await _storage.GetCloudBlockBlobReferenceAsync(blobUri);
if (!await blob.ExistsAsync(cancellationToken))
{
_telemetryService.TrackMetric(
TelemetryConstants.NonExistentBlob,
metric: 1,
properties: GetProperties(packageId, packageNormalizedVersion, blob));
return(item);
}
using (_telemetryService.TrackDuration(
TelemetryConstants.PackageBlobReadSeconds,
GetProperties(packageId, packageNormalizedVersion, blob: null)))
{
await blob.FetchAttributesAsync(cancellationToken);
string packageHash = null;
var etag = blob.ETag;
var metadata = await blob.GetMetadataAsync(cancellationToken);
if (metadata.TryGetValue(Constants.Sha512, out packageHash))
{
using (var stream = await blob.GetStreamAsync(cancellationToken))
{
item = Utils.CreateCatalogItem(
packageItem.ContentUri.ToString(),
stream,
packageItem.CreatedDate,
packageItem.LastEditedDate,
packageItem.PublishedDate,
licenseNames: null,
licenseReportUrl: null,
packageHash: packageHash,
deprecationItem: packageItem.DeprecationInfo);
if (item == null)
{
_logger.LogWarning("Unable to extract metadata from: {PackageDetailsContentUri}", packageItem.ContentUri);
}
}
if (item != null)
{
// Since obtaining the ETag the first time, it's possible (though unlikely) that the blob may
// have changed. Although reading a blob with a single GET request should return the whole
// blob in a consistent state, we're reading the blob using ZipArchive and a seekable stream,
// which results in many GET requests. To guard against the blob having changed since we
// obtained the package hash, we check the ETag one more time. If this check fails, we'll
// fallback to using a single HTTP GET request.
await blob.FetchAttributesAsync(cancellationToken);
if (etag != blob.ETag)
{
item = null;
_telemetryService.TrackMetric(
TelemetryConstants.BlobModified,
metric: 1,
properties: GetProperties(packageId, packageNormalizedVersion, blob));
}
}
}
else
{
_telemetryService.TrackMetric(
TelemetryConstants.NonExistentPackageHash,
metric: 1,
properties: GetProperties(packageId, packageNormalizedVersion, blob));
}
}
return(item);
}
