private async Task <Stream> GetPackageStreamAsync( IAzureStorage sourceStorage, string packageId, string normalizedPackageVersion, CancellationToken cancellationToken) { var packageFileName = PackageUtility.GetPackageFileName(packageId, normalizedPackageVersion); var sourceUri = sourceStorage.ResolveUri(packageFileName); var packageSourceBlob = await sourceStorage.GetCloudBlockBlobReferenceAsync(sourceUri); return(await packageSourceBlob.GetStreamAsync(cancellationToken)); }
private async Task ProcessEmbeddedIconAsync(IStorage destinationStorage, CatalogCommitItem item, string iconFile, CancellationToken cancellationToken) { var packageFilename = PackageUtility.GetPackageFileName(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()).ToLowerInvariant(); var packageUri = _packageStorage.ResolveUri(packageFilename); var packageBlobReference = await _packageStorage.GetCloudBlockBlobReferenceAsync(packageUri); using (_telemetryService.TrackEmbeddedIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString())) { Stream packageStream; try { packageStream = await packageBlobReference.GetStreamAsync(cancellationToken); } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound) { _logger.LogWarning("Package blob not found at {PackageUrl}: {Exception}. Will assume package was deleted and skip", packageUri.AbsoluteUri, ex); return; } catch (Exception ex) { // logging other exceptions here to have proper scope in log message _logger.LogError("Exception while trying to access package blob {PackageUrl}: {Exception}", packageUri.AbsoluteUri, ex); throw; } using (packageStream) { var targetStoragePath = GetTargetStorageIconPath(item); var resultUrl = await _iconProcessor.CopyEmbeddedIconFromPackageAsync( packageStream, iconFile, destinationStorage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); } } }
private async Task ProcessEmbeddedIconAsync(IStorage destinationStorage, CatalogCommitItem item, string iconFile, CancellationToken cancellationToken) { var packageFilename = PackageUtility.GetPackageFileName(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()).ToLowerInvariant(); var packageUri = _packageStorage.ResolveUri(packageFilename); var packageBlobReference = await _packageStorage.GetCloudBlockBlobReferenceAsync(packageUri); using (_telemetryService.TrackEmbeddedIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString())) using (var packageStream = await packageBlobReference.GetStreamAsync(cancellationToken)) { var targetStoragePath = GetTargetStorageIconPath(item); var resultUrl = await _iconProcessor.CopyEmbeddedIconFromPackageAsync( packageStream, iconFile, destinationStorage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); } }
private async Task <PackageCatalogItem> GetPackageViaStorageAsync( FeedPackageDetails packageItem, CancellationToken cancellationToken) { PackageCatalogItem item = null; var packageId = packageItem.PackageId.ToLowerInvariant(); var packageNormalizedVersion = packageItem.PackageNormalizedVersion.ToLowerInvariant(); var packageFileName = PackageUtility.GetPackageFileName(packageId, packageNormalizedVersion); var blobUri = _storage.ResolveUri(packageFileName); var blob = await _storage.GetCloudBlockBlobReferenceAsync(blobUri); if (!await blob.ExistsAsync(cancellationToken)) { _telemetryService.TrackMetric( TelemetryConstants.NonExistentBlob, metric: 1, properties: GetProperties(packageId, packageNormalizedVersion, blob)); return(item); } using (_telemetryService.TrackDuration( TelemetryConstants.PackageBlobReadSeconds, GetProperties(packageId, packageNormalizedVersion, blob: null))) { await blob.FetchAttributesAsync(cancellationToken); string packageHash = null; var etag = blob.ETag; var metadata = await blob.GetMetadataAsync(cancellationToken); if (metadata.TryGetValue(Constants.Sha512, out packageHash)) { using (var stream = await blob.GetStreamAsync(cancellationToken)) { item = Utils.CreateCatalogItem( packageItem.ContentUri.ToString(), stream, packageItem.CreatedDate, packageItem.LastEditedDate, packageItem.PublishedDate, licenseNames: null, licenseReportUrl: null, packageHash: packageHash, deprecationItem: packageItem.DeprecationInfo); if (item == null) { _logger.LogWarning("Unable to extract metadata from: {PackageDetailsContentUri}", packageItem.ContentUri); } } if (item != null) { // Since obtaining the ETag the first time, it's possible (though unlikely) that the blob may // have changed. Although reading a blob with a single GET request should return the whole // blob in a consistent state, we're reading the blob using ZipArchive and a seekable stream, // which results in many GET requests. To guard against the blob having changed since we // obtained the package hash, we check the ETag one more time. If this check fails, we'll // fallback to using a single HTTP GET request. await blob.FetchAttributesAsync(cancellationToken); if (etag != blob.ETag) { item = null; _telemetryService.TrackMetric( TelemetryConstants.BlobModified, metric: 1, properties: GetProperties(packageId, packageNormalizedVersion, blob)); } } } else { _telemetryService.TrackMetric( TelemetryConstants.NonExistentPackageHash, metric: 1, properties: GetProperties(packageId, packageNormalizedVersion, blob)); } } return(item); }