private async Task BackupPackageBinaries(IEnumerable <Package> packages) { // Backup the package binaries and remove from main storage foreach (var package in packages) { // Backup the package and symbols package from the "validating" container. await BackupFromValidationsContainerAsync(_packageFileService, package); await BackupFromValidationsContainerAsync(_symbolPackageFileService, package); // Backup the package and symbols package from the "packages"/"symbol-packages" containers, respectively. await BackupFromPackagesContainerAsync(_packageFileService, package); await BackupFromPackagesContainerAsync(_symbolPackageFileService, package); var id = package.PackageRegistration.Id; var version = string.IsNullOrEmpty(package.NormalizedVersion) ? NuGetVersion.Parse(package.Version).ToNormalizedString() : package.NormalizedVersion; await _packageFileService.DeletePackageFileAsync(id, version); // we didn't backup license file before deleting it because it is backed up as part of the package await _coreLicenseFileService.DeleteLicenseFileAsync(id, version); await _symbolPackageFileService.DeletePackageFileAsync(id, version); await _packageFileService.DeleteValidationPackageFileAsync(id, version); await _symbolPackageFileService.DeleteValidationPackageFileAsync(id, version); // Delete readme file for this package. await TryDeleteReadMeMdFile(package); } }
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile) { if (package.PackageStatusKey != PackageStatus.Available && package.PackageStatusKey != PackageStatus.Validating) { throw new ArgumentException( $"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.", nameof(package)); } try { if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.SaveValidationPackageFileAsync(package, packageFile); } else { await _packageFileService.SavePackageFileAsync(package, packageFile); } } catch (InvalidOperationException ex) { ex.Log(); return(PackageCommitResult.Conflict); } try { // commit all changes to database as an atomic transaction await _entitiesContext.SaveChangesAsync(); } catch { // If saving to the DB fails for any reason we need to delete the package we just saved. if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); } else { await _packageFileService.DeletePackageFileAsync( package.PackageRegistration.Id, package.Version); } throw; } return(PackageCommitResult.Success); }
private async Task BackupPackageBinaries(IEnumerable <Package> packages) { // Backup the package binaries and remove from main storage foreach (var package in packages) { // Backup the package from the "validating" container. using (var packageStream = await _packageFileService.DownloadValidationPackageFileAsync(package)) { if (packageStream != null) { await _packageFileService.StorePackageFileInBackupLocationAsync(package, packageStream); } } // Backup the package from the "packages" container. using (var packageStream = await _packageFileService.DownloadPackageFileAsync(package)) { if (packageStream != null) { await _packageFileService.StorePackageFileInBackupLocationAsync(package, packageStream); } } var id = package.PackageRegistration.Id; var version = string.IsNullOrEmpty(package.NormalizedVersion) ? NuGetVersion.Parse(package.Version).ToNormalizedString() : package.NormalizedVersion; await _packageFileService.DeletePackageFileAsync(id, version); await _packageFileService.DeleteValidationPackageFileAsync(id, version); // Delete any active or pending readme files for this package. await TryDeleteReadMeMdFile(package, false); await TryDeleteReadMeMdFile(package, true); } }
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile) { if (package == null) { throw new ArgumentNullException(nameof(package)); } if (packageFile == null) { throw new ArgumentNullException(nameof(packageFile)); } if (!packageFile.CanSeek) { throw new ArgumentException($"{nameof(packageFile)} argument must be seekable stream", nameof(packageFile)); } await _validationService.UpdatePackageAsync(package); if (package.PackageStatusKey != PackageStatus.Available && package.PackageStatusKey != PackageStatus.Validating) { throw new ArgumentException( $"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.", nameof(package)); } try { if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.SaveValidationPackageFileAsync(package, packageFile); /* Suppose two package upload requests come in at the same time with the same package (same ID and * version). It's possible that one request has committed and validated the package AFTER the other * request has checked that this package does not exist in the database. Observe the following * sequence of events to understand why the packages container check is necessary. * * Request | Step | Component | Success | Notes * ------- | ---------------------------------------------- | ---------------- | ------- | ----- * 1 | version should not exist in DB | gallery | TRUE | 1st duplicate check (catches most cases over time) * 2 | version should not exist in DB | gallery | TRUE | * 1 | upload to validation container | gallery | TRUE | 2nd duplicate check (relevant with high concurrency) * 1 | version should not exist in packages container | gallery | TRUE | 3rd duplicate check (relevant with fast validations) * 1 | commit to DB | gallery | TRUE | * 1 | upload to packages container | async validation | TRUE | * 1 | move package to Available status in DB | async validation | TRUE | * 1 | delete from validation container | async validation | TRUE | * 2 | upload to validation container | gallery | TRUE | * 2 | version should not exist in packages container | gallery | FALSE | * 2 | delete from validation (rollback) | gallery | TRUE | Only occurs in the failure case, as a clean-up. * * Alternatively, we could handle the DB conflict exception that would occur in request 2, but this * would result in an exception that can be avoided and require some ugly code that teases the * unique constraint failure out of a SqlException. * * Another alternative is always leaving the package in the validation container. This is not great * since it doubles the amount of space we need to store packages. Also, it complicates the soft or * hard package delete flow. * * We can safely delete the validation package because we know it's ours. We know this because * saving the validation package succeeded, meaning async validation already successfully moved the * previous package (request 1's package) from the validation container to the package container * and transitioned the package to Available status. * * See the following issue in GitHub for how this case was found: * https://github.com/NuGet/NuGetGallery/issues/5039 */ if (await _packageFileService.DoesPackageFileExistAsync(package)) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); return(PackageCommitResult.Conflict); } } else { if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent) { // if the package is immediately made available, it means there is a high chance we don't have // validation pipeline that would normally store the license file, so we'll do it ourselves here. await _coreLicenseFileService.ExtractAndSaveLicenseFileAsync(package, packageFile); } var isReadmeFileExtractedAndSaved = false; if (package.HasReadMe && package.EmbeddedReadmeType != EmbeddedReadmeFileType.Absent) { await _packageFileService.ExtractAndSaveReadmeFileAsync(package, packageFile); isReadmeFileExtractedAndSaved = true; } try { packageFile.Seek(0, SeekOrigin.Begin); await _packageFileService.SavePackageFileAsync(package, packageFile); } catch when(package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent || isReadmeFileExtractedAndSaved) { if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent) { await _coreLicenseFileService.DeleteLicenseFileAsync( package.PackageRegistration.Id, package.NormalizedVersion); } if (isReadmeFileExtractedAndSaved) { await _packageFileService.DeleteReadMeMdFileAsync(package); } throw; } } } catch (FileAlreadyExistsException ex) { ex.Log(); return(PackageCommitResult.Conflict); } try { // Sending the validation request after copying to prevent multiple validation requests // sent when several pushes for the same package happen concurrently. Copying the file // resolves the race and only one request will "win" and reach this code. await _validationService.StartValidationAsync(package); // commit all changes to database as an atomic transaction await _entitiesContext.SaveChangesAsync(); } catch (Exception ex) { // If sending the validation request or saving to the DB fails for any reason // we need to delete the package we just saved. if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); } else { await _packageFileService.DeletePackageFileAsync( package.PackageRegistration.Id, package.Version); await _coreLicenseFileService.DeleteLicenseFileAsync( package.PackageRegistration.Id, package.NormalizedVersion); await _packageFileService.DeleteReadMeMdFileAsync(package); } return(ReturnConflictOrThrow(ex)); } return(PackageCommitResult.Success); }
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile) { await _validationService.StartValidationAsync(package); if (package.PackageStatusKey != PackageStatus.Available && package.PackageStatusKey != PackageStatus.Validating) { throw new ArgumentException( $"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.", nameof(package)); } try { if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.SaveValidationPackageFileAsync(package, packageFile); /* Suppose two package upload requests come in at the same time with the same package (same ID and * version). It's possible that one request has committed and validated the package AFTER the other * request has checked that this package does not exist in the database. Observe the following * sequence of events to understand why the packages container check is necessary. * * Request | Step | Component | Success | Notes * ------- | ---------------------------------------------- | ---------------- | ------- | ----- * 1 | version should not exist in DB | gallery | TRUE | 1st duplicate check (catches most cases over time) * 2 | version should not exist in DB | gallery | TRUE | * 1 | upload to validation container | gallery | TRUE | 2nd duplicate check (relevant with high concurrency) * 1 | version should not exist in packages container | gallery | TRUE | 3rd duplicate check (relevant with fast validations) * 1 | commit to DB | gallery | TRUE | * 1 | upload to packages container | async validation | TRUE | * 1 | move package to Available status in DB | async validation | TRUE | * 1 | delete from validation container | async validation | TRUE | * 2 | upload to validation container | gallery | TRUE | * 2 | version should not exist in packages container | gallery | FALSE | * 2 | delete from validation (rollback) | gallery | TRUE | Only occurs in the failure case, as a clean-up. * * Alternatively, we could handle the DB conflict exception that would occur in request 2, but this * would result in an exception that can be avoided and require some ugly code that teases the * unique constraint failure out of a SqlException. * * Another alternative is always leaving the package in the validation container. This is not great * since it doubles the amount of space we need to store packages. Also, it complicates the soft or * hard package delete flow. * * We can safely delete the validation package because we know it's ours. We know this because * saving the validation package succeeded, meaning async validation already successfully moved the * previous package (request 1's package) from the validation container to the package container * and transitioned the package to Available status. * * See the following issue in GitHub for how this case was found: * https://github.com/NuGet/NuGetGallery/issues/5039 */ if (await _packageFileService.DoesPackageFileExistAsync(package)) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); return(PackageCommitResult.Conflict); } } else { await _packageFileService.SavePackageFileAsync(package, packageFile); } } catch (FileAlreadyExistsException ex) { ex.Log(); return(PackageCommitResult.Conflict); } try { // commit all changes to database as an atomic transaction await _entitiesContext.SaveChangesAsync(); } catch { // If saving to the DB fails for any reason we need to delete the package we just saved. if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); } else { await _packageFileService.DeletePackageFileAsync( package.PackageRegistration.Id, package.Version); } throw; } return(PackageCommitResult.Success); }