private async Task BackupPackageBinaries(IEnumerable <Package> packages)
{
// Backup the package binaries and remove from main storage
foreach (var package in packages)
{
// Backup the package and symbols package from the "validating" container.
await BackupFromValidationsContainerAsync(_packageFileService, package);
await BackupFromValidationsContainerAsync(_symbolPackageFileService, package);
// Backup the package and symbols package from the "packages"/"symbol-packages" containers, respectively.
await BackupFromPackagesContainerAsync(_packageFileService, package);
await BackupFromPackagesContainerAsync(_symbolPackageFileService, package);
var id = package.PackageRegistration.Id;
var version = string.IsNullOrEmpty(package.NormalizedVersion)
? NuGetVersion.Parse(package.Version).ToNormalizedString()
: package.NormalizedVersion;
await _packageFileService.DeletePackageFileAsync(id, version);
// we didn't backup license file before deleting it because it is backed up as part of the package
await _packageFileService.DeleteLicenseFileAsync(id, version);
await _symbolPackageFileService.DeletePackageFileAsync(id, version);
await _packageFileService.DeleteValidationPackageFileAsync(id, version);
await _symbolPackageFileService.DeleteValidationPackageFileAsync(id, version);
// Delete readme file for this package.
await TryDeleteReadMeMdFile(package);
}
}
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile)
{
await _validationService.StartValidationAsync(package);
if (package.PackageStatusKey != PackageStatus.Available &&
package.PackageStatusKey != PackageStatus.Validating)
{
throw new ArgumentException(
$"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.",
nameof(package));
}
try
{
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.SaveValidationPackageFileAsync(package, packageFile);
/* Suppose two package upload requests come in at the same time with the same package (same ID and
* version). It's possible that one request has committed and validated the package AFTER the other
* request has checked that this package does not exist in the database. Observe the following
* sequence of events to understand why the packages container check is necessary.
*
* Request | Step | Component | Success | Notes
* ------- | ---------------------------------------------- | ---------------- | ------- | -----
* 1 | version should not exist in DB | gallery | TRUE | 1st duplicate check (catches most cases over time)
* 2 | version should not exist in DB | gallery | TRUE |
* 1 | upload to validation container | gallery | TRUE | 2nd duplicate check (relevant with high concurrency)
* 1 | version should not exist in packages container | gallery | TRUE | 3rd duplicate check (relevant with fast validations)
* 1 | commit to DB | gallery | TRUE |
* 1 | upload to packages container | async validation | TRUE |
* 1 | move package to Available status in DB | async validation | TRUE |
* 1 | delete from validation container | async validation | TRUE |
* 2 | upload to validation container | gallery | TRUE |
* 2 | version should not exist in packages container | gallery | FALSE |
* 2 | delete from validation (rollback) | gallery | TRUE | Only occurs in the failure case, as a clean-up.
*
* Alternatively, we could handle the DB conflict exception that would occur in request 2, but this
* would result in an exception that can be avoided and require some ugly code that teases the
* unique constraint failure out of a SqlException.
*
* Another alternative is always leaving the package in the validation container. This is not great
* since it doubles the amount of space we need to store packages. Also, it complicates the soft or
* hard package delete flow.
*
* We can safely delete the validation package because we know it's ours. We know this because
* saving the validation package succeeded, meaning async validation already successfully moved the
* previous package (request 1's package) from the validation container to the package container
* and transitioned the package to Available status.
*
* See the following issue in GitHub for how this case was found:
* https://github.com/NuGet/NuGetGallery/issues/5039
*/
if (await _packageFileService.DoesPackageFileExistAsync(package))
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
return(PackageCommitResult.Conflict);
}
}
else
{
if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
// if the package is immediately made available, it means there is a high chance we don't have
// validation pipeline that would normally store the license file, so we'll do it ourselves here.
await SavePackageLicenseFile(packageFile, package);
}
try
{
await _packageFileService.SavePackageFileAsync(package, packageFile);
}
catch when(package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
await _packageFileService.DeleteLicenseFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
throw;
}
}
}
catch (FileAlreadyExistsException ex)
{
ex.Log();
return(PackageCommitResult.Conflict);
}
try
{
// commit all changes to database as an atomic transaction
await _entitiesContext.SaveChangesAsync();
}
catch (Exception ex)
{
// If saving to the DB fails for any reason we need to delete the package we just saved.
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
else
{
await _packageFileService.DeletePackageFileAsync(
package.PackageRegistration.Id,
package.Version);
await _packageFileService.DeleteLicenseFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
}
return(ReturnConflictOrThrow(ex));
}
return(PackageCommitResult.Success);
}