protected override async Task OnCleanupAfterDatabaseUpdateFailure(
IValidatingEntity <Package> validatingEntity,
PackageValidationSet validationSet)
{
if (validatingEntity.EntityRecord.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
using (_telemetryService.TrackDurationToDeleteLicenseFile(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString()))
{
_logger.LogInformation("Cleaning up the license file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion);
await _coreLicenseFileService.DeleteLicenseFileAsync(validationSet.PackageId, validationSet.PackageNormalizedVersion);
_logger.LogInformation("Deleted the license file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion);
}
}
if (validatingEntity.EntityRecord.HasEmbeddedReadme)
{
using (_telemetryService.TrackDurationToDeleteReadmeFile(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString()))
{
_logger.LogInformation("Cleaning up the readme file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion);
await _coreReadmeFileService.DeleteReadmeFileAsync(validationSet.PackageId, validationSet.PackageNormalizedVersion);
_logger.LogInformation("Deleted the readme file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion);
}
}
}
/// <summary>
/// Delete package readme.md file, if it exists.
/// </summary>
private async Task TryDeleteReadMeMdFile(Package package)
{
try
{
if (package.HasEmbeddedReadme)
{
await _coreReadmeFileService.DeleteReadmeFileAsync(package.Id, package.Version);
}
else
{
await _packageFileService.DeleteReadMeMdFileAsync(package);
}
}
catch (StorageException) { }
}
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile)
{
if (package == null)
{
throw new ArgumentNullException(nameof(package));
}
if (packageFile == null)
{
throw new ArgumentNullException(nameof(packageFile));
}
if (!packageFile.CanSeek)
{
throw new ArgumentException($"{nameof(packageFile)} argument must be seekable stream", nameof(packageFile));
}
await _validationService.UpdatePackageAsync(package);
if (package.PackageStatusKey != PackageStatus.Available &&
package.PackageStatusKey != PackageStatus.Validating)
{
throw new ArgumentException(
$"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.",
nameof(package));
}
try
{
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.SaveValidationPackageFileAsync(package, packageFile);
/* Suppose two package upload requests come in at the same time with the same package (same ID and
* version). It's possible that one request has committed and validated the package AFTER the other
* request has checked that this package does not exist in the database. Observe the following
* sequence of events to understand why the packages container check is necessary.
*
* Request | Step | Component | Success | Notes
* ------- | ---------------------------------------------- | ---------------- | ------- | -----
* 1 | version should not exist in DB | gallery | TRUE | 1st duplicate check (catches most cases over time)
* 2 | version should not exist in DB | gallery | TRUE |
* 1 | upload to validation container | gallery | TRUE | 2nd duplicate check (relevant with high concurrency)
* 1 | version should not exist in packages container | gallery | TRUE | 3rd duplicate check (relevant with fast validations)
* 1 | commit to DB | gallery | TRUE |
* 1 | upload to packages container | async validation | TRUE |
* 1 | move package to Available status in DB | async validation | TRUE |
* 1 | delete from validation container | async validation | TRUE |
* 2 | upload to validation container | gallery | TRUE |
* 2 | version should not exist in packages container | gallery | FALSE |
* 2 | delete from validation (rollback) | gallery | TRUE | Only occurs in the failure case, as a clean-up.
*
* Alternatively, we could handle the DB conflict exception that would occur in request 2, but this
* would result in an exception that can be avoided and require some ugly code that teases the
* unique constraint failure out of a SqlException.
*
* Another alternative is always leaving the package in the validation container. This is not great
* since it doubles the amount of space we need to store packages. Also, it complicates the soft or
* hard package delete flow.
*
* We can safely delete the validation package because we know it's ours. We know this because
* saving the validation package succeeded, meaning async validation already successfully moved the
* previous package (request 1's package) from the validation container to the package container
* and transitioned the package to Available status.
*
* See the following issue in GitHub for how this case was found:
* https://github.com/NuGet/NuGetGallery/issues/5039
*/
if (await _packageFileService.DoesPackageFileExistAsync(package))
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
return(PackageCommitResult.Conflict);
}
}
else
{
if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
// if the package is immediately made available, it means there is a high chance we don't have
// validation pipeline that would normally store the license file, so we'll do it ourselves here.
await _coreLicenseFileService.ExtractAndSaveLicenseFileAsync(package, packageFile);
}
var isReadmeFileExtractedAndSaved = false;
if (package.HasReadMe && package.EmbeddedReadmeType != EmbeddedReadmeFileType.Absent)
{
await _coreReadmeFileService.ExtractAndSaveReadmeFileAsync(package, packageFile);
isReadmeFileExtractedAndSaved = true;
}
try
{
packageFile.Seek(0, SeekOrigin.Begin);
await _packageFileService.SavePackageFileAsync(package, packageFile);
}
catch when(package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent || isReadmeFileExtractedAndSaved)
{
if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
await _coreLicenseFileService.DeleteLicenseFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
}
if (isReadmeFileExtractedAndSaved)
{
await _coreReadmeFileService.DeleteReadmeFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
}
throw;
}
}
}
catch (FileAlreadyExistsException ex)
{
ex.Log();
return(PackageCommitResult.Conflict);
}
try
{
// Sending the validation request after copying to prevent multiple validation requests
// sent when several pushes for the same package happen concurrently. Copying the file
// resolves the race and only one request will "win" and reach this code.
await _validationService.StartValidationAsync(package);
// commit all changes to database as an atomic transaction
await _entitiesContext.SaveChangesAsync();
}
catch (Exception ex)
{
// If sending the validation request or saving to the DB fails for any reason
// we need to delete the package we just saved.
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
else
{
await _packageFileService.DeletePackageFileAsync(
package.PackageRegistration.Id,
package.Version);
await _coreLicenseFileService.DeleteLicenseFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
await _coreReadmeFileService.DeleteReadmeFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
}
if (IsConflict(ex))
{
return(PackageCommitResult.Conflict);
}
else
{
throw;
}
}
return(PackageCommitResult.Success);
}