protected override async Task OnBeforeUpdateDatabaseToMakePackageAvailable(
IValidatingEntity <Package> validatingEntity,
PackageValidationSet validationSet)
{
if (validatingEntity.EntityRecord.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent || validatingEntity.EntityRecord.HasEmbeddedReadme)
{
using (_telemetryService.TrackDurationToExtractLicenseAndReadmeFile(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString()))
using (var packageStream = await _packageFileService.DownloadPackageFileToDiskAsync(validationSet, _sasDefinitionConfiguration.PackageStatusProcessorSasDefinition))
{
if (validatingEntity.EntityRecord.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
_logger.LogInformation("Extracting the license file of type {EmbeddedLicenseFileType} for the package {PackageId} {PackageVersion}",
validatingEntity.EntityRecord.EmbeddedLicenseType,
validationSet.PackageId,
validationSet.PackageNormalizedVersion);
await _coreLicenseFileService.ExtractAndSaveLicenseFileAsync(validatingEntity.EntityRecord, packageStream);
_logger.LogInformation("Successfully extracted the license file.");
}
if (validatingEntity.EntityRecord.HasEmbeddedReadme)
{
_logger.LogInformation("Extracting the readme file of type {EmbeddedReadmeType} for the package {PackageId} {PackageVersion}",
validatingEntity.EntityRecord.EmbeddedReadmeType,
validationSet.PackageId,
validationSet.PackageNormalizedVersion);
await _coreReadmeFileService.ExtractAndSaveReadmeFileAsync(validatingEntity.EntityRecord, packageStream);
_logger.LogInformation("Successfully extracted the readme file.");
}
}
}
}
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile)
{
if (package == null)
{
throw new ArgumentNullException(nameof(package));
}
if (packageFile == null)
{
throw new ArgumentNullException(nameof(packageFile));
}
if (!packageFile.CanSeek)
{
throw new ArgumentException($"{nameof(packageFile)} argument must be seekable stream", nameof(packageFile));
}
await _validationService.UpdatePackageAsync(package);
if (package.PackageStatusKey != PackageStatus.Available &&
package.PackageStatusKey != PackageStatus.Validating)
{
throw new ArgumentException(
$"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.",
nameof(package));
}
try
{
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.SaveValidationPackageFileAsync(package, packageFile);
/* Suppose two package upload requests come in at the same time with the same package (same ID and
* version). It's possible that one request has committed and validated the package AFTER the other
* request has checked that this package does not exist in the database. Observe the following
* sequence of events to understand why the packages container check is necessary.
*
* Request | Step | Component | Success | Notes
* ------- | ---------------------------------------------- | ---------------- | ------- | -----
* 1 | version should not exist in DB | gallery | TRUE | 1st duplicate check (catches most cases over time)
* 2 | version should not exist in DB | gallery | TRUE |
* 1 | upload to validation container | gallery | TRUE | 2nd duplicate check (relevant with high concurrency)
* 1 | version should not exist in packages container | gallery | TRUE | 3rd duplicate check (relevant with fast validations)
* 1 | commit to DB | gallery | TRUE |
* 1 | upload to packages container | async validation | TRUE |
* 1 | move package to Available status in DB | async validation | TRUE |
* 1 | delete from validation container | async validation | TRUE |
* 2 | upload to validation container | gallery | TRUE |
* 2 | version should not exist in packages container | gallery | FALSE |
* 2 | delete from validation (rollback) | gallery | TRUE | Only occurs in the failure case, as a clean-up.
*
* Alternatively, we could handle the DB conflict exception that would occur in request 2, but this
* would result in an exception that can be avoided and require some ugly code that teases the
* unique constraint failure out of a SqlException.
*
* Another alternative is always leaving the package in the validation container. This is not great
* since it doubles the amount of space we need to store packages. Also, it complicates the soft or
* hard package delete flow.
*
* We can safely delete the validation package because we know it's ours. We know this because
* saving the validation package succeeded, meaning async validation already successfully moved the
* previous package (request 1's package) from the validation container to the package container
* and transitioned the package to Available status.
*
* See the following issue in GitHub for how this case was found:
* https://github.com/NuGet/NuGetGallery/issues/5039
*/
if (await _packageFileService.DoesPackageFileExistAsync(package))
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
return(PackageCommitResult.Conflict);
}
}
else
{
if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
// if the package is immediately made available, it means there is a high chance we don't have
// validation pipeline that would normally store the license file, so we'll do it ourselves here.
await _coreLicenseFileService.ExtractAndSaveLicenseFileAsync(package, packageFile);
}
var isReadmeFileExtractedAndSaved = false;
if (package.HasReadMe && package.EmbeddedReadmeType != EmbeddedReadmeFileType.Absent)
{
await _coreReadmeFileService.ExtractAndSaveReadmeFileAsync(package, packageFile);
isReadmeFileExtractedAndSaved = true;
}
try
{
packageFile.Seek(0, SeekOrigin.Begin);
await _packageFileService.SavePackageFileAsync(package, packageFile);
}
catch when(package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent || isReadmeFileExtractedAndSaved)
{
if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
await _coreLicenseFileService.DeleteLicenseFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
}
if (isReadmeFileExtractedAndSaved)
{
await _coreReadmeFileService.DeleteReadmeFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
}
throw;
}
}
}
catch (FileAlreadyExistsException ex)
{
ex.Log();
return(PackageCommitResult.Conflict);
}
try
{
// Sending the validation request after copying to prevent multiple validation requests
// sent when several pushes for the same package happen concurrently. Copying the file
// resolves the race and only one request will "win" and reach this code.
await _validationService.StartValidationAsync(package);
// commit all changes to database as an atomic transaction
await _entitiesContext.SaveChangesAsync();
}
catch (Exception ex)
{
// If sending the validation request or saving to the DB fails for any reason
// we need to delete the package we just saved.
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
else
{
await _packageFileService.DeletePackageFileAsync(
package.PackageRegistration.Id,
package.Version);
await _coreLicenseFileService.DeleteLicenseFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
await _coreReadmeFileService.DeleteReadmeFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
}
if (IsConflict(ex))
{
return(PackageCommitResult.Conflict);
}
else
{
throw;
}
}
return(PackageCommitResult.Success);
}
