public virtual async Task <TEntity> AddAsync(TEntity t)
{
_dbContext.Set <TEntity>().Add(t);
await _dbContext.SaveChangesAsync();
return(t);
}
public virtual async Task <ActionResult> UndoPendingEdits(string id, string version)
{
var package = _packageService.FindPackageByIdAndVersion(id, version);
if (package == null)
{
return(HttpNotFound());
}
if (!package.IsOwner(User))
{
return(new HttpStatusCodeResult(403, "Forbidden"));
}
// To do as much successful cancellation as possible, Will not batch, but will instead try to cancel
// pending edits 1 at a time, starting with oldest first.
var pendingEdits = _entitiesContext.Set <PackageEdit>()
.Where(pe => pe.PackageKey == package.Key)
.OrderBy(pe => pe.Timestamp)
.ToList();
int numOK = 0;
int numConflicts = 0;
foreach (var result in pendingEdits)
{
try
{
_entitiesContext.DeleteOnCommit(result);
await _entitiesContext.SaveChangesAsync();
numOK += 1;
}
catch (DataException)
{
numConflicts += 1;
}
}
if (numConflicts > 0)
{
TempData["Message"] = "Your pending edit has already been completed and could not be canceled.";
}
else if (numOK > 0)
{
TempData["Message"] = "Your pending edits for this package were successfully canceled.";
}
else
{
TempData["Message"] = "No pending edits were found for this package. The edits may have already been completed.";
}
return(Redirect(Url.Package(id, version)));
}
public async Task AddPackageOwnerAsync(PackageRegistration packageRegistration, User user)
{
if (packageRegistration == null)
{
throw new ArgumentNullException(nameof(packageRegistration));
}
if (user == null)
{
throw new ArgumentNullException(nameof(user));
}
using (var strategy = new SuspendDbExecutionStrategy())
using (var transaction = _entitiesContext.GetDatabase().BeginTransaction())
{
Func <ReservedNamespace, bool> predicate =
reservedNamespace => reservedNamespace.IsPrefix
? packageRegistration.Id.StartsWith(reservedNamespace.Value, StringComparison.OrdinalIgnoreCase)
: packageRegistration.Id.Equals(reservedNamespace.Value, StringComparison.OrdinalIgnoreCase);
var userOwnedMatchingNamespacesForId = user
.ReservedNamespaces
.Where(predicate);
if (userOwnedMatchingNamespacesForId.Any())
{
if (!packageRegistration.IsVerified)
{
await _packageService.UpdatePackageVerifiedStatusAsync(new List <PackageRegistration> {
packageRegistration
}, isVerified : true);
}
userOwnedMatchingNamespacesForId
.ToList()
.ForEach(mn =>
_reservedNamespaceService.AddPackageRegistrationToNamespace(mn.Value, packageRegistration));
// The 'AddPackageRegistrationToNamespace' does not commit its changes, so saving changes for consistency.
await _entitiesContext.SaveChangesAsync();
}
await _packageService.AddPackageOwnerAsync(packageRegistration, user);
await DeletePackageOwnershipRequestAsync(packageRegistration, user);
transaction.Commit();
}
await _auditingService.SaveAuditRecordAsync(
new PackageRegistrationAuditRecord(packageRegistration, AuditedPackageRegistrationAction.AddOwner, user.Username));
}
/// <summary>
/// Save a pending ReadMe if changes are detected.
/// </summary>
/// <param name="package">Package entity associated with the ReadMe.</param>
/// <param name="edit">Package version edit readme request.</param>
/// <param name="encoding">The encoding used when reading the existing readme.</param>
/// <param name="commitChanges">Whether or not to commit the pending changes to the database.</param>
/// <returns>True if the package readme changed, otherwise false.</returns>
public async Task <bool> SaveReadMeMdIfChanged(
Package package,
EditPackageVersionReadMeRequest edit,
Encoding encoding,
bool commitChanges)
{
var activeReadMe = package.HasReadMe ?
NormalizeNewLines(await GetReadMeMdAsync(package)) :
null;
var newReadMe = HasReadMeSource(edit?.ReadMe) ?
NormalizeNewLines(await GetReadMeMdAsync(edit.ReadMe, encoding)) :
null;
var hasReadMe = !string.IsNullOrWhiteSpace(newReadMe);
if (hasReadMe && !newReadMe.Equals(activeReadMe))
{
await _packageFileService.SaveReadMeMdFileAsync(package, newReadMe);
edit.ReadMeState = PackageEditReadMeState.Changed;
// Save entity to db.
package.HasReadMe = true;
if (commitChanges)
{
await _entitiesContext.SaveChangesAsync();
}
}
else if (!hasReadMe && !string.IsNullOrEmpty(activeReadMe))
{
await _packageFileService.DeleteReadMeMdFileAsync(package);
edit.ReadMeState = PackageEditReadMeState.Deleted;
// Save entity to db.
package.HasReadMe = false;
if (commitChanges)
{
await _entitiesContext.SaveChangesAsync();
}
}
else
{
edit.ReadMeState = PackageEditReadMeState.Unchanged;
}
return(edit.ReadMeState != PackageEditReadMeState.Unchanged);
}
public async Task MarkPackageListedAsync(Package package, bool commitChanges = true, bool updateIndex = true)
{
if (package == null)
{
throw new ArgumentNullException(nameof(package));
}
if (package.Listed)
{
return;
}
if (package.PackageStatusKey == PackageStatus.Deleted)
{
throw new InvalidOperationException("A deleted package should never be listed!");
}
if (package.PackageStatusKey == PackageStatus.FailedValidation)
{
throw new InvalidOperationException("A package that failed validation should never be listed!");
}
if (!package.Listed && (package.IsLatestStable || package.IsLatest || package.IsLatestSemVer2 || package.IsLatestStableSemVer2))
{
throw new InvalidOperationException("An unlisted package should never be latest or latest stable!");
}
package.Listed = true;
package.LastUpdated = DateTime.UtcNow;
// NOTE: LastEdited will be overwritten by a trigger defined in the migration named "AddTriggerForPackagesLastEdited".
package.LastEdited = DateTime.UtcNow;
await _packageService.UpdateIsLatestAsync(package.PackageRegistration, commitChanges: false);
await _auditingService.SaveAuditRecordAsync(new PackageAuditRecord(package, AuditedPackageAction.List));
_telemetryService.TrackPackageListed(package);
if (commitChanges)
{
await _entitiesContext.SaveChangesAsync();
}
if (updateIndex)
{
_indexingService.UpdatePackage(package);
}
}
public async Task <Package> ReflowAsync(string id, string version)
{
var package = _packageService.FindPackageByIdAndVersionStrict(id, version);
if (package == null)
{
return(null);
}
EntitiesConfiguration.SuspendExecutionStrategy = true;
using (var transaction = _entitiesContext.GetDatabase().BeginTransaction())
{
// 1) Download package binary to memory
using (var packageStream = await _packageFileService.DownloadPackageFileAsync(package))
{
using (var packageArchive = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
{
// 2) Determine package metadata from binary
var packageStreamMetadata = new PackageStreamMetadata
{
HashAlgorithm = Constants.Sha512HashAlgorithmId,
Hash = CryptographyService.GenerateHash(packageStream.AsSeekableStream()),
Size = packageStream.Length,
};
var packageMetadata = PackageMetadata.FromNuspecReader(packageArchive.GetNuspecReader());
// 3) Clear referenced objects that will be reflowed
ClearSupportedFrameworks(package);
ClearAuthors(package);
ClearDependencies(package);
// 4) Reflow the package
var listed = package.Listed;
package = _packageService.EnrichPackageFromNuGetPackage(
package,
packageArchive,
packageMetadata,
packageStreamMetadata,
package.User);
package.LastEdited = DateTime.UtcNow;
package.Listed = listed;
// 5) Update IsLatest so that reflow can correct concurrent updates (see Gallery #2514)
await _packageService.UpdateIsLatestAsync(package.PackageRegistration, commitChanges : false);
// 6) Save and profit
await _entitiesContext.SaveChangesAsync();
}
}
// Commit transaction
transaction.Commit();
}
EntitiesConfiguration.SuspendExecutionStrategy = false;
return(package);
}
private async Task RemoveMemberships(User user, User requestingUser, AccountDeletionOrphanPackagePolicy orphanPackagePolicy)
{
foreach (var membership in user.Organizations.ToArray())
{
var organization = membership.Organization;
var members = organization.Members.ToList();
var collaborators = members.Where(m => !m.IsAdmin).ToList();
var memberCount = members.Count();
user.Organizations.Remove(membership);
if (memberCount < 2)
{
// The user we are deleting is the only member of the organization.
// We should delete the entire organization.
await DeleteAccountImplAsync(organization, requestingUser, orphanPackagePolicy);
}
else if (memberCount - 1 <= collaborators.Count())
{
// All other members of this organization are collaborators, so we should promote them to administrators.
foreach (var collaborator in collaborators)
{
collaborator.IsAdmin = true;
}
}
}
foreach (var membershipRequest in user.OrganizationRequests.ToArray())
{
user.OrganizationRequests.Remove(membershipRequest);
}
foreach (var transformationRequest in user.OrganizationMigrationRequests.ToArray())
{
user.OrganizationMigrationRequests.Remove(transformationRequest);
transformationRequest.NewOrganization.OrganizationMigrationRequest = null;
}
var migrationRequest = user.OrganizationMigrationRequest;
user.OrganizationMigrationRequest = null;
if (migrationRequest != null)
{
migrationRequest.AdminUser.OrganizationMigrationRequests.Remove(migrationRequest);
}
await _entitiesContext.SaveChangesAsync();
}
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile)
{
if (package.PackageStatusKey != PackageStatus.Available &&
package.PackageStatusKey != PackageStatus.Validating)
{
throw new ArgumentException(
$"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.",
nameof(package));
}
try
{
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.SaveValidationPackageFileAsync(package, packageFile);
}
else
{
await _packageFileService.SavePackageFileAsync(package, packageFile);
}
}
catch (InvalidOperationException ex)
{
ex.Log();
return(PackageCommitResult.Conflict);
}
try
{
// commit all changes to database as an atomic transaction
await _entitiesContext.SaveChangesAsync();
}
catch
{
// If saving to the DB fails for any reason we need to delete the package we just saved.
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
else
{
await _packageFileService.DeletePackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
throw;
}
return(PackageCommitResult.Success);
}
public async Task ActivateCertificateAsync(string thumbprint, User account)
{
if (string.IsNullOrEmpty(thumbprint))
{
throw new ArgumentException(Strings.ArgumentCannotBeNullOrEmpty, nameof(thumbprint));
}
if (account == null)
{
throw new ArgumentNullException(nameof(account));
}
var certificate = GetCertificate(thumbprint);
if (certificate == null)
{
throw new ArgumentException(Strings.CertificateDoesNotExist, nameof(thumbprint));
}
var userCertificate = certificate.UserCertificates.SingleOrDefault(uc => uc.UserKey == account.Key);
if (userCertificate == null)
{
userCertificate = new UserCertificate()
{
CertificateKey = certificate.Key,
UserKey = account.Key
};
_entitiesContext.UserCertificates.Add(userCertificate);
await _entitiesContext.SaveChangesAsync();
await _auditingService.SaveAuditRecordAsync(
new CertificateAuditRecord(AuditedCertificateAction.Activate, certificate.Thumbprint));
_telemetryService.TrackCertificateActivated(thumbprint);
}
}
private async Task AddPackageOwnerTask(PackageRegistration packageRegistration, User user, bool commitChanges = true)
{
Func <ReservedNamespace, bool> predicate =
reservedNamespace => reservedNamespace.IsPrefix
? packageRegistration.Id.StartsWith(reservedNamespace.Value, StringComparison.OrdinalIgnoreCase)
: packageRegistration.Id.Equals(reservedNamespace.Value, StringComparison.OrdinalIgnoreCase);
var userOwnedMatchingNamespacesForId = user
.ReservedNamespaces
.Where(predicate);
if (userOwnedMatchingNamespacesForId.Any())
{
if (!packageRegistration.IsVerified)
{
await _packageService.UpdatePackageVerifiedStatusAsync(new List <PackageRegistration> {
packageRegistration
}, isVerified : true, commitChanges : commitChanges);
}
userOwnedMatchingNamespacesForId
.ToList()
.ForEach(mn =>
_reservedNamespaceService.AddPackageRegistrationToNamespace(mn.Value, packageRegistration));
if (commitChanges)
{
// The 'AddPackageRegistrationToNamespace' does not commit its changes, so saving changes for consistency.
await _entitiesContext.SaveChangesAsync();
}
}
await _packageService.AddPackageOwnerAsync(packageRegistration, user, commitChanges);
await DeletePackageOwnershipRequestAsync(packageRegistration, user, commitChanges);
}
private async Task DeleteAccountImplAsync(User userToBeDeleted, User userToExecuteTheDelete, AccountDeletionOrphanPackagePolicy orphanPackagePolicy, bool commitChanges = true)
{
await RemoveReservedNamespaces(userToBeDeleted);
await RemovePackageOwnership(userToBeDeleted, userToExecuteTheDelete, orphanPackagePolicy);
await RemoveMemberships(userToBeDeleted, userToExecuteTheDelete, orphanPackagePolicy);
await RemoveSecurityPolicies(userToBeDeleted);
await RemoveUserCredentials(userToBeDeleted);
await RemovePackageOwnershipRequests(userToBeDeleted);
ResetPackagesAndAccountsDeletedBy(userToBeDeleted);
RemovePackagePushedBy(userToBeDeleted);
RemovePackageDeprecatedBy(userToBeDeleted);
var organizationToBeDeleted = userToBeDeleted as Organization;
if (organizationToBeDeleted != null)
{
RemoveMembers(organizationToBeDeleted);
}
if (!userToBeDeleted.Confirmed)
{
// Unconfirmed users should be hard-deleted.
// Another account with the same username can be created.
RemoveUser(userToBeDeleted);
}
else
{
// Confirmed users should be soft-deleted.
// Another account with the same username cannot be created.
RemoveUserDataInUserTable(userToBeDeleted);
InsertDeleteAccount(
userToBeDeleted,
userToExecuteTheDelete);
}
if (commitChanges)
{
await _entitiesContext.SaveChangesAsync();
}
}
public Task <int> SaveAsync(CancellationToken cancellationToken)
{
return(_dbContext.SaveChangesAsync(cancellationToken));
}
public async Task CommitChangesAsync()
{
await _entities.SaveChangesAsync();
}
public async Task UpdateDeprecation(
IReadOnlyList <Package> packages,
PackageDeprecationStatus status,
PackageRegistration alternatePackageRegistration,
Package alternatePackage,
string customMessage,
User user)
{
if (user == null)
{
throw new ArgumentNullException(nameof(user));
}
if (packages == null || !packages.Any())
{
throw new ArgumentException(nameof(packages));
}
var registration = packages.First().PackageRegistration;
if (packages.Select(p => p.PackageRegistrationKey).Distinct().Count() > 1)
{
throw new ArgumentException("All packages to deprecate must have the same ID.", nameof(packages));
}
using (var strategy = new SuspendDbExecutionStrategy())
using (var transaction = _entitiesContext.GetDatabase().BeginTransaction())
{
var shouldDelete = status == PackageDeprecationStatus.NotDeprecated;
var deprecations = new List <PackageDeprecation>();
foreach (var package in packages)
{
var deprecation = package.Deprecations.SingleOrDefault();
if (shouldDelete)
{
if (deprecation != null)
{
package.Deprecations.Remove(deprecation);
deprecations.Add(deprecation);
}
}
else
{
if (deprecation == null)
{
deprecation = new PackageDeprecation
{
Package = package
};
package.Deprecations.Add(deprecation);
deprecations.Add(deprecation);
}
deprecation.Status = status;
deprecation.DeprecatedByUser = user;
deprecation.AlternatePackageRegistration = alternatePackageRegistration;
deprecation.AlternatePackage = alternatePackage;
deprecation.CustomMessage = customMessage;
}
}
if (shouldDelete)
{
_entitiesContext.Deprecations.RemoveRange(deprecations);
}
else
{
_entitiesContext.Deprecations.AddRange(deprecations);
}
await _entitiesContext.SaveChangesAsync();
await _packageUpdateService.UpdatePackagesAsync(packages);
transaction.Commit();
_telemetryService.TrackPackageDeprecate(
packages,
status,
alternatePackageRegistration,
alternatePackage,
!string.IsNullOrWhiteSpace(customMessage));
foreach (var package in packages)
{
await _auditingService.SaveAuditRecordAsync(
new PackageAuditRecord(
package,
status == PackageDeprecationStatus.NotDeprecated ? AuditedPackageAction.Undeprecate : AuditedPackageAction.Deprecate,
status == PackageDeprecationStatus.NotDeprecated ? PackageUndeprecatedVia.Web : PackageDeprecatedVia.Web));
}
}
}
public async Task <int> CommitChangesAsync()
{
return(await Context.SaveChangesAsync());
}
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile)
{
await _validationService.StartValidationAsync(package);
if (package.PackageStatusKey != PackageStatus.Available &&
package.PackageStatusKey != PackageStatus.Validating)
{
throw new ArgumentException(
$"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.",
nameof(package));
}
try
{
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.SaveValidationPackageFileAsync(package, packageFile);
/* Suppose two package upload requests come in at the same time with the same package (same ID and
* version). It's possible that one request has committed and validated the package AFTER the other
* request has checked that this package does not exist in the database. Observe the following
* sequence of events to understand why the packages container check is necessary.
*
* Request | Step | Component | Success | Notes
* ------- | ---------------------------------------------- | ---------------- | ------- | -----
* 1 | version should not exist in DB | gallery | TRUE | 1st duplicate check (catches most cases over time)
* 2 | version should not exist in DB | gallery | TRUE |
* 1 | upload to validation container | gallery | TRUE | 2nd duplicate check (relevant with high concurrency)
* 1 | version should not exist in packages container | gallery | TRUE | 3rd duplicate check (relevant with fast validations)
* 1 | commit to DB | gallery | TRUE |
* 1 | upload to packages container | async validation | TRUE |
* 1 | move package to Available status in DB | async validation | TRUE |
* 1 | delete from validation container | async validation | TRUE |
* 2 | upload to validation container | gallery | TRUE |
* 2 | version should not exist in packages container | gallery | FALSE |
* 2 | delete from validation (rollback) | gallery | TRUE | Only occurs in the failure case, as a clean-up.
*
* Alternatively, we could handle the DB conflict exception that would occur in request 2, but this
* would result in an exception that can be avoided and require some ugly code that teases the
* unique constraint failure out of a SqlException.
*
* Another alternative is always leaving the package in the validation container. This is not great
* since it doubles the amount of space we need to store packages. Also, it complicates the soft or
* hard package delete flow.
*
* We can safely delete the validation package because we know it's ours. We know this because
* saving the validation package succeeded, meaning async validation already successfully moved the
* previous package (request 1's package) from the validation container to the package container
* and transitioned the package to Available status.
*
* See the following issue in GitHub for how this case was found:
* https://github.com/NuGet/NuGetGallery/issues/5039
*/
if (await _packageFileService.DoesPackageFileExistAsync(package))
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
return(PackageCommitResult.Conflict);
}
}
else
{
await _packageFileService.SavePackageFileAsync(package, packageFile);
}
}
catch (FileAlreadyExistsException ex)
{
ex.Log();
return(PackageCommitResult.Conflict);
}
try
{
// commit all changes to database as an atomic transaction
await _entitiesContext.SaveChangesAsync();
}
catch
{
// If saving to the DB fails for any reason we need to delete the package we just saved.
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
else
{
await _packageFileService.DeletePackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
throw;
}
return(PackageCommitResult.Success);
}
public Task <int> SaveChangesAsync()
{
return(_context.SaveChangesAsync());
}
/// <summary>
/// This method creates the symbol db entities and invokes the validations for the uploaded snupkg.
/// It will send the message for validation and upload the snupkg to the "validations"/"symbols-packages" container
/// based on the result. It will then update the references in the database for persistence with appropriate status.
/// </summary>
/// <param name="package">The package for which symbols package is to be uplloaded</param>
/// <param name="packageStreamMetadata">The package stream metadata for the uploaded symbols package file.</param>
/// <param name="symbolPackageFile">The symbol package file stream.</param>
/// <returns>The <see cref="PackageCommitResult"/> for the symbol package upload flow.</returns>
public async Task <PackageCommitResult> CreateAndUploadSymbolsPackage(Package package, PackageStreamMetadata packageStreamMetadata, Stream symbolPackageFile)
{
var symbolPackage = _symbolPackageService.CreateSymbolPackage(package, packageStreamMetadata);
// TODO: Add Validations for symbols, for now set the status to Available. https://github.com/NuGet/NuGetGallery/issues/6235
// Add validating type to be symbols when sending message to the orchestrator.
symbolPackage.StatusKey = PackageStatus.Available;
symbolPackage.Published = DateTime.UtcNow;
if (symbolPackage.StatusKey != PackageStatus.Available &&
symbolPackage.StatusKey != PackageStatus.Validating)
{
throw new InvalidOperationException(
$"The symbol package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.");
}
try
{
if (symbolPackage.StatusKey == PackageStatus.Validating)
{
await _symbolPackageFileService.SaveValidationPackageFileAsync(symbolPackage.Package, symbolPackageFile);
}
else if (symbolPackage.StatusKey == PackageStatus.Available)
{
// Mark any other associated available symbol packages for deletion.
var availableSymbolPackages = package
.SymbolPackages
.Where(sp => sp.StatusKey == PackageStatus.Available &&
sp != symbolPackage);
var overwrite = false;
if (availableSymbolPackages.Any())
{
// Mark the currently available packages for deletion, and replace the file in the container.
foreach (var availableSymbolPackage in availableSymbolPackages)
{
availableSymbolPackage.StatusKey = PackageStatus.Deleted;
}
overwrite = true;
}
// Caveat: This doesn't really affect our prod flow since the package is validating, however, when the async validation
// is disabled there is a chance that there could be concurrency issues when pushing multiple symbols simultaneously.
// This could result in an inconsistent data or multiple symbol entities marked as available. This could be sovled using etag
// for saving files, however since it doesn't really affect nuget.org which happen have async validations flow I will leave it as is.
await _symbolPackageFileService.SavePackageFileAsync(symbolPackage.Package, symbolPackageFile, overwrite);
}
try
{
// commit all changes to database as an atomic transaction
await _entitiesContext.SaveChangesAsync();
}
catch (Exception ex)
{
ex.Log();
// If saving to the DB fails for any reason we need to delete the package we just saved.
if (symbolPackage.StatusKey == PackageStatus.Validating)
{
await _symbolPackageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
else if (symbolPackage.StatusKey == PackageStatus.Available)
{
await _symbolPackageFileService.DeletePackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
throw ex;
}
}
catch (FileAlreadyExistsException ex)
{
ex.Log();
return(PackageCommitResult.Conflict);
}
return(PackageCommitResult.Success);
}
/// <summary>
/// This method creates the symbol db entities and invokes the validations for the uploaded snupkg.
/// It will send the message for validation and upload the snupkg to the "validations"/"symbols-packages" container
/// based on the result. It will then update the references in the database for persistence with appropriate status.
/// </summary>
/// <param name="package">The package for which symbols package is to be uplloaded</param>
/// <param name="symbolPackageStream">The symbols package stream metadata for the uploaded symbols package file.</param>
/// <returns>The <see cref="PackageCommitResult"/> for the create and upload symbol package flow.</returns>
public async Task <PackageCommitResult> CreateAndUploadSymbolsPackage(Package package, Stream symbolPackageStream)
{
var packageStreamMetadata = new PackageStreamMetadata
{
HashAlgorithm = CoreConstants.Sha512HashAlgorithmId,
Hash = CryptographyService.GenerateHash(
symbolPackageStream.AsSeekableStream(),
CoreConstants.Sha512HashAlgorithmId),
Size = symbolPackageStream.Length
};
Stream symbolPackageFile = symbolPackageStream.AsSeekableStream();
var symbolPackage = _symbolPackageService.CreateSymbolPackage(package, packageStreamMetadata);
await _validationService.StartValidationAsync(symbolPackage);
if (symbolPackage.StatusKey != PackageStatus.Available &&
symbolPackage.StatusKey != PackageStatus.Validating)
{
throw new InvalidOperationException(
$"The symbol package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.");
}
try
{
if (symbolPackage.StatusKey == PackageStatus.Validating)
{
await _symbolPackageFileService.SaveValidationPackageFileAsync(symbolPackage.Package, symbolPackageFile);
}
else if (symbolPackage.StatusKey == PackageStatus.Available)
{
if (!symbolPackage.Published.HasValue)
{
symbolPackage.Published = DateTime.UtcNow;
}
// Mark any other associated available symbol packages for deletion.
var availableSymbolPackages = package
.SymbolPackages
.Where(sp => sp.StatusKey == PackageStatus.Available &&
sp != symbolPackage);
var overwrite = false;
if (availableSymbolPackages.Any())
{
// Mark the currently available packages for deletion, and replace the file in the container.
foreach (var availableSymbolPackage in availableSymbolPackages)
{
availableSymbolPackage.StatusKey = PackageStatus.Deleted;
}
overwrite = true;
}
// Caveat: This doesn't really affect our prod flow since the package is validating, however, when the async validation
// is disabled there is a chance that there could be concurrency issues when pushing multiple symbols simultaneously.
// This could result in an inconsistent data or multiple symbol entities marked as available. This could be sovled using etag
// for saving files, however since it doesn't really affect nuget.org which happen have async validations flow I will leave it as is.
await _symbolPackageFileService.SavePackageFileAsync(symbolPackage.Package, symbolPackageFile, overwrite);
}
try
{
// commit all changes to database as an atomic transaction
await _entitiesContext.SaveChangesAsync();
}
catch (Exception ex)
{
ex.Log();
// If saving to the DB fails for any reason we need to delete the package we just saved.
if (symbolPackage.StatusKey == PackageStatus.Validating)
{
await _symbolPackageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
else if (symbolPackage.StatusKey == PackageStatus.Available)
{
await _symbolPackageFileService.DeletePackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
throw ex;
}
}
catch (FileAlreadyExistsException ex)
{
ex.Log();
return(PackageCommitResult.Conflict);
}
_telemetryService.TrackSymbolPackagePushEvent(package.Id, package.NormalizedVersion);
return(PackageCommitResult.Success);
}
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile)
{
if (package == null)
{
throw new ArgumentNullException(nameof(package));
}
if (packageFile == null)
{
throw new ArgumentNullException(nameof(packageFile));
}
if (!packageFile.CanSeek)
{
throw new ArgumentException($"{nameof(packageFile)} argument must be seekable stream", nameof(packageFile));
}
await _validationService.UpdatePackageAsync(package);
if (package.PackageStatusKey != PackageStatus.Available &&
package.PackageStatusKey != PackageStatus.Validating)
{
throw new ArgumentException(
$"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.",
nameof(package));
}
try
{
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.SaveValidationPackageFileAsync(package, packageFile);
/* Suppose two package upload requests come in at the same time with the same package (same ID and
* version). It's possible that one request has committed and validated the package AFTER the other
* request has checked that this package does not exist in the database. Observe the following
* sequence of events to understand why the packages container check is necessary.
*
* Request | Step | Component | Success | Notes
* ------- | ---------------------------------------------- | ---------------- | ------- | -----
* 1 | version should not exist in DB | gallery | TRUE | 1st duplicate check (catches most cases over time)
* 2 | version should not exist in DB | gallery | TRUE |
* 1 | upload to validation container | gallery | TRUE | 2nd duplicate check (relevant with high concurrency)
* 1 | version should not exist in packages container | gallery | TRUE | 3rd duplicate check (relevant with fast validations)
* 1 | commit to DB | gallery | TRUE |
* 1 | upload to packages container | async validation | TRUE |
* 1 | move package to Available status in DB | async validation | TRUE |
* 1 | delete from validation container | async validation | TRUE |
* 2 | upload to validation container | gallery | TRUE |
* 2 | version should not exist in packages container | gallery | FALSE |
* 2 | delete from validation (rollback) | gallery | TRUE | Only occurs in the failure case, as a clean-up.
*
* Alternatively, we could handle the DB conflict exception that would occur in request 2, but this
* would result in an exception that can be avoided and require some ugly code that teases the
* unique constraint failure out of a SqlException.
*
* Another alternative is always leaving the package in the validation container. This is not great
* since it doubles the amount of space we need to store packages. Also, it complicates the soft or
* hard package delete flow.
*
* We can safely delete the validation package because we know it's ours. We know this because
* saving the validation package succeeded, meaning async validation already successfully moved the
* previous package (request 1's package) from the validation container to the package container
* and transitioned the package to Available status.
*
* See the following issue in GitHub for how this case was found:
* https://github.com/NuGet/NuGetGallery/issues/5039
*/
if (await _packageFileService.DoesPackageFileExistAsync(package))
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
return(PackageCommitResult.Conflict);
}
}
else
{
if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
// if the package is immediately made available, it means there is a high chance we don't have
// validation pipeline that would normally store the license file, so we'll do it ourselves here.
await _coreLicenseFileService.ExtractAndSaveLicenseFileAsync(package, packageFile);
}
var isReadmeFileExtractedAndSaved = false;
if (package.HasReadMe && package.EmbeddedReadmeType != EmbeddedReadmeFileType.Absent)
{
await _packageFileService.ExtractAndSaveReadmeFileAsync(package, packageFile);
isReadmeFileExtractedAndSaved = true;
}
try
{
packageFile.Seek(0, SeekOrigin.Begin);
await _packageFileService.SavePackageFileAsync(package, packageFile);
}
catch when(package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent || isReadmeFileExtractedAndSaved)
{
if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
await _coreLicenseFileService.DeleteLicenseFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
}
if (isReadmeFileExtractedAndSaved)
{
await _packageFileService.DeleteReadMeMdFileAsync(package);
}
throw;
}
}
}
catch (FileAlreadyExistsException ex)
{
ex.Log();
return(PackageCommitResult.Conflict);
}
try
{
// Sending the validation request after copying to prevent multiple validation requests
// sent when several pushes for the same package happen concurrently. Copying the file
// resolves the race and only one request will "win" and reach this code.
await _validationService.StartValidationAsync(package);
// commit all changes to database as an atomic transaction
await _entitiesContext.SaveChangesAsync();
}
catch (Exception ex)
{
// If sending the validation request or saving to the DB fails for any reason
// we need to delete the package we just saved.
if (package.PackageStatusKey == PackageStatus.Validating)
{
await _packageFileService.DeleteValidationPackageFileAsync(
package.PackageRegistration.Id,
package.Version);
}
else
{
await _packageFileService.DeletePackageFileAsync(
package.PackageRegistration.Id,
package.Version);
await _coreLicenseFileService.DeleteLicenseFileAsync(
package.PackageRegistration.Id,
package.NormalizedVersion);
await _packageFileService.DeleteReadMeMdFileAsync(package);
}
return(ReturnConflictOrThrow(ex));
}
return(PackageCommitResult.Success);
}
public void CommitAsync()
{
_context.SaveChangesAsync();
}
