public virtual async Task <TEntity> AddAsync(TEntity t) { _dbContext.Set <TEntity>().Add(t); await _dbContext.SaveChangesAsync(); return(t); }
public virtual async Task <ActionResult> UndoPendingEdits(string id, string version) { var package = _packageService.FindPackageByIdAndVersion(id, version); if (package == null) { return(HttpNotFound()); } if (!package.IsOwner(User)) { return(new HttpStatusCodeResult(403, "Forbidden")); } // To do as much successful cancellation as possible, Will not batch, but will instead try to cancel // pending edits 1 at a time, starting with oldest first. var pendingEdits = _entitiesContext.Set <PackageEdit>() .Where(pe => pe.PackageKey == package.Key) .OrderBy(pe => pe.Timestamp) .ToList(); int numOK = 0; int numConflicts = 0; foreach (var result in pendingEdits) { try { _entitiesContext.DeleteOnCommit(result); await _entitiesContext.SaveChangesAsync(); numOK += 1; } catch (DataException) { numConflicts += 1; } } if (numConflicts > 0) { TempData["Message"] = "Your pending edit has already been completed and could not be canceled."; } else if (numOK > 0) { TempData["Message"] = "Your pending edits for this package were successfully canceled."; } else { TempData["Message"] = "No pending edits were found for this package. The edits may have already been completed."; } return(Redirect(Url.Package(id, version))); }
public async Task AddPackageOwnerAsync(PackageRegistration packageRegistration, User user) { if (packageRegistration == null) { throw new ArgumentNullException(nameof(packageRegistration)); } if (user == null) { throw new ArgumentNullException(nameof(user)); } using (var strategy = new SuspendDbExecutionStrategy()) using (var transaction = _entitiesContext.GetDatabase().BeginTransaction()) { Func <ReservedNamespace, bool> predicate = reservedNamespace => reservedNamespace.IsPrefix ? packageRegistration.Id.StartsWith(reservedNamespace.Value, StringComparison.OrdinalIgnoreCase) : packageRegistration.Id.Equals(reservedNamespace.Value, StringComparison.OrdinalIgnoreCase); var userOwnedMatchingNamespacesForId = user .ReservedNamespaces .Where(predicate); if (userOwnedMatchingNamespacesForId.Any()) { if (!packageRegistration.IsVerified) { await _packageService.UpdatePackageVerifiedStatusAsync(new List <PackageRegistration> { packageRegistration }, isVerified : true); } userOwnedMatchingNamespacesForId .ToList() .ForEach(mn => _reservedNamespaceService.AddPackageRegistrationToNamespace(mn.Value, packageRegistration)); // The 'AddPackageRegistrationToNamespace' does not commit its changes, so saving changes for consistency. await _entitiesContext.SaveChangesAsync(); } await _packageService.AddPackageOwnerAsync(packageRegistration, user); await DeletePackageOwnershipRequestAsync(packageRegistration, user); transaction.Commit(); } await _auditingService.SaveAuditRecordAsync( new PackageRegistrationAuditRecord(packageRegistration, AuditedPackageRegistrationAction.AddOwner, user.Username)); }
/// <summary> /// Save a pending ReadMe if changes are detected. /// </summary> /// <param name="package">Package entity associated with the ReadMe.</param> /// <param name="edit">Package version edit readme request.</param> /// <param name="encoding">The encoding used when reading the existing readme.</param> /// <param name="commitChanges">Whether or not to commit the pending changes to the database.</param> /// <returns>True if the package readme changed, otherwise false.</returns> public async Task <bool> SaveReadMeMdIfChanged( Package package, EditPackageVersionReadMeRequest edit, Encoding encoding, bool commitChanges) { var activeReadMe = package.HasReadMe ? NormalizeNewLines(await GetReadMeMdAsync(package)) : null; var newReadMe = HasReadMeSource(edit?.ReadMe) ? NormalizeNewLines(await GetReadMeMdAsync(edit.ReadMe, encoding)) : null; var hasReadMe = !string.IsNullOrWhiteSpace(newReadMe); if (hasReadMe && !newReadMe.Equals(activeReadMe)) { await _packageFileService.SaveReadMeMdFileAsync(package, newReadMe); edit.ReadMeState = PackageEditReadMeState.Changed; // Save entity to db. package.HasReadMe = true; if (commitChanges) { await _entitiesContext.SaveChangesAsync(); } } else if (!hasReadMe && !string.IsNullOrEmpty(activeReadMe)) { await _packageFileService.DeleteReadMeMdFileAsync(package); edit.ReadMeState = PackageEditReadMeState.Deleted; // Save entity to db. package.HasReadMe = false; if (commitChanges) { await _entitiesContext.SaveChangesAsync(); } } else { edit.ReadMeState = PackageEditReadMeState.Unchanged; } return(edit.ReadMeState != PackageEditReadMeState.Unchanged); }
public async Task MarkPackageListedAsync(Package package, bool commitChanges = true, bool updateIndex = true) { if (package == null) { throw new ArgumentNullException(nameof(package)); } if (package.Listed) { return; } if (package.PackageStatusKey == PackageStatus.Deleted) { throw new InvalidOperationException("A deleted package should never be listed!"); } if (package.PackageStatusKey == PackageStatus.FailedValidation) { throw new InvalidOperationException("A package that failed validation should never be listed!"); } if (!package.Listed && (package.IsLatestStable || package.IsLatest || package.IsLatestSemVer2 || package.IsLatestStableSemVer2)) { throw new InvalidOperationException("An unlisted package should never be latest or latest stable!"); } package.Listed = true; package.LastUpdated = DateTime.UtcNow; // NOTE: LastEdited will be overwritten by a trigger defined in the migration named "AddTriggerForPackagesLastEdited". package.LastEdited = DateTime.UtcNow; await _packageService.UpdateIsLatestAsync(package.PackageRegistration, commitChanges: false); await _auditingService.SaveAuditRecordAsync(new PackageAuditRecord(package, AuditedPackageAction.List)); _telemetryService.TrackPackageListed(package); if (commitChanges) { await _entitiesContext.SaveChangesAsync(); } if (updateIndex) { _indexingService.UpdatePackage(package); } }
public async Task <Package> ReflowAsync(string id, string version) { var package = _packageService.FindPackageByIdAndVersionStrict(id, version); if (package == null) { return(null); } EntitiesConfiguration.SuspendExecutionStrategy = true; using (var transaction = _entitiesContext.GetDatabase().BeginTransaction()) { // 1) Download package binary to memory using (var packageStream = await _packageFileService.DownloadPackageFileAsync(package)) { using (var packageArchive = new PackageArchiveReader(packageStream, leaveStreamOpen: false)) { // 2) Determine package metadata from binary var packageStreamMetadata = new PackageStreamMetadata { HashAlgorithm = Constants.Sha512HashAlgorithmId, Hash = CryptographyService.GenerateHash(packageStream.AsSeekableStream()), Size = packageStream.Length, }; var packageMetadata = PackageMetadata.FromNuspecReader(packageArchive.GetNuspecReader()); // 3) Clear referenced objects that will be reflowed ClearSupportedFrameworks(package); ClearAuthors(package); ClearDependencies(package); // 4) Reflow the package var listed = package.Listed; package = _packageService.EnrichPackageFromNuGetPackage( package, packageArchive, packageMetadata, packageStreamMetadata, package.User); package.LastEdited = DateTime.UtcNow; package.Listed = listed; // 5) Update IsLatest so that reflow can correct concurrent updates (see Gallery #2514) await _packageService.UpdateIsLatestAsync(package.PackageRegistration, commitChanges : false); // 6) Save and profit await _entitiesContext.SaveChangesAsync(); } } // Commit transaction transaction.Commit(); } EntitiesConfiguration.SuspendExecutionStrategy = false; return(package); }
private async Task RemoveMemberships(User user, User requestingUser, AccountDeletionOrphanPackagePolicy orphanPackagePolicy) { foreach (var membership in user.Organizations.ToArray()) { var organization = membership.Organization; var members = organization.Members.ToList(); var collaborators = members.Where(m => !m.IsAdmin).ToList(); var memberCount = members.Count(); user.Organizations.Remove(membership); if (memberCount < 2) { // The user we are deleting is the only member of the organization. // We should delete the entire organization. await DeleteAccountImplAsync(organization, requestingUser, orphanPackagePolicy); } else if (memberCount - 1 <= collaborators.Count()) { // All other members of this organization are collaborators, so we should promote them to administrators. foreach (var collaborator in collaborators) { collaborator.IsAdmin = true; } } } foreach (var membershipRequest in user.OrganizationRequests.ToArray()) { user.OrganizationRequests.Remove(membershipRequest); } foreach (var transformationRequest in user.OrganizationMigrationRequests.ToArray()) { user.OrganizationMigrationRequests.Remove(transformationRequest); transformationRequest.NewOrganization.OrganizationMigrationRequest = null; } var migrationRequest = user.OrganizationMigrationRequest; user.OrganizationMigrationRequest = null; if (migrationRequest != null) { migrationRequest.AdminUser.OrganizationMigrationRequests.Remove(migrationRequest); } await _entitiesContext.SaveChangesAsync(); }
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile) { if (package.PackageStatusKey != PackageStatus.Available && package.PackageStatusKey != PackageStatus.Validating) { throw new ArgumentException( $"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.", nameof(package)); } try { if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.SaveValidationPackageFileAsync(package, packageFile); } else { await _packageFileService.SavePackageFileAsync(package, packageFile); } } catch (InvalidOperationException ex) { ex.Log(); return(PackageCommitResult.Conflict); } try { // commit all changes to database as an atomic transaction await _entitiesContext.SaveChangesAsync(); } catch { // If saving to the DB fails for any reason we need to delete the package we just saved. if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); } else { await _packageFileService.DeletePackageFileAsync( package.PackageRegistration.Id, package.Version); } throw; } return(PackageCommitResult.Success); }
public async Task ActivateCertificateAsync(string thumbprint, User account) { if (string.IsNullOrEmpty(thumbprint)) { throw new ArgumentException(Strings.ArgumentCannotBeNullOrEmpty, nameof(thumbprint)); } if (account == null) { throw new ArgumentNullException(nameof(account)); } var certificate = GetCertificate(thumbprint); if (certificate == null) { throw new ArgumentException(Strings.CertificateDoesNotExist, nameof(thumbprint)); } var userCertificate = certificate.UserCertificates.SingleOrDefault(uc => uc.UserKey == account.Key); if (userCertificate == null) { userCertificate = new UserCertificate() { CertificateKey = certificate.Key, UserKey = account.Key }; _entitiesContext.UserCertificates.Add(userCertificate); await _entitiesContext.SaveChangesAsync(); await _auditingService.SaveAuditRecordAsync( new CertificateAuditRecord(AuditedCertificateAction.Activate, certificate.Thumbprint)); _telemetryService.TrackCertificateActivated(thumbprint); } }
private async Task AddPackageOwnerTask(PackageRegistration packageRegistration, User user, bool commitChanges = true) { Func <ReservedNamespace, bool> predicate = reservedNamespace => reservedNamespace.IsPrefix ? packageRegistration.Id.StartsWith(reservedNamespace.Value, StringComparison.OrdinalIgnoreCase) : packageRegistration.Id.Equals(reservedNamespace.Value, StringComparison.OrdinalIgnoreCase); var userOwnedMatchingNamespacesForId = user .ReservedNamespaces .Where(predicate); if (userOwnedMatchingNamespacesForId.Any()) { if (!packageRegistration.IsVerified) { await _packageService.UpdatePackageVerifiedStatusAsync(new List <PackageRegistration> { packageRegistration }, isVerified : true, commitChanges : commitChanges); } userOwnedMatchingNamespacesForId .ToList() .ForEach(mn => _reservedNamespaceService.AddPackageRegistrationToNamespace(mn.Value, packageRegistration)); if (commitChanges) { // The 'AddPackageRegistrationToNamespace' does not commit its changes, so saving changes for consistency. await _entitiesContext.SaveChangesAsync(); } } await _packageService.AddPackageOwnerAsync(packageRegistration, user, commitChanges); await DeletePackageOwnershipRequestAsync(packageRegistration, user, commitChanges); }
private async Task DeleteAccountImplAsync(User userToBeDeleted, User userToExecuteTheDelete, AccountDeletionOrphanPackagePolicy orphanPackagePolicy, bool commitChanges = true) { await RemoveReservedNamespaces(userToBeDeleted); await RemovePackageOwnership(userToBeDeleted, userToExecuteTheDelete, orphanPackagePolicy); await RemoveMemberships(userToBeDeleted, userToExecuteTheDelete, orphanPackagePolicy); await RemoveSecurityPolicies(userToBeDeleted); await RemoveUserCredentials(userToBeDeleted); await RemovePackageOwnershipRequests(userToBeDeleted); ResetPackagesAndAccountsDeletedBy(userToBeDeleted); RemovePackagePushedBy(userToBeDeleted); RemovePackageDeprecatedBy(userToBeDeleted); var organizationToBeDeleted = userToBeDeleted as Organization; if (organizationToBeDeleted != null) { RemoveMembers(organizationToBeDeleted); } if (!userToBeDeleted.Confirmed) { // Unconfirmed users should be hard-deleted. // Another account with the same username can be created. RemoveUser(userToBeDeleted); } else { // Confirmed users should be soft-deleted. // Another account with the same username cannot be created. RemoveUserDataInUserTable(userToBeDeleted); InsertDeleteAccount( userToBeDeleted, userToExecuteTheDelete); } if (commitChanges) { await _entitiesContext.SaveChangesAsync(); } }
public Task <int> SaveAsync(CancellationToken cancellationToken) { return(_dbContext.SaveChangesAsync(cancellationToken)); }
public async Task CommitChangesAsync() { await _entities.SaveChangesAsync(); }
public async Task UpdateDeprecation( IReadOnlyList <Package> packages, PackageDeprecationStatus status, PackageRegistration alternatePackageRegistration, Package alternatePackage, string customMessage, User user) { if (user == null) { throw new ArgumentNullException(nameof(user)); } if (packages == null || !packages.Any()) { throw new ArgumentException(nameof(packages)); } var registration = packages.First().PackageRegistration; if (packages.Select(p => p.PackageRegistrationKey).Distinct().Count() > 1) { throw new ArgumentException("All packages to deprecate must have the same ID.", nameof(packages)); } using (var strategy = new SuspendDbExecutionStrategy()) using (var transaction = _entitiesContext.GetDatabase().BeginTransaction()) { var shouldDelete = status == PackageDeprecationStatus.NotDeprecated; var deprecations = new List <PackageDeprecation>(); foreach (var package in packages) { var deprecation = package.Deprecations.SingleOrDefault(); if (shouldDelete) { if (deprecation != null) { package.Deprecations.Remove(deprecation); deprecations.Add(deprecation); } } else { if (deprecation == null) { deprecation = new PackageDeprecation { Package = package }; package.Deprecations.Add(deprecation); deprecations.Add(deprecation); } deprecation.Status = status; deprecation.DeprecatedByUser = user; deprecation.AlternatePackageRegistration = alternatePackageRegistration; deprecation.AlternatePackage = alternatePackage; deprecation.CustomMessage = customMessage; } } if (shouldDelete) { _entitiesContext.Deprecations.RemoveRange(deprecations); } else { _entitiesContext.Deprecations.AddRange(deprecations); } await _entitiesContext.SaveChangesAsync(); await _packageUpdateService.UpdatePackagesAsync(packages); transaction.Commit(); _telemetryService.TrackPackageDeprecate( packages, status, alternatePackageRegistration, alternatePackage, !string.IsNullOrWhiteSpace(customMessage)); foreach (var package in packages) { await _auditingService.SaveAuditRecordAsync( new PackageAuditRecord( package, status == PackageDeprecationStatus.NotDeprecated ? AuditedPackageAction.Undeprecate : AuditedPackageAction.Deprecate, status == PackageDeprecationStatus.NotDeprecated ? PackageUndeprecatedVia.Web : PackageDeprecatedVia.Web)); } } }
public async Task <int> CommitChangesAsync() { return(await Context.SaveChangesAsync()); }
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile) { await _validationService.StartValidationAsync(package); if (package.PackageStatusKey != PackageStatus.Available && package.PackageStatusKey != PackageStatus.Validating) { throw new ArgumentException( $"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.", nameof(package)); } try { if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.SaveValidationPackageFileAsync(package, packageFile); /* Suppose two package upload requests come in at the same time with the same package (same ID and * version). It's possible that one request has committed and validated the package AFTER the other * request has checked that this package does not exist in the database. Observe the following * sequence of events to understand why the packages container check is necessary. * * Request | Step | Component | Success | Notes * ------- | ---------------------------------------------- | ---------------- | ------- | ----- * 1 | version should not exist in DB | gallery | TRUE | 1st duplicate check (catches most cases over time) * 2 | version should not exist in DB | gallery | TRUE | * 1 | upload to validation container | gallery | TRUE | 2nd duplicate check (relevant with high concurrency) * 1 | version should not exist in packages container | gallery | TRUE | 3rd duplicate check (relevant with fast validations) * 1 | commit to DB | gallery | TRUE | * 1 | upload to packages container | async validation | TRUE | * 1 | move package to Available status in DB | async validation | TRUE | * 1 | delete from validation container | async validation | TRUE | * 2 | upload to validation container | gallery | TRUE | * 2 | version should not exist in packages container | gallery | FALSE | * 2 | delete from validation (rollback) | gallery | TRUE | Only occurs in the failure case, as a clean-up. * * Alternatively, we could handle the DB conflict exception that would occur in request 2, but this * would result in an exception that can be avoided and require some ugly code that teases the * unique constraint failure out of a SqlException. * * Another alternative is always leaving the package in the validation container. This is not great * since it doubles the amount of space we need to store packages. Also, it complicates the soft or * hard package delete flow. * * We can safely delete the validation package because we know it's ours. We know this because * saving the validation package succeeded, meaning async validation already successfully moved the * previous package (request 1's package) from the validation container to the package container * and transitioned the package to Available status. * * See the following issue in GitHub for how this case was found: * https://github.com/NuGet/NuGetGallery/issues/5039 */ if (await _packageFileService.DoesPackageFileExistAsync(package)) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); return(PackageCommitResult.Conflict); } } else { await _packageFileService.SavePackageFileAsync(package, packageFile); } } catch (FileAlreadyExistsException ex) { ex.Log(); return(PackageCommitResult.Conflict); } try { // commit all changes to database as an atomic transaction await _entitiesContext.SaveChangesAsync(); } catch { // If saving to the DB fails for any reason we need to delete the package we just saved. if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); } else { await _packageFileService.DeletePackageFileAsync( package.PackageRegistration.Id, package.Version); } throw; } return(PackageCommitResult.Success); }
public Task <int> SaveChangesAsync() { return(_context.SaveChangesAsync()); }
/// <summary> /// This method creates the symbol db entities and invokes the validations for the uploaded snupkg. /// It will send the message for validation and upload the snupkg to the "validations"/"symbols-packages" container /// based on the result. It will then update the references in the database for persistence with appropriate status. /// </summary> /// <param name="package">The package for which symbols package is to be uplloaded</param> /// <param name="packageStreamMetadata">The package stream metadata for the uploaded symbols package file.</param> /// <param name="symbolPackageFile">The symbol package file stream.</param> /// <returns>The <see cref="PackageCommitResult"/> for the symbol package upload flow.</returns> public async Task <PackageCommitResult> CreateAndUploadSymbolsPackage(Package package, PackageStreamMetadata packageStreamMetadata, Stream symbolPackageFile) { var symbolPackage = _symbolPackageService.CreateSymbolPackage(package, packageStreamMetadata); // TODO: Add Validations for symbols, for now set the status to Available. https://github.com/NuGet/NuGetGallery/issues/6235 // Add validating type to be symbols when sending message to the orchestrator. symbolPackage.StatusKey = PackageStatus.Available; symbolPackage.Published = DateTime.UtcNow; if (symbolPackage.StatusKey != PackageStatus.Available && symbolPackage.StatusKey != PackageStatus.Validating) { throw new InvalidOperationException( $"The symbol package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status."); } try { if (symbolPackage.StatusKey == PackageStatus.Validating) { await _symbolPackageFileService.SaveValidationPackageFileAsync(symbolPackage.Package, symbolPackageFile); } else if (symbolPackage.StatusKey == PackageStatus.Available) { // Mark any other associated available symbol packages for deletion. var availableSymbolPackages = package .SymbolPackages .Where(sp => sp.StatusKey == PackageStatus.Available && sp != symbolPackage); var overwrite = false; if (availableSymbolPackages.Any()) { // Mark the currently available packages for deletion, and replace the file in the container. foreach (var availableSymbolPackage in availableSymbolPackages) { availableSymbolPackage.StatusKey = PackageStatus.Deleted; } overwrite = true; } // Caveat: This doesn't really affect our prod flow since the package is validating, however, when the async validation // is disabled there is a chance that there could be concurrency issues when pushing multiple symbols simultaneously. // This could result in an inconsistent data or multiple symbol entities marked as available. This could be sovled using etag // for saving files, however since it doesn't really affect nuget.org which happen have async validations flow I will leave it as is. await _symbolPackageFileService.SavePackageFileAsync(symbolPackage.Package, symbolPackageFile, overwrite); } try { // commit all changes to database as an atomic transaction await _entitiesContext.SaveChangesAsync(); } catch (Exception ex) { ex.Log(); // If saving to the DB fails for any reason we need to delete the package we just saved. if (symbolPackage.StatusKey == PackageStatus.Validating) { await _symbolPackageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); } else if (symbolPackage.StatusKey == PackageStatus.Available) { await _symbolPackageFileService.DeletePackageFileAsync( package.PackageRegistration.Id, package.Version); } throw ex; } } catch (FileAlreadyExistsException ex) { ex.Log(); return(PackageCommitResult.Conflict); } return(PackageCommitResult.Success); }
/// <summary> /// This method creates the symbol db entities and invokes the validations for the uploaded snupkg. /// It will send the message for validation and upload the snupkg to the "validations"/"symbols-packages" container /// based on the result. It will then update the references in the database for persistence with appropriate status. /// </summary> /// <param name="package">The package for which symbols package is to be uplloaded</param> /// <param name="symbolPackageStream">The symbols package stream metadata for the uploaded symbols package file.</param> /// <returns>The <see cref="PackageCommitResult"/> for the create and upload symbol package flow.</returns> public async Task <PackageCommitResult> CreateAndUploadSymbolsPackage(Package package, Stream symbolPackageStream) { var packageStreamMetadata = new PackageStreamMetadata { HashAlgorithm = CoreConstants.Sha512HashAlgorithmId, Hash = CryptographyService.GenerateHash( symbolPackageStream.AsSeekableStream(), CoreConstants.Sha512HashAlgorithmId), Size = symbolPackageStream.Length }; Stream symbolPackageFile = symbolPackageStream.AsSeekableStream(); var symbolPackage = _symbolPackageService.CreateSymbolPackage(package, packageStreamMetadata); await _validationService.StartValidationAsync(symbolPackage); if (symbolPackage.StatusKey != PackageStatus.Available && symbolPackage.StatusKey != PackageStatus.Validating) { throw new InvalidOperationException( $"The symbol package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status."); } try { if (symbolPackage.StatusKey == PackageStatus.Validating) { await _symbolPackageFileService.SaveValidationPackageFileAsync(symbolPackage.Package, symbolPackageFile); } else if (symbolPackage.StatusKey == PackageStatus.Available) { if (!symbolPackage.Published.HasValue) { symbolPackage.Published = DateTime.UtcNow; } // Mark any other associated available symbol packages for deletion. var availableSymbolPackages = package .SymbolPackages .Where(sp => sp.StatusKey == PackageStatus.Available && sp != symbolPackage); var overwrite = false; if (availableSymbolPackages.Any()) { // Mark the currently available packages for deletion, and replace the file in the container. foreach (var availableSymbolPackage in availableSymbolPackages) { availableSymbolPackage.StatusKey = PackageStatus.Deleted; } overwrite = true; } // Caveat: This doesn't really affect our prod flow since the package is validating, however, when the async validation // is disabled there is a chance that there could be concurrency issues when pushing multiple symbols simultaneously. // This could result in an inconsistent data or multiple symbol entities marked as available. This could be sovled using etag // for saving files, however since it doesn't really affect nuget.org which happen have async validations flow I will leave it as is. await _symbolPackageFileService.SavePackageFileAsync(symbolPackage.Package, symbolPackageFile, overwrite); } try { // commit all changes to database as an atomic transaction await _entitiesContext.SaveChangesAsync(); } catch (Exception ex) { ex.Log(); // If saving to the DB fails for any reason we need to delete the package we just saved. if (symbolPackage.StatusKey == PackageStatus.Validating) { await _symbolPackageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); } else if (symbolPackage.StatusKey == PackageStatus.Available) { await _symbolPackageFileService.DeletePackageFileAsync( package.PackageRegistration.Id, package.Version); } throw ex; } } catch (FileAlreadyExistsException ex) { ex.Log(); return(PackageCommitResult.Conflict); } _telemetryService.TrackSymbolPackagePushEvent(package.Id, package.NormalizedVersion); return(PackageCommitResult.Success); }
public async Task <PackageCommitResult> CommitPackageAsync(Package package, Stream packageFile) { if (package == null) { throw new ArgumentNullException(nameof(package)); } if (packageFile == null) { throw new ArgumentNullException(nameof(packageFile)); } if (!packageFile.CanSeek) { throw new ArgumentException($"{nameof(packageFile)} argument must be seekable stream", nameof(packageFile)); } await _validationService.UpdatePackageAsync(package); if (package.PackageStatusKey != PackageStatus.Available && package.PackageStatusKey != PackageStatus.Validating) { throw new ArgumentException( $"The package to commit must have either the {PackageStatus.Available} or {PackageStatus.Validating} package status.", nameof(package)); } try { if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.SaveValidationPackageFileAsync(package, packageFile); /* Suppose two package upload requests come in at the same time with the same package (same ID and * version). It's possible that one request has committed and validated the package AFTER the other * request has checked that this package does not exist in the database. Observe the following * sequence of events to understand why the packages container check is necessary. * * Request | Step | Component | Success | Notes * ------- | ---------------------------------------------- | ---------------- | ------- | ----- * 1 | version should not exist in DB | gallery | TRUE | 1st duplicate check (catches most cases over time) * 2 | version should not exist in DB | gallery | TRUE | * 1 | upload to validation container | gallery | TRUE | 2nd duplicate check (relevant with high concurrency) * 1 | version should not exist in packages container | gallery | TRUE | 3rd duplicate check (relevant with fast validations) * 1 | commit to DB | gallery | TRUE | * 1 | upload to packages container | async validation | TRUE | * 1 | move package to Available status in DB | async validation | TRUE | * 1 | delete from validation container | async validation | TRUE | * 2 | upload to validation container | gallery | TRUE | * 2 | version should not exist in packages container | gallery | FALSE | * 2 | delete from validation (rollback) | gallery | TRUE | Only occurs in the failure case, as a clean-up. * * Alternatively, we could handle the DB conflict exception that would occur in request 2, but this * would result in an exception that can be avoided and require some ugly code that teases the * unique constraint failure out of a SqlException. * * Another alternative is always leaving the package in the validation container. This is not great * since it doubles the amount of space we need to store packages. Also, it complicates the soft or * hard package delete flow. * * We can safely delete the validation package because we know it's ours. We know this because * saving the validation package succeeded, meaning async validation already successfully moved the * previous package (request 1's package) from the validation container to the package container * and transitioned the package to Available status. * * See the following issue in GitHub for how this case was found: * https://github.com/NuGet/NuGetGallery/issues/5039 */ if (await _packageFileService.DoesPackageFileExistAsync(package)) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); return(PackageCommitResult.Conflict); } } else { if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent) { // if the package is immediately made available, it means there is a high chance we don't have // validation pipeline that would normally store the license file, so we'll do it ourselves here. await _coreLicenseFileService.ExtractAndSaveLicenseFileAsync(package, packageFile); } var isReadmeFileExtractedAndSaved = false; if (package.HasReadMe && package.EmbeddedReadmeType != EmbeddedReadmeFileType.Absent) { await _packageFileService.ExtractAndSaveReadmeFileAsync(package, packageFile); isReadmeFileExtractedAndSaved = true; } try { packageFile.Seek(0, SeekOrigin.Begin); await _packageFileService.SavePackageFileAsync(package, packageFile); } catch when(package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent || isReadmeFileExtractedAndSaved) { if (package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent) { await _coreLicenseFileService.DeleteLicenseFileAsync( package.PackageRegistration.Id, package.NormalizedVersion); } if (isReadmeFileExtractedAndSaved) { await _packageFileService.DeleteReadMeMdFileAsync(package); } throw; } } } catch (FileAlreadyExistsException ex) { ex.Log(); return(PackageCommitResult.Conflict); } try { // Sending the validation request after copying to prevent multiple validation requests // sent when several pushes for the same package happen concurrently. Copying the file // resolves the race and only one request will "win" and reach this code. await _validationService.StartValidationAsync(package); // commit all changes to database as an atomic transaction await _entitiesContext.SaveChangesAsync(); } catch (Exception ex) { // If sending the validation request or saving to the DB fails for any reason // we need to delete the package we just saved. if (package.PackageStatusKey == PackageStatus.Validating) { await _packageFileService.DeleteValidationPackageFileAsync( package.PackageRegistration.Id, package.Version); } else { await _packageFileService.DeletePackageFileAsync( package.PackageRegistration.Id, package.Version); await _coreLicenseFileService.DeleteLicenseFileAsync( package.PackageRegistration.Id, package.NormalizedVersion); await _packageFileService.DeleteReadMeMdFileAsync(package); } return(ReturnConflictOrThrow(ex)); } return(PackageCommitResult.Success); }
public void CommitAsync() { _context.SaveChangesAsync(); }