private async Task SetValidationStatusAsync( PackageValidation packageValidation, IValidationResult validationResult, DateTime now) { if (validationResult.Status != ValidationStatus.Incomplete) { AddValidationIssues(packageValidation, validationResult.Issues); } if (validationResult.Status == ValidationStatus.Succeeded && validationResult.NupkgUrl != null) { if (!_validatorProvider.IsProcessor(packageValidation.Type)) { throw new InvalidOperationException( $"The validator '{packageValidation.Type}' is not a processor but returned a .nupkg URL as " + $"part of the validation result."); } await _packageFileService.CopyPackageUrlForValidationSetAsync( packageValidation.PackageValidationSet, validationResult.NupkgUrl); } packageValidation.ValidationStatus = validationResult.Status; packageValidation.ValidationStatusTimestamp = now; await _validationContext.SaveChangesAsync(); TrackValidationStatus(packageValidation); }
private void CheckForCyclesAndParallelProcessors() { var processorNames = _configuration .Validations .Select(x => x.Name) .Where(x => _validatorProvider.IsProcessor(x)) .ToList(); TopologicalSort.Validate(_configuration.Validations, processorNames); }
private async Task <bool> UpdatePublicPackageAsync(PackageValidationSet validationSet) { _logger.LogInformation("Copying .nupkg to public storage for package {PackageId} {PackageVersion}, validation set {ValidationSetId}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); // If the validation set contains any processors, we must use the copy of the package that is specific to // this validation set. We can't use the original validation package because it does not have any of the // changes that the processors made. If the validation set package does not exist for some reason and there // are processors in the validation set, this indicates a bug and an exception will be thrown by the copy // operation below. This will cause the validation queue message to eventually dead-letter at which point // the on-call person should investigate. bool copied; if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type)) || await _packageFileService.DoesValidationSetPackageExistAsync(validationSet)) { IAccessCondition destAccessCondition; // The package etag will be null if this validation set is expecting the package to not yet exist in // the packages container. if (validationSet.PackageETag == null) { // This will fail with HTTP 409 if the package already exists. This means that another validation // set has completed and moved the package into the Available state first, with different package // content. destAccessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition(); _logger.LogInformation( "Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" + " the packages container, assuming that the package does not already exist.", validationSet.ValidationTrackingId, validationSet.PackageId, validationSet.PackageNormalizedVersion); } else { // This will fail with HTTP 412 if the package has been modified by another validation set. This // would only happen if this validation set and another validation set are operating on a package // already in the Available state. destAccessCondition = AccessConditionWrapper.GenerateIfMatchCondition(validationSet.PackageETag); _logger.LogInformation( "Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" + " the packages container, assuming that the package has etag {PackageETag}.", validationSet.ValidationTrackingId, validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.PackageETag); } // Failures here should result in an unhandled exception. This means that this validation set has // modified the package but is unable to copy the modified package into the packages container because // another validation set completed first. await _packageFileService.CopyValidationSetPackageToPackageFileAsync( validationSet, destAccessCondition); copied = true; } else { _logger.LogInformation( "The package specific to the validation set does not exist. Falling back to the validation " + "container for package {PackageId} {PackageVersion}, validation set {ValidationSetId}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); try { await _packageFileService.CopyValidationPackageToPackageFileAsync(validationSet); copied = true; } catch (InvalidOperationException) { // The package already exists in the packages container. This can happen if the DB commit below fails // and this flow is retried or another validation set for the package completed first. Either way, we // will later attempt to use the hash from the package in the packages container (the destination). // In other words, we don't care which copy wins when copying from the validation package because // we know the package has not been modified. _logger.LogInformation( "Package already exists in packages container for {PackageId} {PackageVersion}, validation set {ValidationSetId}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); copied = false; } } return(copied); }
public async Task <PackageValidationSet> TryGetOrCreateValidationSetAsync(PackageValidationMessageData message, IValidatingEntity <T> validatingEntity) { var validationSet = await _validationStorageService.GetValidationSetAsync(message.ValidationTrackingId); if (validationSet == null) { var shouldSkip = await _validationStorageService.OtherRecentValidationSetForPackageExists( validatingEntity, _validationConfiguration.NewValidationRequestDeduplicationWindow, message.ValidationTrackingId); if (shouldSkip) { return(null); } validationSet = InitializeValidationSet(message, validatingEntity); if (validatingEntity.Status == PackageStatus.Available) { var packageETag = await _packageFileService.CopyPackageFileForValidationSetAsync(validationSet); // This indicates that the package in the package container is expected to not change. validationSet.PackageETag = packageETag; } else { await _packageFileService.CopyValidationPackageForValidationSetAsync(validationSet); // A symbols package for the same id and version can be re-submitted. // When this happens a new validation is submitted. After validation the new symbols package will overwrite the old symbols package. // Because of this when a new validation for a symbols package is received it can already exist a symbols package in the public symbols container. if (validatingEntity.ValidatingType == ValidatingType.SymbolPackage) { validationSet.PackageETag = await _packageFileService.GetPublicPackageBlobETagOrNullAsync(validationSet); } else { // This indicates that the package in the packages container is expected to not exist (i.e. it has // has no etag at all). validationSet.PackageETag = null; } } // If there are any processors in the validation set, back up the original. We back up from the // validation set copy to avoid concurrency issues. if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type))) { await _packageFileService.BackupPackageFileFromValidationSetPackageAsync(validationSet); } validationSet = await PersistValidationSetAsync(validationSet, validatingEntity); } else { var sameKey = validatingEntity.Key == validationSet.PackageKey; if (!sameKey) { throw new InvalidOperationException($"Validation set key ({validationSet.PackageKey}) " + $"does not match expected {validatingEntity.EntityRecord.GetType().Name} key ({validatingEntity.Key})."); } } return(validationSet); }
public async Task <PackageValidationSet> TryGetOrCreateValidationSetAsync(Guid validationTrackingId, Package package) { var validationSet = await _validationStorageService.GetValidationSetAsync(validationTrackingId); if (validationSet == null) { var shouldSkip = await _validationStorageService.OtherRecentValidationSetForPackageExists( package.Key, _validationConfiguration.NewValidationRequestDeduplicationWindow, validationTrackingId); if (shouldSkip) { return(null); } validationSet = InitializeValidationSet(validationTrackingId, package); if (package.PackageStatusKey == PackageStatus.Available) { var packageETag = await _packageFileService.CopyPackageFileForValidationSetAsync(validationSet); // This indicates that the package in the package container is expected to not change. validationSet.PackageETag = packageETag; } else { await _packageFileService.CopyValidationPackageForValidationSetAsync(validationSet); // This indicates that the package in the packages container is expected to not exist (i.e. it has // has no etag at all). validationSet.PackageETag = null; } // If there are any processors in the validation set, back up the original. We back up from the // validation set copy to avoid concurrency issues. if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type))) { await _packageFileService.BackupPackageFileFromValidationSetPackageAsync(package, validationSet); } validationSet = await PersistValidationSetAsync(validationSet, package); } else { var sameId = package.PackageRegistration.Id.Equals( validationSet.PackageId, StringComparison.InvariantCultureIgnoreCase); var sameVersion = package.NormalizedVersion.Equals( validationSet.PackageNormalizedVersion, StringComparison.InvariantCultureIgnoreCase); if (!sameId || !sameVersion) { throw new InvalidOperationException( $"Validation set package identity ({validationSet.PackageId} {validationSet.PackageNormalizedVersion})" + $"does not match expected package identity ({package.PackageRegistration.Id} {package.NormalizedVersion})."); } var sameKey = package.Key == validationSet.PackageKey; if (!sameKey) { throw new InvalidOperationException($"Validation set package key ({validationSet.PackageKey}) " + $"does not match expected package key ({package.Key})."); } } return(validationSet); }