protected override async Task OnBeforeUpdateDatabaseToMakePackageAvailable( IValidatingEntity <Package> validatingEntity, PackageValidationSet validationSet) { if (validatingEntity.EntityRecord.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent || validatingEntity.EntityRecord.HasEmbeddedReadme) { using (_telemetryService.TrackDurationToExtractLicenseAndReadmeFile(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString())) using (var packageStream = await _packageFileService.DownloadPackageFileToDiskAsync(validationSet, _sasDefinitionConfiguration.PackageStatusProcessorSasDefinition)) { if (validatingEntity.EntityRecord.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent) { _logger.LogInformation("Extracting the license file of type {EmbeddedLicenseFileType} for the package {PackageId} {PackageVersion}", validatingEntity.EntityRecord.EmbeddedLicenseType, validationSet.PackageId, validationSet.PackageNormalizedVersion); await _coreLicenseFileService.ExtractAndSaveLicenseFileAsync(validatingEntity.EntityRecord, packageStream); _logger.LogInformation("Successfully extracted the license file."); } if (validatingEntity.EntityRecord.HasEmbeddedReadme) { _logger.LogInformation("Extracting the readme file of type {EmbeddedReadmeType} for the package {PackageId} {PackageVersion}", validatingEntity.EntityRecord.EmbeddedReadmeType, validationSet.PackageId, validationSet.PackageNormalizedVersion); await _coreReadmeFileService.ExtractAndSaveReadmeFileAsync(validatingEntity.EntityRecord, packageStream); _logger.LogInformation("Successfully extracted the readme file."); } } } }
private async Task ProcessValidationSetAsync( IValidatingEntity <TEntity> entity, PackageValidationSet validationSet, bool scheduleNextCheck) { if (validationSet.ValidationSetStatus == ValidationSetStatus.Completed) { _logger.LogInformation( "The validation set {ValidatingType} {PackageId} {PackageVersion} {Key} {ValidationSetId} is " + "already completed. Discarding the message.", ValidatingType, validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.PackageKey, validationSet.ValidationTrackingId); return; } var processorStats = await _validationSetProcessor.ProcessValidationsAsync(validationSet); await _validationOutcomeProcessor.ProcessValidationOutcomeAsync( validationSet, entity, processorStats, scheduleNextCheck); }
public void ItShouldNotProceedWhenFromAvailableState() { // Arrange IValidatingEntity <SymbolPackage> validatingSymbolPackage = null; SymbolsPackageServiceMock.Setup(sp => sp.FindPackageByIdAndVersionStrict(PackageId, PackageVersion)) .Returns(validatingSymbolPackage); var validationSet = new PackageValidationSet { PackageId = AvailableSymbolPackage.Id, PackageNormalizedVersion = AvailableSymbolPackage.Version, PackageKey = AvailableSymbolPackage.Key, PackageValidations = new List <PackageValidation> { new PackageValidation { Type = "SomeValidator" }, } }; // Act bool result = Target.CanProceedToMakePackageAvailable(AvailableSymbolPackageValidatingEntity, validationSet); Target.SetStatusAsync(AvailableSymbolPackageValidatingEntity, validationSet, PackageStatus.Available); // Assert Assert.False(result); SymbolPackageFileServiceMock.Verify(spfs => spfs.DoesValidationSetPackageExistAsync(It.IsAny <PackageValidationSet>()), Times.Never); SymbolPackageFileServiceMock.Verify(spfs => spfs.CopyValidationSetPackageToPackageFileAsync(It.IsAny <PackageValidationSet>(), It.IsAny <IAccessCondition>()), Times.Never); SymbolPackageFileServiceMock.Verify(spfs => spfs.CopyValidationPackageToPackageFileAsync(It.IsAny <PackageValidationSet>()), Times.Never); SymbolPackageFileServiceMock.Verify(spfs => spfs.UpdatePackageBlobMetadataInValidationSetAsync(It.IsAny <PackageValidationSet>()), Times.Never); SymbolPackageFileServiceMock.Verify(spfs => spfs.UpdatePackageBlobMetadataInValidationAsync(It.IsAny <PackageValidationSet>()), Times.Never); SymbolsPackageServiceMock.Verify(sps => sps.UpdateStatusAsync(It.IsAny <SymbolPackage>(), It.IsAny <PackageStatus>(), It.IsAny <bool>()), Times.Never); }
public void ItShouldProceedWhenFromFailedStateWithNoValidationInProgress() { // Arrange IValidatingEntity <SymbolPackage> validatingSymbolPackage = null; SymbolsPackageServiceMock.Setup(sp => sp.FindPackageByIdAndVersionStrict(PackageId, PackageVersion)) .Returns(validatingSymbolPackage); var validationSet = new PackageValidationSet { PackageId = FailedSymbolPackage.Id, PackageNormalizedVersion = FailedSymbolPackage.Version, PackageKey = FailedSymbolPackageValidatingEntity.Key, PackageValidations = new List <PackageValidation> { new PackageValidation { Type = "SomeValidator" }, } }; // Act bool result = Target.CanProceedToMakePackageAvailable(FailedSymbolPackageValidatingEntity, validationSet); Target.SetStatusAsync(FailedSymbolPackageValidatingEntity, validationSet, PackageStatus.Available); // Assert Assert.True(result); SymbolPackageFileServiceMock.Verify(spfs => spfs.DoesValidationSetPackageExistAsync(It.IsAny <PackageValidationSet>()), Times.Once); }
/// <summary> /// Proceed to change the state only if: /// 1.the current symbol entity is in a failed state and there is not an existent symbol push already started by the user. This state can happen on revalidation only. /// or /// 2. the current validation is in validating state /// If the symbols validation would have processors as validators the copy should be done on other states as well. /// </summary> /// <param name="validatingEntity">The <see cref="IValidatingEntity<SymbolPackage>"/> that is under current revalidation.</param> /// <param name="validationSet">The validation set for the current validation.</param> /// <returns>True if the package should be made available (copied to the public container, db updated etc.)</returns> public bool CanProceedToMakePackageAvailable(IValidatingEntity <SymbolPackage> validatingEntity, PackageValidationSet validationSet) { // _galleryPackageService.FindPackageByIdAndVersionStrict will return the symbol entity that is in Validating state or null if // there not any symbols entity in validating state. var validatingSymbolsEntity = _galleryPackageService.FindPackageByIdAndVersionStrict(validationSet.PackageId, validationSet.PackageNormalizedVersion); // If the current entity is in validating mode a new symbolPush is not allowed, so it is safe to copy. var aNewEntityInValidatingStateExists = validatingSymbolsEntity != null; var proceed = validatingEntity.Status == PackageStatus.Validating || (!aNewEntityInValidatingStateExists && validatingEntity.Status == PackageStatus.FailedValidation); _logger.LogInformation("Proceed to make symbols available check: " + "PackageId {PackageId} " + "PackageVersion {PackageVersion} " + "ValidationTrackingId {ValidationTrackingId} " + "CurrentValidating entity status {CurrentEntityStatus}" + "ANewEntityInValidatingStateExists {ANewEntityInValidatingStateExists}" + "Proceed {Proceed}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, validatingEntity.Status, aNewEntityInValidatingStateExists, proceed ); return(proceed); }
protected override async Task OnCleanupAfterDatabaseUpdateFailure( IValidatingEntity <Package> validatingEntity, PackageValidationSet validationSet) { if (validatingEntity.EntityRecord.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent) { using (_telemetryService.TrackDurationToDeleteLicenseFile(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString())) { _logger.LogInformation("Cleaning up the license file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion); await _coreLicenseFileService.DeleteLicenseFileAsync(validationSet.PackageId, validationSet.PackageNormalizedVersion); _logger.LogInformation("Deleted the license file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion); } } if (validatingEntity.EntityRecord.HasEmbeddedReadme) { using (_telemetryService.TrackDurationToDeleteReadmeFile(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString())) { _logger.LogInformation("Cleaning up the readme file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion); await _coreReadmeFileService.DeleteReadmeFileAsync(validationSet.PackageId, validationSet.PackageNormalizedVersion); _logger.LogInformation("Deleted the readme file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion); } } }
private async Task <PackageStatus> MarkPackageAsAvailableAsync( PackageValidationSet validationSet, IValidatingEntity <T> validatingEntity, PackageStreamMetadata streamMetadata, bool copied) { // Use whatever package made it into the packages container. This is what customers will consume so the DB // record must match. // We don't immediately commit here. Later, we will commit these changes as well as the new package // status as part of the same transaction. await _galleryPackageService.UpdateMetadataAsync( validatingEntity.EntityRecord, streamMetadata, commitChanges : false); _logger.LogInformation("Marking package {PackageId} {PackageVersion}, validation set {ValidationSetId} as {PackageStatus} in DB", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, PackageStatus.Available); var fromStatus = validatingEntity.Status; try { // Make the package available and commit any other pending changes (e.g. updated hash). await _galleryPackageService.UpdateStatusAsync(validatingEntity.EntityRecord, PackageStatus.Available, commitChanges : true); } catch (Exception e) { _logger.LogError( Error.UpdatingPackageDbStatusFailed, e, "Failed to update package status in Gallery Db. Package {PackageId} {PackageVersion}, validation set {ValidationSetId}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); // If this execution was not the one to copy the package, then don't delete the package on failure. // This prevents a missing passing in the (unlikely) case where two actors attempt the DB update, one // succeeds and one fails. We don't want an available package record with nothing in the packages // container! if (copied && fromStatus != PackageStatus.Available) { await _packageFileService.DeletePackageFileAsync(validationSet); await OnCleanupAfterDatabaseUpdateFailure(validatingEntity, validationSet); } throw; } return(fromStatus); }
protected virtual async Task MakePackageFailedValidationAsync(IValidatingEntity <T> validatingEntity, PackageValidationSet validationSet) { var fromStatus = validatingEntity.Status; await _galleryPackageService.UpdateStatusAsync(validatingEntity.EntityRecord, PackageStatus.FailedValidation, commitChanges : true); if (fromStatus != PackageStatus.FailedValidation) { _telemetryService.TrackPackageStatusChange(fromStatus, PackageStatus.FailedValidation); } }
protected override async Task MakePackageAvailableAsync(IValidatingEntity <SymbolPackage> validatingEntity, PackageValidationSet validationSet) { if (!CanProceedToMakePackageAvailable(validatingEntity, validationSet)) { _logger.LogInformation("SymbolsPackage PackageId {PackageId} PackageVersion {PackageVersion} Status {Status} was not made available again.", validationSet.PackageId, validationSet.PackageNormalizedVersion, validatingEntity.Status); return; } await base.MakePackageAvailableAsync(validatingEntity, validationSet); }
public async Task <bool> OtherRecentValidationSetForPackageExists <T>( IValidatingEntity <T> validatingEntity, TimeSpan recentDuration, Guid currentValidationSetTrackingId) where T : class, IEntity { var cutoffTimestamp = DateTime.UtcNow - recentDuration; return(await _validationContext .PackageValidationSets .AnyAsync(pvs => pvs.PackageKey == validatingEntity.Key && pvs.Created > cutoffTimestamp && pvs.ValidationTrackingId != currentValidationSetTrackingId && pvs.ValidatingType == validatingEntity.ValidatingType)); }
protected virtual async Task MakePackageAvailableAsync(IValidatingEntity <T> validatingEntity, PackageValidationSet validationSet) { // 1) Operate on blob storage. var copied = await UpdatePublicPackageAsync(validationSet); // 2) Update the package's blob metadata in the public blob storage container. var metadata = await _packageFileService.UpdatePackageBlobMetadataAsync(validationSet); // 3) Update the package's blob properties in the public blob storage container. await _packageFileService.UpdatePackageBlobPropertiesAsync(validationSet); // 3.5) Allow descendants to do their own things before we update the database await OnBeforeUpdateDatabaseToMakePackageAvailable(validatingEntity, validationSet); // 4) Operate on the database. var fromStatus = await MarkPackageAsAvailableAsync(validationSet, validatingEntity, metadata, copied); // 5) Emit telemetry and clean up. if (fromStatus != PackageStatus.Available) { _telemetryService.TrackPackageStatusChange(fromStatus, PackageStatus.Available); _logger.LogInformation("Deleting from the source for package {PackageId} {PackageVersion}, validation set {ValidationSetId}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); await _packageFileService.DeleteValidationPackageFileAsync(validationSet); } // 5) Verify the package still exists (we've had bugs here before). if (validatingEntity.Status == PackageStatus.Available && !await _packageFileService.DoesPackageFileExistAsync(validationSet)) { var validationPackageAvailable = await _packageFileService.DoesValidationPackageFileExistAsync(validationSet); _logger.LogWarning("Package {PackageId} {PackageVersion} is marked as available, but does not exist " + "in public container. Does package exist in validation container: {ExistsInValidation}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationPackageAvailable); // Report missing package, don't try to fix up anything. This shouldn't happen and needs an investigation. _telemetryService.TrackMissingNupkgForAvailablePackage( validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString()); } }
public Task SetStatusAsync( IValidatingEntity <T> validatingEntity, PackageValidationSet validationSet, PackageStatus status) { if (validatingEntity == null) { throw new ArgumentNullException(nameof(validatingEntity)); } if (validationSet == null) { throw new ArgumentNullException(nameof(validationSet)); } if (validatingEntity.Status == PackageStatus.Deleted) { throw new ArgumentException( $"A package in the {nameof(PackageStatus.Deleted)} state cannot be processed.", nameof(validatingEntity)); } if (validatingEntity.Status == PackageStatus.Available && status == PackageStatus.FailedValidation) { throw new ArgumentException( $"A package cannot transition from {nameof(PackageStatus.Available)} to {nameof(PackageStatus.FailedValidation)}.", nameof(status)); } switch (status) { case PackageStatus.Available: return(MakePackageAvailableAsync(validatingEntity, validationSet)); case PackageStatus.FailedValidation: return(MakePackageFailedValidationAsync(validatingEntity, validationSet)); default: throw new ArgumentException( $"A package can only transition to the {nameof(PackageStatus.Available)} or " + $"{nameof(PackageStatus.FailedValidation)} states.", nameof(status)); } }
public Facts(ITestOutputHelper output) { _packageKey = 23; _validatorType = "ExampleValidator"; _packageValidation = new PackageValidation { Key = Guid.Parse("dc10bf5a-0557-459c-b33f-ea6738b8a044"), Type = _validatorType, ValidationStatus = ValidationStatus.Incomplete, Started = new DateTime(2017, 1, 1, 8, 30, 0, DateTimeKind.Utc), PackageValidationIssues = new List <PackageValidationIssue>(), PackageValidationSet = new PackageValidationSet(), }; var package = new Package() { Key = _packageKey }; _packageValidatingEntity = new PackageValidatingEntity(package); _nupkgUrl = "https://example/nuget.versioning.4.3.0.nupkg"; _entitiesContext = new Mock <IValidationEntitiesContext>(); _entitiesContext .Setup(x => x.PackageValidationSets) .Returns(DbSetMockFactory.Create <PackageValidationSet>()); _entitiesContext .Setup(x => x.PackageValidations) .Returns(DbSetMockFactory.Create <PackageValidation>()); _packageFileService = new Mock <IValidationFileService>(); _validatorProvider = new Mock <IValidatorProvider>(); _telemetryService = new Mock <ITelemetryService>(); _loggerFactory = new LoggerFactory(); _loggerFactory.AddXunit(output); InitializeTarget(); }
private async Task ScheduleCheckIfNotTimedOut( PackageValidationSet validationSet, IValidatingEntity <T> validatingEntity, bool scheduleNextCheck, bool tooLongNotificationAllowed) { var validationSetDuration = await UpdateValidationDurationAsync(validationSet, validatingEntity, tooLongNotificationAllowed); // Schedule another check if we haven't reached the validation set timeout yet. if (validationSetDuration <= _validationConfiguration.TimeoutValidationSetAfter) { if (scheduleNextCheck) { var messageData = PackageValidationMessageData.NewProcessValidationSet( validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, validationSet.ValidatingType, entityKey: validationSet.PackageKey); var postponeUntil = DateTimeOffset.UtcNow + _validationConfiguration.ValidationMessageRecheckPeriod; await _validationEnqueuer.SendMessageAsync(messageData, postponeUntil); } } else { _logger.LogWarning("Abandoning checking status of validation set {ValidationTrackingId} for " + "package {PackageId} {PackageVersion} because it took too long (Duration: {Duration}, CutOffDuration: {CutOffDuration})", validationSet.ValidationTrackingId, validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSetDuration, _validationConfiguration.TimeoutValidationSetAfter); _telemetryService.TrackValidationSetTimeout(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); } }
private PackageValidationSet InitializeValidationSet(ProcessValidationSetData message, IValidatingEntity <T> validatingEntity) { // If message would have the package Key the package will not need to be passed as an argument _logger.LogInformation("Initializing validation set {ValidationSetId} for package {PackageId} {PackageVersion} (package key {PackageKey})", message.ValidationTrackingId, message.PackageId, message.PackageVersion, validatingEntity.Key); var now = DateTime.UtcNow; var validationSet = new PackageValidationSet { Created = now, PackageId = message.PackageId, PackageNormalizedVersion = message.PackageNormalizedVersion, PackageKey = validatingEntity.Key, PackageValidations = new List <PackageValidation>(), Updated = now, ValidationTrackingId = message.ValidationTrackingId, ValidatingType = message.ValidatingType, ValidationSetStatus = ValidationSetStatus.InProgress, }; var validationsToStart = _validationConfiguration .Validations .Where(v => v.ShouldStart); foreach (var validation in validationsToStart) { var packageValidation = new PackageValidation { PackageValidationSet = validationSet, ValidationStatus = ValidationStatus.NotStarted, Type = validation.Name, ValidationStatusTimestamp = now, }; validationSet.PackageValidations.Add(packageValidation); } return(validationSet); }
public async Task ProcessValidationOutcomeAsync(PackageValidationSet validationSet, IValidatingEntity <T> validatingEntity, ValidationSetProcessorResult currentCallStats) { var failedValidations = GetFailedValidations(validationSet); if (failedValidations.Any()) { _logger.LogWarning("Some validations failed for package {PackageId} {PackageVersion}, validation set {ValidationSetId}: {FailedValidations}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, failedValidations.Select(x => x.Type).ToList()); // The only way we can move to the failed validation state is if the package is currently in the // validating state. This has a beneficial side effect of only sending a failed validation email to the // customer when the package first moves to the failed validation state. If an admin comes along and // revalidates the package and the package fails validation again, we don't want another email going // out since that would be noisy for the customer. if (validatingEntity.Status == PackageStatus.Validating) { await _packageStateProcessor.SetStatusAsync(validatingEntity, validationSet, PackageStatus.FailedValidation); await MarkValidationSetAsCompletedAsync(validationSet); await _messageService.SendValidationFailedMessageAsync(validatingEntity.EntityRecord, validationSet); } else { // The case when validation fails while PackageStatus not validating is the case of // manual revalidation. In this case we don't want to take package down automatically // and let the person who requested revalidation to decide how to proceed. Ops will be // alerted by failed validation monitoring. _logger.LogInformation("Package {PackageId} {PackageVersion} was {PackageStatus} when validation set {ValidationSetId} failed. Will not mark it as failed.", validationSet.PackageId, validationSet.PackageNormalizedVersion, validatingEntity.Status, validationSet.ValidationTrackingId); await MarkValidationSetAsCompletedAsync(validationSet); } TrackValidationSetCompletion(validationSet, isSuccess: false); await _packageFileService.DeletePackageForValidationSetAsync(validationSet); } else if (AllRequiredValidationsSucceeded(validationSet)) { _logger.LogInformation("All validations are complete for the package {PackageId} {PackageVersion}, validation set {ValidationSetId}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); var fromStatus = validatingEntity.Status; // Always set the package status to available so that processors can have a chance to fix packages // that are already available. Processors should no-op when their work is already done, so the // modification of an already available package should be rare. The most common case for this is if // the processor has never been run on a package that was published before the processor was // implemented. In this case, the processor has to play catch-up. await _packageStateProcessor.SetStatusAsync(validatingEntity, validationSet, PackageStatus.Available); var areOptionalValidationsRunning = AreOptionalValidationsRunning(validationSet); if (!areOptionalValidationsRunning) { await MarkValidationSetAsCompletedAsync(validationSet); } // Only send the email when first transitioning into the Available state. if (fromStatus != PackageStatus.Available) { await _messageService.SendPublishedMessageAsync(validatingEntity.EntityRecord); } if (currentCallStats.AnyRequiredValidationSucceeded) { TrackValidationSetCompletion(validationSet, isSuccess: true); } if (areOptionalValidationsRunning) { await ScheduleCheckIfNotTimedOut(validationSet, validatingEntity, tooLongNotificationAllowed : false); } // TODO: implement delayed cleanup that would allow internal services // to access original packages for some time after package become available: // https://github.com/NuGet/Engineering/issues/2506 } else { await ScheduleCheckIfNotTimedOut(validationSet, validatingEntity, tooLongNotificationAllowed : true); } }
private async Task <TimeSpan> UpdateValidationDurationAsync(PackageValidationSet validationSet, IValidatingEntity <T> validatingEntity, bool tooLongNotificationAllowed) { // There are no failed validations and some validations are still in progress. Update // the validation set's Updated field and send a notice if the validation set is taking // too long to complete. var previousUpdateTime = validationSet.Updated; await _validationStorageService.UpdateValidationSetAsync(validationSet); var validationSetDuration = validationSet.Updated - validationSet.Created; var previousDuration = previousUpdateTime - validationSet.Created; // Only send a "validating taking too long" notice once. This is ensured by verifying this is // the package's first validation set and that this is the first time the validation set duration // is greater than the configured threshold. Service Bus message duplication for a single validation // set will not cause multiple notices to be sent due to the row version on PackageValidationSet. if (tooLongNotificationAllowed && validatingEntity.Status == PackageStatus.Validating && validationSetDuration > _validationConfiguration.ValidationSetNotificationTimeout && previousDuration <= _validationConfiguration.ValidationSetNotificationTimeout && await _validationStorageService.GetValidationSetCountAsync(validatingEntity) == 1) { _logger.LogWarning("Sending message that validation set {ValidationTrackingId} for package {PackageId} {PackageVersion} is taking too long", validationSet.ValidationTrackingId, validationSet.PackageId, validationSet.PackageNormalizedVersion); await _messageService.SendValidationTakingTooLongMessageAsync(validatingEntity.EntityRecord); _telemetryService.TrackSentValidationTakingTooLongMessage(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); } // Track any validations that are past their expected thresholds. var timedOutValidations = GetIncompleteTimedOutValidations(validationSet); if (timedOutValidations.Any()) { foreach (var validation in timedOutValidations) { var duration = DateTime.UtcNow - validation.Started; _logger.LogWarning("Validation {Validation} for package {PackageId} {PackageVersion} is past its expected duration after {Duration}", validation.Type, validationSet.PackageId, validationSet.PackageNormalizedVersion, duration); _telemetryService.TrackValidatorTimeout(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, validation.Type); } } return(validationSetDuration); }
/// <summary> /// Allows descendants to do additional cleanup on failure to update DB when marking package as available. /// Only called if package was copied to public container before trying to update DB. /// </summary> /// <param name="validatingEntity">Entity being marked as available.</param> /// <param name="validationSet">Validation set that was completed.</param> protected virtual Task OnCleanupAfterDatabaseUpdateFailure( IValidatingEntity <T> validatingEntity, PackageValidationSet validationSet) { return(Task.CompletedTask); }
private async Task <PackageValidationSet> PersistValidationSetAsync(PackageValidationSet validationSet, IValidatingEntity <T> validatingEntity) { _logger.LogInformation("Persisting validation set {ValidationSetId} for package {PackageId} {PackageVersion} (package key {PackageKey})", validationSet.ValidationTrackingId, validationSet.PackageId, validationSet.PackageNormalizedVersion, validatingEntity.Key); var persistedValidationSet = await _validationStorageService.CreateValidationSetAsync(validationSet); // Only track the validation set creation time when this is the first validation set to be created for that // package. There will be more than one validation set when an admin has requested a manual revalidation. // This can happen much later than when the package was created so the duration is less interesting in that // case. if (await _validationStorageService.GetValidationSetCountAsync(validatingEntity) == 1) { _telemetryService.TrackDurationToValidationSetCreation(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, validationSet.Created - validatingEntity.Created); } return(persistedValidationSet); }
/// <summary> /// Allows descendants to do additional operations before database is updated to mark package as available. /// </summary> /// <param name="validatingEntity">Entity being marked as available.</param> /// <param name="validationSet">Validation set that was completed.</param> protected virtual Task OnBeforeUpdateDatabaseToMakePackageAvailable(IValidatingEntity <T> validatingEntity, PackageValidationSet validationSet) { return(Task.CompletedTask); }
public async Task <PackageValidationSet> TryGetOrCreateValidationSetAsync(ProcessValidationSetData message, IValidatingEntity <T> validatingEntity) { var validationSet = await _validationStorageService.GetValidationSetAsync(message.ValidationTrackingId); if (validationSet == null) { var shouldSkip = await _validationStorageService.OtherRecentValidationSetForPackageExists( validatingEntity, _validationConfiguration.NewValidationRequestDeduplicationWindow, message.ValidationTrackingId); if (shouldSkip) { return(null); } validationSet = InitializeValidationSet(message, validatingEntity); if (validatingEntity.Status == PackageStatus.Available) { var packageETag = await _packageFileService.CopyPackageFileForValidationSetAsync(validationSet); // This indicates that the package in the package container is expected to not change. validationSet.PackageETag = packageETag; } else { await _packageFileService.CopyValidationPackageForValidationSetAsync(validationSet); // A symbols package for the same id and version can be re-submitted. // When this happens a new validation is submitted. After validation the new symbols package will overwrite the old symbols package. // Because of this when a new validation for a symbols package is received it can already exist a symbols package in the public symbols container. if (validatingEntity.ValidatingType == ValidatingType.SymbolPackage) { validationSet.PackageETag = await _packageFileService.GetPublicPackageBlobETagOrNullAsync(validationSet); } else { // This indicates that the package in the packages container is expected to not exist (i.e. it has // has no etag at all). validationSet.PackageETag = null; } } // If there are any processors in the validation set, back up the original. We back up from the // validation set copy to avoid concurrency issues. if (validationSet.PackageValidations.Any(x => _validatorProvider.IsNuGetProcessor(x.Type))) { await _packageFileService.BackupPackageFileFromValidationSetPackageAsync(validationSet, _sasDefinitionConfiguration.ValidationSetProviderSasDefinition); } validationSet = await PersistValidationSetAsync(validationSet, validatingEntity); } else { var sameKey = validatingEntity.Key == validationSet.PackageKey; if (!sameKey) { throw new InvalidOperationException($"Validation set key ({validationSet.PackageKey}) " + $"does not match expected {validatingEntity.EntityRecord.GetType().Name} key ({validatingEntity.Key})."); } } return(validationSet); }
public async Task <int> GetValidationSetCountAsync <T>(IValidatingEntity <T> validatingEntity) where T : class, IEntity { return(await _validationContext .PackageValidationSets .CountAsync(x => x.PackageKey == validatingEntity.Key && x.ValidatingType == validatingEntity.ValidatingType)); }