protected override async Task OnBeforeUpdateDatabaseToMakePackageAvailable(
IValidatingEntity <Package> validatingEntity,
PackageValidationSet validationSet)
{
if (validatingEntity.EntityRecord.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent || validatingEntity.EntityRecord.HasEmbeddedReadme)
{
using (_telemetryService.TrackDurationToExtractLicenseAndReadmeFile(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString()))
using (var packageStream = await _packageFileService.DownloadPackageFileToDiskAsync(validationSet, _sasDefinitionConfiguration.PackageStatusProcessorSasDefinition))
{
if (validatingEntity.EntityRecord.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
_logger.LogInformation("Extracting the license file of type {EmbeddedLicenseFileType} for the package {PackageId} {PackageVersion}",
validatingEntity.EntityRecord.EmbeddedLicenseType,
validationSet.PackageId,
validationSet.PackageNormalizedVersion);
await _coreLicenseFileService.ExtractAndSaveLicenseFileAsync(validatingEntity.EntityRecord, packageStream);
_logger.LogInformation("Successfully extracted the license file.");
}
if (validatingEntity.EntityRecord.HasEmbeddedReadme)
{
_logger.LogInformation("Extracting the readme file of type {EmbeddedReadmeType} for the package {PackageId} {PackageVersion}",
validatingEntity.EntityRecord.EmbeddedReadmeType,
validationSet.PackageId,
validationSet.PackageNormalizedVersion);
await _coreReadmeFileService.ExtractAndSaveReadmeFileAsync(validatingEntity.EntityRecord, packageStream);
_logger.LogInformation("Successfully extracted the readme file.");
}
}
}
}
private async Task ProcessValidationSetAsync(
IValidatingEntity <TEntity> entity,
PackageValidationSet validationSet,
bool scheduleNextCheck)
{
if (validationSet.ValidationSetStatus == ValidationSetStatus.Completed)
{
_logger.LogInformation(
"The validation set {ValidatingType} {PackageId} {PackageVersion} {Key} {ValidationSetId} is " +
"already completed. Discarding the message.",
ValidatingType,
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.PackageKey,
validationSet.ValidationTrackingId);
return;
}
var processorStats = await _validationSetProcessor.ProcessValidationsAsync(validationSet);
await _validationOutcomeProcessor.ProcessValidationOutcomeAsync(
validationSet,
entity,
processorStats,
scheduleNextCheck);
}
public void ItShouldNotProceedWhenFromAvailableState()
{
// Arrange
IValidatingEntity <SymbolPackage> validatingSymbolPackage = null;
SymbolsPackageServiceMock.Setup(sp => sp.FindPackageByIdAndVersionStrict(PackageId, PackageVersion))
.Returns(validatingSymbolPackage);
var validationSet = new PackageValidationSet
{
PackageId = AvailableSymbolPackage.Id,
PackageNormalizedVersion = AvailableSymbolPackage.Version,
PackageKey = AvailableSymbolPackage.Key,
PackageValidations = new List <PackageValidation>
{
new PackageValidation {
Type = "SomeValidator"
},
}
};
// Act
bool result = Target.CanProceedToMakePackageAvailable(AvailableSymbolPackageValidatingEntity, validationSet);
Target.SetStatusAsync(AvailableSymbolPackageValidatingEntity, validationSet, PackageStatus.Available);
// Assert
Assert.False(result);
SymbolPackageFileServiceMock.Verify(spfs => spfs.DoesValidationSetPackageExistAsync(It.IsAny <PackageValidationSet>()), Times.Never);
SymbolPackageFileServiceMock.Verify(spfs => spfs.CopyValidationSetPackageToPackageFileAsync(It.IsAny <PackageValidationSet>(), It.IsAny <IAccessCondition>()), Times.Never);
SymbolPackageFileServiceMock.Verify(spfs => spfs.CopyValidationPackageToPackageFileAsync(It.IsAny <PackageValidationSet>()), Times.Never);
SymbolPackageFileServiceMock.Verify(spfs => spfs.UpdatePackageBlobMetadataInValidationSetAsync(It.IsAny <PackageValidationSet>()), Times.Never);
SymbolPackageFileServiceMock.Verify(spfs => spfs.UpdatePackageBlobMetadataInValidationAsync(It.IsAny <PackageValidationSet>()), Times.Never);
SymbolsPackageServiceMock.Verify(sps => sps.UpdateStatusAsync(It.IsAny <SymbolPackage>(), It.IsAny <PackageStatus>(), It.IsAny <bool>()), Times.Never);
}
public void ItShouldProceedWhenFromFailedStateWithNoValidationInProgress()
{
// Arrange
IValidatingEntity <SymbolPackage> validatingSymbolPackage = null;
SymbolsPackageServiceMock.Setup(sp => sp.FindPackageByIdAndVersionStrict(PackageId, PackageVersion))
.Returns(validatingSymbolPackage);
var validationSet = new PackageValidationSet
{
PackageId = FailedSymbolPackage.Id,
PackageNormalizedVersion = FailedSymbolPackage.Version,
PackageKey = FailedSymbolPackageValidatingEntity.Key,
PackageValidations = new List <PackageValidation>
{
new PackageValidation {
Type = "SomeValidator"
},
}
};
// Act
bool result = Target.CanProceedToMakePackageAvailable(FailedSymbolPackageValidatingEntity, validationSet);
Target.SetStatusAsync(FailedSymbolPackageValidatingEntity, validationSet, PackageStatus.Available);
// Assert
Assert.True(result);
SymbolPackageFileServiceMock.Verify(spfs => spfs.DoesValidationSetPackageExistAsync(It.IsAny <PackageValidationSet>()), Times.Once);
}
/// <summary>
/// Proceed to change the state only if:
/// 1.the current symbol entity is in a failed state and there is not an existent symbol push already started by the user. This state can happen on revalidation only.
/// or
/// 2. the current validation is in validating state
/// If the symbols validation would have processors as validators the copy should be done on other states as well.
/// </summary>
/// <param name="validatingEntity">The <see cref="IValidatingEntity<SymbolPackage>"/> that is under current revalidation.</param>
/// <param name="validationSet">The validation set for the current validation.</param>
/// <returns>True if the package should be made available (copied to the public container, db updated etc.)</returns>
public bool CanProceedToMakePackageAvailable(IValidatingEntity <SymbolPackage> validatingEntity, PackageValidationSet validationSet)
{
// _galleryPackageService.FindPackageByIdAndVersionStrict will return the symbol entity that is in Validating state or null if
// there not any symbols entity in validating state.
var validatingSymbolsEntity = _galleryPackageService.FindPackageByIdAndVersionStrict(validationSet.PackageId, validationSet.PackageNormalizedVersion);
// If the current entity is in validating mode a new symbolPush is not allowed, so it is safe to copy.
var aNewEntityInValidatingStateExists = validatingSymbolsEntity != null;
var proceed = validatingEntity.Status == PackageStatus.Validating || (!aNewEntityInValidatingStateExists && validatingEntity.Status == PackageStatus.FailedValidation);
_logger.LogInformation("Proceed to make symbols available check: "
+ "PackageId {PackageId} "
+ "PackageVersion {PackageVersion} "
+ "ValidationTrackingId {ValidationTrackingId} "
+ "CurrentValidating entity status {CurrentEntityStatus}"
+ "ANewEntityInValidatingStateExists {ANewEntityInValidatingStateExists}"
+ "Proceed {Proceed}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId,
validatingEntity.Status,
aNewEntityInValidatingStateExists,
proceed
);
return(proceed);
}
protected override async Task OnCleanupAfterDatabaseUpdateFailure(
IValidatingEntity <Package> validatingEntity,
PackageValidationSet validationSet)
{
if (validatingEntity.EntityRecord.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent)
{
using (_telemetryService.TrackDurationToDeleteLicenseFile(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString()))
{
_logger.LogInformation("Cleaning up the license file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion);
await _coreLicenseFileService.DeleteLicenseFileAsync(validationSet.PackageId, validationSet.PackageNormalizedVersion);
_logger.LogInformation("Deleted the license file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion);
}
}
if (validatingEntity.EntityRecord.HasEmbeddedReadme)
{
using (_telemetryService.TrackDurationToDeleteReadmeFile(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId.ToString()))
{
_logger.LogInformation("Cleaning up the readme file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion);
await _coreReadmeFileService.DeleteReadmeFileAsync(validationSet.PackageId, validationSet.PackageNormalizedVersion);
_logger.LogInformation("Deleted the readme file for the package {PackageId} {PackageVersion}", validationSet.PackageId, validationSet.PackageNormalizedVersion);
}
}
}
private async Task <PackageStatus> MarkPackageAsAvailableAsync(
PackageValidationSet validationSet,
IValidatingEntity <T> validatingEntity,
PackageStreamMetadata streamMetadata,
bool copied)
{
// Use whatever package made it into the packages container. This is what customers will consume so the DB
// record must match.
// We don't immediately commit here. Later, we will commit these changes as well as the new package
// status as part of the same transaction.
await _galleryPackageService.UpdateMetadataAsync(
validatingEntity.EntityRecord,
streamMetadata,
commitChanges : false);
_logger.LogInformation("Marking package {PackageId} {PackageVersion}, validation set {ValidationSetId} as {PackageStatus} in DB",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId,
PackageStatus.Available);
var fromStatus = validatingEntity.Status;
try
{
// Make the package available and commit any other pending changes (e.g. updated hash).
await _galleryPackageService.UpdateStatusAsync(validatingEntity.EntityRecord, PackageStatus.Available, commitChanges : true);
}
catch (Exception e)
{
_logger.LogError(
Error.UpdatingPackageDbStatusFailed,
e,
"Failed to update package status in Gallery Db. Package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
// If this execution was not the one to copy the package, then don't delete the package on failure.
// This prevents a missing passing in the (unlikely) case where two actors attempt the DB update, one
// succeeds and one fails. We don't want an available package record with nothing in the packages
// container!
if (copied && fromStatus != PackageStatus.Available)
{
await _packageFileService.DeletePackageFileAsync(validationSet);
await OnCleanupAfterDatabaseUpdateFailure(validatingEntity, validationSet);
}
throw;
}
return(fromStatus);
}
protected virtual async Task MakePackageFailedValidationAsync(IValidatingEntity <T> validatingEntity, PackageValidationSet validationSet)
{
var fromStatus = validatingEntity.Status;
await _galleryPackageService.UpdateStatusAsync(validatingEntity.EntityRecord, PackageStatus.FailedValidation, commitChanges : true);
if (fromStatus != PackageStatus.FailedValidation)
{
_telemetryService.TrackPackageStatusChange(fromStatus, PackageStatus.FailedValidation);
}
}
protected override async Task MakePackageAvailableAsync(IValidatingEntity <SymbolPackage> validatingEntity, PackageValidationSet validationSet)
{
if (!CanProceedToMakePackageAvailable(validatingEntity, validationSet))
{
_logger.LogInformation("SymbolsPackage PackageId {PackageId} PackageVersion {PackageVersion} Status {Status} was not made available again.",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validatingEntity.Status);
return;
}
await base.MakePackageAvailableAsync(validatingEntity, validationSet);
}
public async Task <bool> OtherRecentValidationSetForPackageExists <T>(
IValidatingEntity <T> validatingEntity,
TimeSpan recentDuration,
Guid currentValidationSetTrackingId) where T : class, IEntity
{
var cutoffTimestamp = DateTime.UtcNow - recentDuration;
return(await _validationContext
.PackageValidationSets
.AnyAsync(pvs => pvs.PackageKey == validatingEntity.Key &&
pvs.Created > cutoffTimestamp &&
pvs.ValidationTrackingId != currentValidationSetTrackingId &&
pvs.ValidatingType == validatingEntity.ValidatingType));
}
protected virtual async Task MakePackageAvailableAsync(IValidatingEntity <T> validatingEntity, PackageValidationSet validationSet)
{
// 1) Operate on blob storage.
var copied = await UpdatePublicPackageAsync(validationSet);
// 2) Update the package's blob metadata in the public blob storage container.
var metadata = await _packageFileService.UpdatePackageBlobMetadataAsync(validationSet);
// 3) Update the package's blob properties in the public blob storage container.
await _packageFileService.UpdatePackageBlobPropertiesAsync(validationSet);
// 3.5) Allow descendants to do their own things before we update the database
await OnBeforeUpdateDatabaseToMakePackageAvailable(validatingEntity, validationSet);
// 4) Operate on the database.
var fromStatus = await MarkPackageAsAvailableAsync(validationSet, validatingEntity, metadata, copied);
// 5) Emit telemetry and clean up.
if (fromStatus != PackageStatus.Available)
{
_telemetryService.TrackPackageStatusChange(fromStatus, PackageStatus.Available);
_logger.LogInformation("Deleting from the source for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
await _packageFileService.DeleteValidationPackageFileAsync(validationSet);
}
// 5) Verify the package still exists (we've had bugs here before).
if (validatingEntity.Status == PackageStatus.Available &&
!await _packageFileService.DoesPackageFileExistAsync(validationSet))
{
var validationPackageAvailable = await _packageFileService.DoesValidationPackageFileExistAsync(validationSet);
_logger.LogWarning("Package {PackageId} {PackageVersion} is marked as available, but does not exist " +
"in public container. Does package exist in validation container: {ExistsInValidation}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationPackageAvailable);
// Report missing package, don't try to fix up anything. This shouldn't happen and needs an investigation.
_telemetryService.TrackMissingNupkgForAvailablePackage(
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId.ToString());
}
}
public Task SetStatusAsync(
IValidatingEntity <T> validatingEntity,
PackageValidationSet validationSet,
PackageStatus status)
{
if (validatingEntity == null)
{
throw new ArgumentNullException(nameof(validatingEntity));
}
if (validationSet == null)
{
throw new ArgumentNullException(nameof(validationSet));
}
if (validatingEntity.Status == PackageStatus.Deleted)
{
throw new ArgumentException(
$"A package in the {nameof(PackageStatus.Deleted)} state cannot be processed.",
nameof(validatingEntity));
}
if (validatingEntity.Status == PackageStatus.Available &&
status == PackageStatus.FailedValidation)
{
throw new ArgumentException(
$"A package cannot transition from {nameof(PackageStatus.Available)} to {nameof(PackageStatus.FailedValidation)}.",
nameof(status));
}
switch (status)
{
case PackageStatus.Available:
return(MakePackageAvailableAsync(validatingEntity, validationSet));
case PackageStatus.FailedValidation:
return(MakePackageFailedValidationAsync(validatingEntity, validationSet));
default:
throw new ArgumentException(
$"A package can only transition to the {nameof(PackageStatus.Available)} or " +
$"{nameof(PackageStatus.FailedValidation)} states.", nameof(status));
}
}
public Facts(ITestOutputHelper output)
{
_packageKey = 23;
_validatorType = "ExampleValidator";
_packageValidation = new PackageValidation
{
Key = Guid.Parse("dc10bf5a-0557-459c-b33f-ea6738b8a044"),
Type = _validatorType,
ValidationStatus = ValidationStatus.Incomplete,
Started = new DateTime(2017, 1, 1, 8, 30, 0, DateTimeKind.Utc),
PackageValidationIssues = new List <PackageValidationIssue>(),
PackageValidationSet = new PackageValidationSet(),
};
var package = new Package()
{
Key = _packageKey
};
_packageValidatingEntity = new PackageValidatingEntity(package);
_nupkgUrl = "https://example/nuget.versioning.4.3.0.nupkg";
_entitiesContext = new Mock <IValidationEntitiesContext>();
_entitiesContext
.Setup(x => x.PackageValidationSets)
.Returns(DbSetMockFactory.Create <PackageValidationSet>());
_entitiesContext
.Setup(x => x.PackageValidations)
.Returns(DbSetMockFactory.Create <PackageValidation>());
_packageFileService = new Mock <IValidationFileService>();
_validatorProvider = new Mock <IValidatorProvider>();
_telemetryService = new Mock <ITelemetryService>();
_loggerFactory = new LoggerFactory();
_loggerFactory.AddXunit(output);
InitializeTarget();
}
private async Task ScheduleCheckIfNotTimedOut(
PackageValidationSet validationSet,
IValidatingEntity <T> validatingEntity,
bool scheduleNextCheck,
bool tooLongNotificationAllowed)
{
var validationSetDuration = await UpdateValidationDurationAsync(validationSet, validatingEntity, tooLongNotificationAllowed);
// Schedule another check if we haven't reached the validation set timeout yet.
if (validationSetDuration <= _validationConfiguration.TimeoutValidationSetAfter)
{
if (scheduleNextCheck)
{
var messageData = PackageValidationMessageData.NewProcessValidationSet(
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId,
validationSet.ValidatingType,
entityKey: validationSet.PackageKey);
var postponeUntil = DateTimeOffset.UtcNow + _validationConfiguration.ValidationMessageRecheckPeriod;
await _validationEnqueuer.SendMessageAsync(messageData, postponeUntil);
}
}
else
{
_logger.LogWarning("Abandoning checking status of validation set {ValidationTrackingId} for " +
"package {PackageId} {PackageVersion} because it took too long (Duration: {Duration}, CutOffDuration: {CutOffDuration})",
validationSet.ValidationTrackingId,
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSetDuration,
_validationConfiguration.TimeoutValidationSetAfter);
_telemetryService.TrackValidationSetTimeout(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId);
}
}
private PackageValidationSet InitializeValidationSet(ProcessValidationSetData message, IValidatingEntity <T> validatingEntity)
{
// If message would have the package Key the package will not need to be passed as an argument
_logger.LogInformation("Initializing validation set {ValidationSetId} for package {PackageId} {PackageVersion} (package key {PackageKey})",
message.ValidationTrackingId,
message.PackageId,
message.PackageVersion,
validatingEntity.Key);
var now = DateTime.UtcNow;
var validationSet = new PackageValidationSet
{
Created = now,
PackageId = message.PackageId,
PackageNormalizedVersion = message.PackageNormalizedVersion,
PackageKey = validatingEntity.Key,
PackageValidations = new List <PackageValidation>(),
Updated = now,
ValidationTrackingId = message.ValidationTrackingId,
ValidatingType = message.ValidatingType,
ValidationSetStatus = ValidationSetStatus.InProgress,
};
var validationsToStart = _validationConfiguration
.Validations
.Where(v => v.ShouldStart);
foreach (var validation in validationsToStart)
{
var packageValidation = new PackageValidation
{
PackageValidationSet = validationSet,
ValidationStatus = ValidationStatus.NotStarted,
Type = validation.Name,
ValidationStatusTimestamp = now,
};
validationSet.PackageValidations.Add(packageValidation);
}
return(validationSet);
}
public async Task ProcessValidationOutcomeAsync(PackageValidationSet validationSet, IValidatingEntity <T> validatingEntity, ValidationSetProcessorResult currentCallStats)
{
var failedValidations = GetFailedValidations(validationSet);
if (failedValidations.Any())
{
_logger.LogWarning("Some validations failed for package {PackageId} {PackageVersion}, validation set {ValidationSetId}: {FailedValidations}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId,
failedValidations.Select(x => x.Type).ToList());
// The only way we can move to the failed validation state is if the package is currently in the
// validating state. This has a beneficial side effect of only sending a failed validation email to the
// customer when the package first moves to the failed validation state. If an admin comes along and
// revalidates the package and the package fails validation again, we don't want another email going
// out since that would be noisy for the customer.
if (validatingEntity.Status == PackageStatus.Validating)
{
await _packageStateProcessor.SetStatusAsync(validatingEntity, validationSet, PackageStatus.FailedValidation);
await MarkValidationSetAsCompletedAsync(validationSet);
await _messageService.SendValidationFailedMessageAsync(validatingEntity.EntityRecord, validationSet);
}
else
{
// The case when validation fails while PackageStatus not validating is the case of
// manual revalidation. In this case we don't want to take package down automatically
// and let the person who requested revalidation to decide how to proceed. Ops will be
// alerted by failed validation monitoring.
_logger.LogInformation("Package {PackageId} {PackageVersion} was {PackageStatus} when validation set {ValidationSetId} failed. Will not mark it as failed.",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validatingEntity.Status,
validationSet.ValidationTrackingId);
await MarkValidationSetAsCompletedAsync(validationSet);
}
TrackValidationSetCompletion(validationSet, isSuccess: false);
await _packageFileService.DeletePackageForValidationSetAsync(validationSet);
}
else if (AllRequiredValidationsSucceeded(validationSet))
{
_logger.LogInformation("All validations are complete for the package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
var fromStatus = validatingEntity.Status;
// Always set the package status to available so that processors can have a chance to fix packages
// that are already available. Processors should no-op when their work is already done, so the
// modification of an already available package should be rare. The most common case for this is if
// the processor has never been run on a package that was published before the processor was
// implemented. In this case, the processor has to play catch-up.
await _packageStateProcessor.SetStatusAsync(validatingEntity, validationSet, PackageStatus.Available);
var areOptionalValidationsRunning = AreOptionalValidationsRunning(validationSet);
if (!areOptionalValidationsRunning)
{
await MarkValidationSetAsCompletedAsync(validationSet);
}
// Only send the email when first transitioning into the Available state.
if (fromStatus != PackageStatus.Available)
{
await _messageService.SendPublishedMessageAsync(validatingEntity.EntityRecord);
}
if (currentCallStats.AnyRequiredValidationSucceeded)
{
TrackValidationSetCompletion(validationSet, isSuccess: true);
}
if (areOptionalValidationsRunning)
{
await ScheduleCheckIfNotTimedOut(validationSet, validatingEntity, tooLongNotificationAllowed : false);
}
// TODO: implement delayed cleanup that would allow internal services
// to access original packages for some time after package become available:
// https://github.com/NuGet/Engineering/issues/2506
}
else
{
await ScheduleCheckIfNotTimedOut(validationSet, validatingEntity, tooLongNotificationAllowed : true);
}
}
private async Task <TimeSpan> UpdateValidationDurationAsync(PackageValidationSet validationSet, IValidatingEntity <T> validatingEntity, bool tooLongNotificationAllowed)
{
// There are no failed validations and some validations are still in progress. Update
// the validation set's Updated field and send a notice if the validation set is taking
// too long to complete.
var previousUpdateTime = validationSet.Updated;
await _validationStorageService.UpdateValidationSetAsync(validationSet);
var validationSetDuration = validationSet.Updated - validationSet.Created;
var previousDuration = previousUpdateTime - validationSet.Created;
// Only send a "validating taking too long" notice once. This is ensured by verifying this is
// the package's first validation set and that this is the first time the validation set duration
// is greater than the configured threshold. Service Bus message duplication for a single validation
// set will not cause multiple notices to be sent due to the row version on PackageValidationSet.
if (tooLongNotificationAllowed &&
validatingEntity.Status == PackageStatus.Validating &&
validationSetDuration > _validationConfiguration.ValidationSetNotificationTimeout &&
previousDuration <= _validationConfiguration.ValidationSetNotificationTimeout &&
await _validationStorageService.GetValidationSetCountAsync(validatingEntity) == 1)
{
_logger.LogWarning("Sending message that validation set {ValidationTrackingId} for package {PackageId} {PackageVersion} is taking too long",
validationSet.ValidationTrackingId,
validationSet.PackageId,
validationSet.PackageNormalizedVersion);
await _messageService.SendValidationTakingTooLongMessageAsync(validatingEntity.EntityRecord);
_telemetryService.TrackSentValidationTakingTooLongMessage(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId);
}
// Track any validations that are past their expected thresholds.
var timedOutValidations = GetIncompleteTimedOutValidations(validationSet);
if (timedOutValidations.Any())
{
foreach (var validation in timedOutValidations)
{
var duration = DateTime.UtcNow - validation.Started;
_logger.LogWarning("Validation {Validation} for package {PackageId} {PackageVersion} is past its expected duration after {Duration}",
validation.Type,
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
duration);
_telemetryService.TrackValidatorTimeout(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, validation.Type);
}
}
return(validationSetDuration);
}
/// <summary>
/// Allows descendants to do additional cleanup on failure to update DB when marking package as available.
/// Only called if package was copied to public container before trying to update DB.
/// </summary>
/// <param name="validatingEntity">Entity being marked as available.</param>
/// <param name="validationSet">Validation set that was completed.</param>
protected virtual Task OnCleanupAfterDatabaseUpdateFailure(
IValidatingEntity <T> validatingEntity,
PackageValidationSet validationSet)
{
return(Task.CompletedTask);
}
private async Task <PackageValidationSet> PersistValidationSetAsync(PackageValidationSet validationSet, IValidatingEntity <T> validatingEntity)
{
_logger.LogInformation("Persisting validation set {ValidationSetId} for package {PackageId} {PackageVersion} (package key {PackageKey})",
validationSet.ValidationTrackingId,
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validatingEntity.Key);
var persistedValidationSet = await _validationStorageService.CreateValidationSetAsync(validationSet);
// Only track the validation set creation time when this is the first validation set to be created for that
// package. There will be more than one validation set when an admin has requested a manual revalidation.
// This can happen much later than when the package was created so the duration is less interesting in that
// case.
if (await _validationStorageService.GetValidationSetCountAsync(validatingEntity) == 1)
{
_telemetryService.TrackDurationToValidationSetCreation(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, validationSet.Created - validatingEntity.Created);
}
return(persistedValidationSet);
}
/// <summary>
/// Allows descendants to do additional operations before database is updated to mark package as available.
/// </summary>
/// <param name="validatingEntity">Entity being marked as available.</param>
/// <param name="validationSet">Validation set that was completed.</param>
protected virtual Task OnBeforeUpdateDatabaseToMakePackageAvailable(IValidatingEntity <T> validatingEntity, PackageValidationSet validationSet)
{
return(Task.CompletedTask);
}
public async Task <PackageValidationSet> TryGetOrCreateValidationSetAsync(ProcessValidationSetData message, IValidatingEntity <T> validatingEntity)
{
var validationSet = await _validationStorageService.GetValidationSetAsync(message.ValidationTrackingId);
if (validationSet == null)
{
var shouldSkip = await _validationStorageService.OtherRecentValidationSetForPackageExists(
validatingEntity,
_validationConfiguration.NewValidationRequestDeduplicationWindow,
message.ValidationTrackingId);
if (shouldSkip)
{
return(null);
}
validationSet = InitializeValidationSet(message, validatingEntity);
if (validatingEntity.Status == PackageStatus.Available)
{
var packageETag = await _packageFileService.CopyPackageFileForValidationSetAsync(validationSet);
// This indicates that the package in the package container is expected to not change.
validationSet.PackageETag = packageETag;
}
else
{
await _packageFileService.CopyValidationPackageForValidationSetAsync(validationSet);
// A symbols package for the same id and version can be re-submitted.
// When this happens a new validation is submitted. After validation the new symbols package will overwrite the old symbols package.
// Because of this when a new validation for a symbols package is received it can already exist a symbols package in the public symbols container.
if (validatingEntity.ValidatingType == ValidatingType.SymbolPackage)
{
validationSet.PackageETag = await _packageFileService.GetPublicPackageBlobETagOrNullAsync(validationSet);
}
else
{
// This indicates that the package in the packages container is expected to not exist (i.e. it has
// has no etag at all).
validationSet.PackageETag = null;
}
}
// If there are any processors in the validation set, back up the original. We back up from the
// validation set copy to avoid concurrency issues.
if (validationSet.PackageValidations.Any(x => _validatorProvider.IsNuGetProcessor(x.Type)))
{
await _packageFileService.BackupPackageFileFromValidationSetPackageAsync(validationSet, _sasDefinitionConfiguration.ValidationSetProviderSasDefinition);
}
validationSet = await PersistValidationSetAsync(validationSet, validatingEntity);
}
else
{
var sameKey = validatingEntity.Key == validationSet.PackageKey;
if (!sameKey)
{
throw new InvalidOperationException($"Validation set key ({validationSet.PackageKey}) " +
$"does not match expected {validatingEntity.EntityRecord.GetType().Name} key ({validatingEntity.Key}).");
}
}
return(validationSet);
}
public async Task <int> GetValidationSetCountAsync <T>(IValidatingEntity <T> validatingEntity) where T : class, IEntity
{
return(await _validationContext
.PackageValidationSets
.CountAsync(x => x.PackageKey == validatingEntity.Key && x.ValidatingType == validatingEntity.ValidatingType));
}
