private async Task SetValidationStatusAsync(
PackageValidation packageValidation,
IValidationResult validationResult,
DateTime now)
{
if (validationResult.Status != ValidationStatus.Incomplete)
{
AddValidationIssues(packageValidation, validationResult.Issues);
}
if (validationResult.Status == ValidationStatus.Succeeded &&
validationResult.NupkgUrl != null)
{
if (!_validatorProvider.IsProcessor(packageValidation.Type))
{
throw new InvalidOperationException(
$"The validator '{packageValidation.Type}' is not a processor but returned a .nupkg URL as " +
$"part of the validation result.");
}
await _packageFileService.CopyPackageUrlForValidationSetAsync(
packageValidation.PackageValidationSet,
validationResult.NupkgUrl);
}
packageValidation.ValidationStatus = validationResult.Status;
packageValidation.ValidationStatusTimestamp = now;
await _validationContext.SaveChangesAsync();
TrackValidationStatus(packageValidation);
}
private void CheckForCyclesAndParallelProcessors()
{
var processorNames = _configuration
.Validations
.Select(x => x.Name)
.Where(x => _validatorProvider.IsProcessor(x))
.ToList();
TopologicalSort.Validate(_configuration.Validations, processorNames);
}
private async Task <bool> UpdatePublicPackageAsync(PackageValidationSet validationSet)
{
_logger.LogInformation("Copying .nupkg to public storage for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
// If the validation set contains any processors, we must use the copy of the package that is specific to
// this validation set. We can't use the original validation package because it does not have any of the
// changes that the processors made. If the validation set package does not exist for some reason and there
// are processors in the validation set, this indicates a bug and an exception will be thrown by the copy
// operation below. This will cause the validation queue message to eventually dead-letter at which point
// the on-call person should investigate.
bool copied;
if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type)) ||
await _packageFileService.DoesValidationSetPackageExistAsync(validationSet))
{
IAccessCondition destAccessCondition;
// The package etag will be null if this validation set is expecting the package to not yet exist in
// the packages container.
if (validationSet.PackageETag == null)
{
// This will fail with HTTP 409 if the package already exists. This means that another validation
// set has completed and moved the package into the Available state first, with different package
// content.
destAccessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
_logger.LogInformation(
"Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" +
" the packages container, assuming that the package does not already exist.",
validationSet.ValidationTrackingId,
validationSet.PackageId,
validationSet.PackageNormalizedVersion);
}
else
{
// This will fail with HTTP 412 if the package has been modified by another validation set. This
// would only happen if this validation set and another validation set are operating on a package
// already in the Available state.
destAccessCondition = AccessConditionWrapper.GenerateIfMatchCondition(validationSet.PackageETag);
_logger.LogInformation(
"Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" +
" the packages container, assuming that the package has etag {PackageETag}.",
validationSet.ValidationTrackingId,
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.PackageETag);
}
// Failures here should result in an unhandled exception. This means that this validation set has
// modified the package but is unable to copy the modified package into the packages container because
// another validation set completed first.
await _packageFileService.CopyValidationSetPackageToPackageFileAsync(
validationSet,
destAccessCondition);
copied = true;
}
else
{
_logger.LogInformation(
"The package specific to the validation set does not exist. Falling back to the validation " +
"container for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
try
{
await _packageFileService.CopyValidationPackageToPackageFileAsync(validationSet);
copied = true;
}
catch (InvalidOperationException)
{
// The package already exists in the packages container. This can happen if the DB commit below fails
// and this flow is retried or another validation set for the package completed first. Either way, we
// will later attempt to use the hash from the package in the packages container (the destination).
// In other words, we don't care which copy wins when copying from the validation package because
// we know the package has not been modified.
_logger.LogInformation(
"Package already exists in packages container for {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
copied = false;
}
}
return(copied);
}
public async Task <PackageValidationSet> TryGetOrCreateValidationSetAsync(PackageValidationMessageData message, IValidatingEntity <T> validatingEntity)
{
var validationSet = await _validationStorageService.GetValidationSetAsync(message.ValidationTrackingId);
if (validationSet == null)
{
var shouldSkip = await _validationStorageService.OtherRecentValidationSetForPackageExists(
validatingEntity,
_validationConfiguration.NewValidationRequestDeduplicationWindow,
message.ValidationTrackingId);
if (shouldSkip)
{
return(null);
}
validationSet = InitializeValidationSet(message, validatingEntity);
if (validatingEntity.Status == PackageStatus.Available)
{
var packageETag = await _packageFileService.CopyPackageFileForValidationSetAsync(validationSet);
// This indicates that the package in the package container is expected to not change.
validationSet.PackageETag = packageETag;
}
else
{
await _packageFileService.CopyValidationPackageForValidationSetAsync(validationSet);
// A symbols package for the same id and version can be re-submitted.
// When this happens a new validation is submitted. After validation the new symbols package will overwrite the old symbols package.
// Because of this when a new validation for a symbols package is received it can already exist a symbols package in the public symbols container.
if (validatingEntity.ValidatingType == ValidatingType.SymbolPackage)
{
validationSet.PackageETag = await _packageFileService.GetPublicPackageBlobETagOrNullAsync(validationSet);
}
else
{
// This indicates that the package in the packages container is expected to not exist (i.e. it has
// has no etag at all).
validationSet.PackageETag = null;
}
}
// If there are any processors in the validation set, back up the original. We back up from the
// validation set copy to avoid concurrency issues.
if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type)))
{
await _packageFileService.BackupPackageFileFromValidationSetPackageAsync(validationSet);
}
validationSet = await PersistValidationSetAsync(validationSet, validatingEntity);
}
else
{
var sameKey = validatingEntity.Key == validationSet.PackageKey;
if (!sameKey)
{
throw new InvalidOperationException($"Validation set key ({validationSet.PackageKey}) " +
$"does not match expected {validatingEntity.EntityRecord.GetType().Name} key ({validatingEntity.Key}).");
}
}
return(validationSet);
}
public async Task <PackageValidationSet> TryGetOrCreateValidationSetAsync(Guid validationTrackingId, Package package)
{
var validationSet = await _validationStorageService.GetValidationSetAsync(validationTrackingId);
if (validationSet == null)
{
var shouldSkip = await _validationStorageService.OtherRecentValidationSetForPackageExists(
package.Key,
_validationConfiguration.NewValidationRequestDeduplicationWindow,
validationTrackingId);
if (shouldSkip)
{
return(null);
}
validationSet = InitializeValidationSet(validationTrackingId, package);
if (package.PackageStatusKey == PackageStatus.Available)
{
var packageETag = await _packageFileService.CopyPackageFileForValidationSetAsync(validationSet);
// This indicates that the package in the package container is expected to not change.
validationSet.PackageETag = packageETag;
}
else
{
await _packageFileService.CopyValidationPackageForValidationSetAsync(validationSet);
// This indicates that the package in the packages container is expected to not exist (i.e. it has
// has no etag at all).
validationSet.PackageETag = null;
}
// If there are any processors in the validation set, back up the original. We back up from the
// validation set copy to avoid concurrency issues.
if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type)))
{
await _packageFileService.BackupPackageFileFromValidationSetPackageAsync(package, validationSet);
}
validationSet = await PersistValidationSetAsync(validationSet, package);
}
else
{
var sameId = package.PackageRegistration.Id.Equals(
validationSet.PackageId,
StringComparison.InvariantCultureIgnoreCase);
var sameVersion = package.NormalizedVersion.Equals(
validationSet.PackageNormalizedVersion,
StringComparison.InvariantCultureIgnoreCase);
if (!sameId || !sameVersion)
{
throw new InvalidOperationException(
$"Validation set package identity ({validationSet.PackageId} {validationSet.PackageNormalizedVersion})" +
$"does not match expected package identity ({package.PackageRegistration.Id} {package.NormalizedVersion}).");
}
var sameKey = package.Key == validationSet.PackageKey;
if (!sameKey)
{
throw new InvalidOperationException($"Validation set package key ({validationSet.PackageKey}) " +
$"does not match expected package key ({package.Key}).");
}
}
return(validationSet);
}