public void GenerateIfMatchCondition(string etag)
{
var actual = AccessConditionWrapper.GenerateIfMatchCondition(etag);
Assert.Equal(etag, actual.IfMatchETag);
Assert.Null(actual.IfNoneMatchETag);
}
public async Task <ResultAndAccessCondition <VersionListData> > ReadAsync(string id)
{
var blobReference = Container.GetBlobReference(GetFileName(id));
_logger.LogInformation("Reading the version list for package ID {PackageId}.", id);
VersionListData data;
IAccessCondition accessCondition;
try
{
using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition()))
using (var streamReader = new StreamReader(stream))
using (var jsonTextReader = new JsonTextReader(streamReader))
{
data = Serializer.Deserialize <VersionListData>(jsonTextReader);
}
accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag);
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
data = new VersionListData(new Dictionary <string, VersionPropertiesData>());
accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
}
return(new ResultAndAccessCondition <VersionListData>(data, accessCondition));
}
public async Task <ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > > > ReadLatestIndexedAsync()
{
var stopwatch = Stopwatch.StartNew();
var blobName = GetLatestIndexedBlobName();
var blobReference = Container.GetBlobReference(blobName);
_logger.LogInformation("Reading the latest indexed owners from {BlobName}.", blobName);
var builder = new PackageIdToOwnersBuilder(_logger);
IAccessCondition accessCondition;
try
{
using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition()))
{
accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag);
ReadStream(stream, builder.Add);
}
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
_logger.LogInformation("The blob {BlobName} does not exist.", blobName);
}
var output = new ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > >(
builder.GetResult(),
accessCondition);
stopwatch.Stop();
_telemetryService.TrackReadLatestIndexedOwners(output.Result.Count, stopwatch.Elapsed);
return(output);
}
public async Task CopyValidationSetPackageToPackageFileAsync()
{
var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(_etag);
_fileStorageService
.Setup(x => x.CopyFileAsync(
_validationContainerName,
_validationSetPackageFileName,
_packagesContainerName,
_packageFileName,
accessCondition))
.ReturnsAsync(_etag)
.Verifiable();
await _target.CopyValidationSetPackageToPackageFileAsync(_validationSet, accessCondition);
_fileStorageService.Verify();
}
public async Task <FeatureFlagSaveResult> TrySaveAsync(FeatureFlags flags, string contentId)
{
var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(contentId);
try
{
using (var stream = new MemoryStream())
using (var writer = new StreamWriter(stream))
using (var jsonWriter = new JsonTextWriter(writer))
{
Serializer.Serialize(jsonWriter, flags);
jsonWriter.Flush();
stream.Position = 0;
await _storage.SaveFileAsync(CoreConstants.Folders.ContentFolderName, CoreConstants.FeatureFlagsFileName, stream, accessCondition);
return(FeatureFlagSaveResult.Ok);
}
}
catch (StorageException e) when(e.IsPreconditionFailedException())
{
return(FeatureFlagSaveResult.Conflict);
}
}
private async Task <bool> UpdatePublicPackageAsync(PackageValidationSet validationSet)
{
_logger.LogInformation("Copying .nupkg to public storage for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
// If the validation set contains any processors, we must use the copy of the package that is specific to
// this validation set. We can't use the original validation package because it does not have any of the
// changes that the processors made. If the validation set package does not exist for some reason and there
// are processors in the validation set, this indicates a bug and an exception will be thrown by the copy
// operation below. This will cause the validation queue message to eventually dead-letter at which point
// the on-call person should investigate.
bool copied;
if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type)) ||
await _packageFileService.DoesValidationSetPackageExistAsync(validationSet))
{
IAccessCondition destAccessCondition;
// The package etag will be null if this validation set is expecting the package to not yet exist in
// the packages container.
if (validationSet.PackageETag == null)
{
// This will fail with HTTP 409 if the package already exists. This means that another validation
// set has completed and moved the package into the Available state first, with different package
// content.
destAccessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
_logger.LogInformation(
"Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" +
" the packages container, assuming that the package does not already exist.",
validationSet.ValidationTrackingId,
validationSet.PackageId,
validationSet.PackageNormalizedVersion);
}
else
{
// This will fail with HTTP 412 if the package has been modified by another validation set. This
// would only happen if this validation set and another validation set are operating on a package
// already in the Available state.
destAccessCondition = AccessConditionWrapper.GenerateIfMatchCondition(validationSet.PackageETag);
_logger.LogInformation(
"Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" +
" the packages container, assuming that the package has etag {PackageETag}.",
validationSet.ValidationTrackingId,
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.PackageETag);
}
// Failures here should result in an unhandled exception. This means that this validation set has
// modified the package but is unable to copy the modified package into the packages container because
// another validation set completed first.
await _packageFileService.CopyValidationSetPackageToPackageFileAsync(
validationSet,
destAccessCondition);
copied = true;
}
else
{
_logger.LogInformation(
"The package specific to the validation set does not exist. Falling back to the validation " +
"container for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
try
{
await _packageFileService.CopyValidationPackageToPackageFileAsync(validationSet);
copied = true;
}
catch (InvalidOperationException)
{
// The package already exists in the packages container. This can happen if the DB commit below fails
// and this flow is retried or another validation set for the package completed first. Either way, we
// will later attempt to use the hash from the package in the packages container (the destination).
// In other words, we don't care which copy wins when copying from the validation package because
// we know the package has not been modified.
_logger.LogInformation(
"Package already exists in packages container for {PackageId} {PackageVersion}, validation set {ValidationSetId}",
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
validationSet.ValidationTrackingId);
copied = false;
}
}
return(copied);
}
public IAccessCondition GetIfMatchCondition()
{
return(AccessConditionWrapper.GenerateIfMatchCondition(ETag));
}