public async Task PushesVerifiedPackagesData()
{
HashSet <string> data = null;
IAccessCondition accessCondition = null;
_verifiedPackagesDataClient
.Setup(x => x.ReplaceLatestAsync(It.IsAny <HashSet <string> >(), It.IsAny <IAccessCondition>()))
.Returns(Task.CompletedTask)
.Callback <HashSet <string>, IAccessCondition>((d, a) =>
{
data = d;
accessCondition = a;
});
await _target.ExecuteAsync();
Assert.Same(_initialAuxiliaryData.VerifiedPackages, data);
Assert.Equal("*", accessCondition.IfNoneMatchETag);
Assert.Null(accessCondition.IfMatchETag);
_verifiedPackagesDataClient.Verify(
x => x.ReplaceLatestAsync(It.IsAny <HashSet <string> >(), It.IsAny <IAccessCondition>()),
Times.Once);
}
public async Task ReplaceAsync(string id, VersionListData data, IAccessCondition accessCondition)
{
using (var stream = new MemoryStream())
{
using (var streamWriter = new StreamWriter(
stream,
encoding: new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true),
bufferSize: 1024,
leaveOpen: true))
using (var jsonTextWriter = new JsonTextWriter(streamWriter))
{
Serializer.Serialize(jsonTextWriter, data);
}
stream.Position = 0;
_logger.LogInformation("Replacing the version list for package ID {PackageId}.", id);
var mappedAccessCondition = new AccessCondition
{
IfNoneMatchETag = accessCondition.IfNoneMatchETag,
IfMatchETag = accessCondition.IfMatchETag,
};
var blobReference = Container.GetBlobReference(GetFileName(id));
blobReference.Properties.ContentType = "application/json";
await blobReference.UploadFromStreamAsync(
stream,
mappedAccessCondition);
}
}
public async Task PushesPopularityTransferData()
{
PopularityTransferData data = null;
IAccessCondition accessCondition = null;
_popularityTransferDataClient
.Setup(x => x.ReplaceLatestIndexedAsync(It.IsAny <PopularityTransferData>(), It.IsAny <IAccessCondition>()))
.Returns(Task.CompletedTask)
.Callback <PopularityTransferData, IAccessCondition>((d, a) =>
{
data = d;
accessCondition = a;
});
await _target.ExecuteAsync();
Assert.Same(_initialAuxiliaryData.PopularityTransfers, data);
Assert.Equal("*", accessCondition.IfNoneMatchETag);
Assert.Null(accessCondition.IfMatchETag);
_popularityTransferDataClient.Verify(
x => x.ReplaceLatestIndexedAsync(It.IsAny <PopularityTransferData>(), It.IsAny <IAccessCondition>()),
Times.Once);
}
public Task CopyFileAsync(Uri srcUri, string destFolderName, string destFileName,
IAccessCondition destAccessCondition)
{
// We could theoretically support this by downloading the source URI to the destination path. This is not
// needed today so this method will remain unimplemented until it is needed.
throw new NotImplementedException();
}
public async Task PushesOwnerData()
{
SortedDictionary <string, SortedSet <string> > data = null;
IAccessCondition accessCondition = null;
_ownerDataClient
.Setup(x => x.ReplaceLatestIndexedAsync(It.IsAny <SortedDictionary <string, SortedSet <string> > >(), It.IsAny <IAccessCondition>()))
.Returns(Task.CompletedTask)
.Callback <SortedDictionary <string, SortedSet <string> >, IAccessCondition>((d, a) =>
{
data = d;
accessCondition = a;
});
await _target.ExecuteAsync();
Assert.Same(_initialAuxiliaryData.Owners, data);
Assert.Equal("*", accessCondition.IfNoneMatchETag);
Assert.Null(accessCondition.IfMatchETag);
_ownerDataClient.Verify(
x => x.ReplaceLatestIndexedAsync(It.IsAny <SortedDictionary <string, SortedSet <string> > >(), It.IsAny <IAccessCondition>()),
Times.Once);
}
public async Task ReplaceLatestIndexedAsync(
PopularityTransferData newData,
IAccessCondition accessCondition)
{
using (_telemetryService.TrackReplaceLatestIndexedPopularityTransfers(newData.Count))
{
var blobName = GetLatestIndexedBlobName();
_logger.LogInformation("Replacing the latest indexed popularity transfers from {BlobName}.", blobName);
var mappedAccessCondition = new AccessCondition
{
IfNoneMatchETag = accessCondition.IfNoneMatchETag,
IfMatchETag = accessCondition.IfMatchETag,
};
var blobReference = Container.GetBlobReference(blobName);
using (var stream = await blobReference.OpenWriteAsync(mappedAccessCondition))
using (var streamWriter = new StreamWriter(stream))
using (var jsonTextWriter = new JsonTextWriter(streamWriter))
{
blobReference.Properties.ContentType = "application/json";
Serializer.Serialize(jsonTextWriter, newData);
}
}
}
private static AccessCondition MapAccessCondition(IAccessCondition accessCondition)
{
return(new AccessCondition
{
IfNoneMatchETag = accessCondition.IfNoneMatchETag,
IfMatchETag = accessCondition.IfMatchETag,
});
}
public async Task <AuxiliaryFileResult <DownloadData> > ReadLatestIndexedAsync(
IAccessCondition accessCondition,
StringCache stringCache)
{
var stopwatch = Stopwatch.StartNew();
var blobName = GetLatestIndexedBlobName();
var blobReference = Container.GetBlobReference(blobName);
_logger.LogInformation("Reading the latest indexed downloads from {BlobName}.", blobName);
bool modified;
var downloads = new DownloadData();
AuxiliaryFileMetadata metadata;
try
{
using (var stream = await blobReference.OpenReadAsync(accessCondition))
{
ReadStream(
stream,
(id, version, downloadCount) =>
{
id = stringCache.Dedupe(id);
version = stringCache.Dedupe(version);
downloads.SetDownloadCount(id, version, downloadCount);
});
modified = true;
metadata = new AuxiliaryFileMetadata(
lastModified: new DateTimeOffset(blobReference.LastModifiedUtc, TimeSpan.Zero),
loadDuration: stopwatch.Elapsed,
fileSize: blobReference.Properties.Length,
etag: blobReference.ETag);
}
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotModified)
{
_logger.LogInformation("The blob {BlobName} has not changed.", blobName);
modified = false;
downloads = null;
metadata = null;
}
stopwatch.Stop();
_telemetryService.TrackReadLatestIndexedDownloads(downloads?.Count, modified, stopwatch.Elapsed);
return(new AuxiliaryFileResult <DownloadData>(
modified,
downloads,
metadata));
}
public Task <string> CopyFileAsync(string srcFolderName, string srcFileName, string destFolderName,
string destFileName, IAccessCondition destAccessCondition)
{
if (srcFolderName == null)
{
throw new ArgumentNullException(nameof(srcFolderName));
}
if (srcFileName == null)
{
throw new ArgumentNullException(nameof(srcFileName));
}
if (destFolderName == null)
{
throw new ArgumentNullException(nameof(destFolderName));
}
if (destFileName == null)
{
throw new ArgumentNullException(nameof(destFileName));
}
var srcFilePath = buildPath(srcFolderName, srcFileName);
var destFilePath = buildPath(destFolderName, destFileName);
_fileSystemService.CreateDirectory(Path.GetDirectoryName(destFilePath));
if (!_fileSystemService.FileExists(srcFilePath))
{
throw new InvalidOperationException("Could not copy because source file does not exists");
}
if (_fileSystemService.FileExists(destFilePath))
{
throw new InvalidOperationException("Could not copy because destination file already exists");
}
try
{
_fileSystemService.Copy(srcFilePath, destFilePath, overwrite: false);
}
catch (IOException e)
{
throw new InvalidOperationException("Could not copy because destination file already exists", e);
}
return(Task.FromResult <string>(null));
}
public Task CopyFileAsync(
Uri srcUri,
string destFolderName,
string destFileName,
IAccessCondition destAccessCondition)
{
if (srcUri == null)
{
throw new ArgumentNullException(nameof(srcUri));
}
var srcBlob = _client.GetBlobFromUri(srcUri);
return(CopyFileAsync(srcBlob, destFolderName, destFileName, destAccessCondition));
}
public virtual Task CopyValidationSetPackageToPackageFileAsync(
PackageValidationSet validationSet,
IAccessCondition destAccessCondition)
{
var srcFileName = BuildValidationSetPackageFileName(validationSet,
_fileMetadataService.FileExtension);
var destFileName = BuildFileName(validationSet,
_fileMetadataService.FileSavePathTemplate,
_fileMetadataService.FileExtension);
return(CopyFileAsync(
_fileMetadataService.ValidationFolderName,
srcFileName,
_fileMetadataService.FileFolderName,
destFileName,
destAccessCondition));
}
public async Task <AuxiliaryFileResult <HashSet <string> > > ReadLatestAsync(
IAccessCondition accessCondition,
StringCache stringCache)
{
var stopwatch = Stopwatch.StartNew();
var blobName = GetLatestIndexedBlobName();
var blobReference = Container.GetBlobReference(blobName);
_logger.LogInformation("Reading the latest verified packages from {BlobName}.", blobName);
bool modified;
var data = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
AuxiliaryFileMetadata metadata;
try
{
using (var stream = await blobReference.OpenReadAsync(accessCondition))
{
ReadStream(stream, id => data.Add(stringCache.Dedupe(id)));
modified = true;
metadata = new AuxiliaryFileMetadata(
lastModified: new DateTimeOffset(blobReference.LastModifiedUtc, TimeSpan.Zero),
loadDuration: stopwatch.Elapsed,
fileSize: blobReference.Properties.Length,
etag: blobReference.ETag);
}
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotModified)
{
_logger.LogInformation("The blob {BlobName} has not changed.", blobName);
modified = false;
data = null;
metadata = null;
}
stopwatch.Stop();
_telemetryService.TrackReadLatestVerifiedPackages(data?.Count, modified, stopwatch.Elapsed);
return(new AuxiliaryFileResult <HashSet <string> >(
modified,
data,
metadata));
}
public Task <string> CopyFileAsync(
string srcFolderName,
string srcFileName,
string destFolderName,
string destFileName,
IAccessCondition destAccessCondition)
{
if (srcFolderName == null)
{
throw new ArgumentNullException(nameof(srcFolderName));
}
if (srcFileName == null)
{
throw new ArgumentNullException(nameof(srcFileName));
}
if (destFolderName == null)
{
throw new ArgumentNullException(nameof(destFolderName));
}
if (destFileName == null)
{
throw new ArgumentNullException(nameof(destFileName));
}
var srcFilePath = BuildPath(_configuration.FileStorageDirectory, srcFolderName, srcFileName);
var destFilePath = BuildPath(_configuration.FileStorageDirectory, destFolderName, destFileName);
_fileSystemService.CreateDirectory(Path.GetDirectoryName(destFilePath));
try
{
_fileSystemService.Copy(srcFilePath, destFilePath, overwrite: false);
}
catch (IOException e)
{
throw new FileAlreadyExistsException("Could not copy because destination file already exists", e);
}
return(Task.FromResult <string>(null));
}
public Task CopyValidationSetPackageToPackageFileAsync(
PackageValidationSet validationSet,
IAccessCondition destAccessCondition)
{
var srcFileName = BuildValidationSetPackageFileName(validationSet);
var destFileName = BuildFileName(
validationSet.PackageId,
validationSet.PackageNormalizedVersion,
CoreConstants.PackageFileSavePathTemplate,
CoreConstants.NuGetPackageFileExtension);
return(CopyFileAsync(
CoreConstants.ValidationFolderName,
srcFileName,
CoreConstants.PackagesFolderName,
destFileName,
destAccessCondition));
}
public async Task ReplaceLatestAsync(
HashSet <string> newData,
IAccessCondition accessCondition)
{
using (_telemetryService.TrackReplaceLatestVerifiedPackages(newData.Count))
{
var blobName = GetLatestIndexedBlobName();
_logger.LogInformation("Replacing the latest verified packages from {BlobName}.", blobName);
var blobReference = Container.GetBlobReference(blobName);
using (var stream = await blobReference.OpenWriteAsync(accessCondition))
using (var streamWriter = new StreamWriter(stream))
using (var jsonTextWriter = new JsonTextWriter(streamWriter))
{
blobReference.Properties.ContentType = "application/json";
Serializer.Serialize(jsonTextWriter, newData);
}
}
}
private Task <string> CopyFileAsync(
string srcFolderName,
string srcFileName,
string destFolderName,
string destFileName,
IAccessCondition destAccessCondition)
{
_logger.LogInformation(
"Copying file {SrcFolderName}/{SrcFileName} to {DestFolderName}/{DestFileName}.",
srcFolderName,
srcFileName,
destFolderName,
destFileName);
return(_fileStorageService.CopyFileAsync(
srcFolderName,
srcFileName,
destFolderName,
destFileName,
destAccessCondition));
}
public async Task <string> CopyFileAsync(
string srcFolderName,
string srcFileName,
string destFolderName,
string destFileName,
IAccessCondition destAccessCondition)
{
if (srcFolderName == null)
{
throw new ArgumentNullException(nameof(srcFolderName));
}
if (srcFileName == null)
{
throw new ArgumentNullException(nameof(srcFileName));
}
var srcContainer = await GetContainerAsync(srcFolderName);
var srcBlob = srcContainer.GetBlobReference(srcFileName);
return(await CopyFileAsync(srcBlob, destFolderName, destFileName, destAccessCondition));
}
public ResultAndAccessCondition(T result, IAccessCondition accessCondition)
{
Result = result;
AccessCondition = accessCondition;
}
public async Task SaveFileAsync(string folderName, string fileName, Stream file, IAccessCondition accessConditions)
{
ICloudBlobContainer container = await GetContainerAsync(folderName);
var blob = container.GetBlobReference(fileName);
accessConditions = accessConditions ?? AccessConditionWrapper.GenerateIfNotExistsCondition();
var mappedAccessCondition = new AccessCondition
{
IfNoneMatchETag = accessConditions.IfNoneMatchETag,
IfMatchETag = accessConditions.IfMatchETag,
};
try
{
await blob.UploadFromStreamAsync(file, mappedAccessCondition);
}
catch (StorageException ex) when(ex.IsFileAlreadyExistsException())
{
throw new FileAlreadyExistsException(
String.Format(
CultureInfo.CurrentCulture,
"There is already a blob with name {0} in container {1}.",
fileName,
folderName),
ex);
}
blob.Properties.ContentType = GetContentType(folderName);
await blob.SetPropertiesAsync();
}
private async Task <string> CopyFileAsync(
ISimpleCloudBlob srcBlob,
string destFolderName,
string destFileName,
IAccessCondition destAccessCondition)
{
if (destFolderName == null)
{
throw new ArgumentNullException(nameof(destFolderName));
}
if (destFileName == null)
{
throw new ArgumentNullException(nameof(destFileName));
}
var destContainer = await GetContainerAsync(destFolderName);
var destBlob = destContainer.GetBlobReference(destFileName);
destAccessCondition = destAccessCondition ?? AccessConditionWrapper.GenerateIfNotExistsCondition();
var mappedDestAccessCondition = new AccessCondition
{
IfNoneMatchETag = destAccessCondition.IfNoneMatchETag,
IfMatchETag = destAccessCondition.IfMatchETag,
};
if (!await srcBlob.ExistsAsync())
{
_trace.TraceEvent(
TraceEventType.Warning,
id: 0,
message: $"Before calling FetchAttributesAsync(), the source blob '{srcBlob.Name}' does not exist.");
}
// Determine the source blob etag.
await srcBlob.FetchAttributesAsync();
var srcAccessCondition = AccessCondition.GenerateIfMatchCondition(srcBlob.ETag);
// Check if the destination blob already exists and fetch attributes.
if (await destBlob.ExistsAsync())
{
if (destBlob.CopyState?.Status == CopyStatus.Failed)
{
// If the last copy failed, allow this copy to occur no matter what the caller's destination
// condition is. This is because the source blob is preferable over a failed copy. We use the etag
// of the failed blob to avoid inadvertently replacing a blob that is now valid (i.e. has a
// successful copy status).
_trace.TraceEvent(
TraceEventType.Information,
id: 0,
message: $"Destination blob '{destFolderName}/{destFileName}' already exists but has a " +
$"failed copy status. This blob will be replaced if the etag matches '{destBlob.ETag}'.");
mappedDestAccessCondition = AccessCondition.GenerateIfMatchCondition(destBlob.ETag);
}
else if ((srcBlob.Properties.ContentMD5 != null &&
srcBlob.Properties.ContentMD5 == destBlob.Properties.ContentMD5 &&
srcBlob.Properties.Length == destBlob.Properties.Length))
{
// If the blob hash is the same and the length is the same, no-op the copy.
_trace.TraceEvent(
TraceEventType.Information,
id: 0,
message: $"Destination blob '{destFolderName}/{destFileName}' already has hash " +
$"'{destBlob.Properties.ContentMD5}' and length '{destBlob.Properties.Length}'. The copy " +
$"will be skipped.");
return(srcBlob.ETag);
}
}
_trace.TraceEvent(
TraceEventType.Information,
id: 0,
message: $"Copying of source blob '{srcBlob.Uri}' to '{destFolderName}/{destFileName}' with source " +
$"access condition {Log(srcAccessCondition)} and destination access condition " +
$"{Log(mappedDestAccessCondition)}.");
// Start the server-side copy and wait for it to complete. If "If-None-Match: *" was specified and the
// destination already exists, HTTP 409 is thrown. If "If-Match: ETAG" was specified and the destination
// has changed, HTTP 412 is thrown.
try
{
await destBlob.StartCopyAsync(
srcBlob,
srcAccessCondition,
mappedDestAccessCondition);
}
catch (StorageException ex) when(ex.IsFileAlreadyExistsException())
{
throw new FileAlreadyExistsException(
String.Format(
CultureInfo.CurrentCulture,
"There is already a blob with name {0} in container {1}.",
destFileName,
destFolderName),
ex);
}
var stopwatch = Stopwatch.StartNew();
while (destBlob.CopyState.Status == CopyStatus.Pending &&
stopwatch.Elapsed < MaxCopyDuration)
{
if (!await destBlob.ExistsAsync())
{
_trace.TraceEvent(
TraceEventType.Warning,
id: 0,
message: $"Before calling FetchAttributesAsync(), the destination blob '{destBlob.Name}' does not exist.");
}
await destBlob.FetchAttributesAsync();
await Task.Delay(CopyPollFrequency);
}
if (destBlob.CopyState.Status == CopyStatus.Pending)
{
throw new TimeoutException($"Waiting for the blob copy operation to complete timed out after {MaxCopyDuration.TotalSeconds} seconds.");
}
else if (destBlob.CopyState.Status != CopyStatus.Success)
{
throw new StorageException($"The blob copy operation had copy status {destBlob.CopyState.Status} ({destBlob.CopyState.StatusDescription}).");
}
return(srcBlob.ETag);
}
public static async Task <Stream> OpenWriteAsync(this ISimpleCloudBlob blob, IAccessCondition accessCondition)
{
return(await blob.OpenWriteAsync(MapAccessCondition(accessCondition)));
}
public async Task SaveFileAsync(string folderName, string fileName, Stream file, IAccessCondition condition)
{
await SaveFileAsync(folderName, fileName, file);
}
