public async Task SaveFileAsync(string folderName, string fileName, Stream file, IAccessCondition accessConditions)
{
ICloudBlobContainer container = await GetContainerAsync(folderName);
var blob = container.GetBlobReference(fileName);
accessConditions = accessConditions ?? AccessConditionWrapper.GenerateIfNotExistsCondition();
var mappedAccessCondition = new AccessCondition
{
IfNoneMatchETag = accessConditions.IfNoneMatchETag,
IfMatchETag = accessConditions.IfMatchETag,
};
try
{
await blob.UploadFromStreamAsync(file, mappedAccessCondition);
}
catch (StorageException ex) when(ex.IsFileAlreadyExistsException())
{
throw new FileAlreadyExistsException(
String.Format(
CultureInfo.CurrentCulture,
"There is already a blob with name {0} in container {1}.",
fileName,
folderName),
ex);
}
blob.Properties.ContentType = GetContentType(folderName);
await blob.SetPropertiesAsync();
}
public async Task UsesProvidedMatchETag()
{
// Arrange
AccessCondition srcAccessCondition = null;
AccessCondition destAccessCondition = null;
ISimpleCloudBlob srcBlob = null;
_destBlobMock
.Setup(x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()))
.Returns(Task.FromResult(0))
.Callback <ISimpleCloudBlob, AccessCondition, AccessCondition>((b, s, d) =>
{
srcBlob = b;
srcAccessCondition = s;
destAccessCondition = d;
SetDestCopyStatus(CopyStatus.Success);
});
// Act
await _target.CopyFileAsync(
_srcFolderName,
_srcFileName,
_destFolderName,
_destFileName,
AccessConditionWrapper.GenerateIfMatchCondition("etag!"));
// Assert
_destBlobMock.Verify(
x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()),
Times.Once);
Assert.Equal("etag!", destAccessCondition.IfMatchETag);
Assert.Null(destAccessCondition.IfNoneMatchETag);
}
public async Task NoOpsIfPackageLengthAndHashMatch()
{
// Arrange
SetBlobContentMD5(_srcProperties, "mwgwUC0MwohHxgMmvQzO7A==");
SetBlobLength(_srcProperties, 42);
SetBlobContentMD5(_destProperties, _srcProperties.ContentMD5);
SetBlobLength(_destProperties, _srcProperties.Length);
_destBlobMock
.Setup(x => x.ExistsAsync())
.ReturnsAsync(true);
// Act
await _target.CopyFileAsync(
_srcFolderName,
_srcFileName,
_destFolderName,
_destFileName,
AccessConditionWrapper.GenerateIfNotExistsCondition());
// Assert
_destBlobMock.Verify(
x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()),
Times.Never);
}
public async Task WillCopyTheFileIfDestinationDoesNotExist()
{
// Arrange
AccessCondition srcAccessCondition = null;
AccessCondition destAccessCondition = null;
ISimpleCloudBlob srcBlob = null;
_destBlobMock
.Setup(x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()))
.Returns(Task.FromResult(0))
.Callback <ISimpleCloudBlob, AccessCondition, AccessCondition>((b, s, d) =>
{
srcBlob = b;
srcAccessCondition = s;
destAccessCondition = d;
});
// Act
await _target.CopyFileAsync(
_srcFolderName,
_srcFileName,
_destFolderName,
_destFileName,
AccessConditionWrapper.GenerateIfNotExistsCondition());
// Assert
_destBlobMock.Verify(
x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()),
Times.Once);
Assert.Equal(_srcFileName, srcBlob.Name);
Assert.Equal(_srcETag, srcAccessCondition.IfMatchETag);
Assert.Equal("*", destAccessCondition.IfNoneMatchETag);
}
public async Task WillCopyBlobFromSourceUri()
{
// Arrange
_blobClient
.Setup(x => x.GetBlobFromUri(It.IsAny <Uri>()))
.Returns(_srcBlobMock.Object);
_destBlobMock
.Setup(x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()))
.Returns(Task.FromResult(0))
.Callback <ISimpleCloudBlob, AccessCondition, AccessCondition>((_, __, ___) =>
{
SetDestCopyStatus(CopyStatus.Success);
});
// Act
await _target.CopyFileAsync(
_srcUri,
_destFolderName,
_destFileName,
AccessConditionWrapper.GenerateIfNotExistsCondition());
// Assert
_destBlobMock.Verify(
x => x.StartCopyAsync(_srcBlobMock.Object, It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()),
Times.Once);
_destBlobMock.Verify(
x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()),
Times.Once);
_blobClient.Verify(
x => x.GetBlobFromUri(_srcUri),
Times.Once);
}
public async Task CopiesWhenDestinationHasNotHashButContentsAreTheSame()
{
// Arrange
var srcFolderName = CoreConstants.Folders.ValidationFolderName;
var srcFileName = $"{_prefixA}/src";
var srcContent = "Hello, world.";
var srcSha512 = "AD0C37C31D69B315F3A81F13C8CDE701094AD91725BA1B0DC3199CA9713661B8280" +
"D6EF7E68F133E6211E2E5A9A3150445D76F1708E04521B0EE034F0B0BAF26";
var destFolderName = CoreConstants.Folders.PackagesFolderName;
var destFileName = $"{_prefixB}/dest";
await _targetA.SaveFileAsync(
srcFolderName,
srcFileName,
new MemoryStream(Encoding.ASCII.GetBytes(srcContent)),
overwrite : false);
await _targetA.SetMetadataAsync(srcFolderName, srcFileName, (lazyStream, metadata) =>
{
metadata[CoreConstants.Sha512HashAlgorithmId] = srcSha512;
return(Task.FromResult(true));
});
await _targetB.SaveFileAsync(
destFolderName,
destFileName,
new MemoryStream(Encoding.ASCII.GetBytes(srcContent)),
overwrite : false);
var originalDestFileReference = await _targetB.GetFileReferenceAsync(destFolderName, destFileName);
var originalDestETag = originalDestFileReference.ContentId;
var srcUri = await _targetA.GetFileReadUriAsync(
srcFolderName,
srcFileName,
DateTimeOffset.UtcNow.AddHours(1));
var destAccessCondition = AccessConditionWrapper.GenerateEmptyCondition();
// Act
await _targetB.CopyFileAsync(
srcUri,
destFolderName,
destFileName,
destAccessCondition);
// Assert
var finalDestFileReference = await _targetB.GetFileReferenceAsync(destFolderName, destFileName);
var finalDestETag = finalDestFileReference.ContentId;
Assert.NotEqual(originalDestETag, finalDestETag);
}
public async Task CopiesWhenDestinationHasNotHashButContentsAreTheSame()
{
// Arrange
var srcFolderName = CoreConstants.Folders.ValidationFolderName;
var srcFileName = $"{_prefixA}/src";
var srcContent = "Hello, world.";
var destFolderName = CoreConstants.Folders.PackagesFolderName;
var destFileName = $"{_prefixB}/dest";
await _targetA.SaveFileAsync(
srcFolderName,
srcFileName,
new MemoryStream(Encoding.ASCII.GetBytes(srcContent)),
overwrite : false);
await _targetB.SaveFileAsync(
destFolderName,
destFileName,
new MemoryStream(Encoding.ASCII.GetBytes(srcContent)),
overwrite : false);
var originalDestFileReference = await _targetB.GetFileReferenceAsync(destFolderName, destFileName);
var originalDestETag = originalDestFileReference.ContentId;
await ClearContentMD5(_blobClientB, destFolderName, destFileName);
var srcUri = await _targetA.GetFileReadUriAsync(
srcFolderName,
srcFileName,
DateTimeOffset.UtcNow.AddHours(1));
var destAccessCondition = AccessConditionWrapper.GenerateEmptyCondition();
// Act
await _targetB.CopyFileAsync(
srcUri,
destFolderName,
destFileName,
destAccessCondition);
// Assert
var finalDestFileReference = await _targetB.GetFileReferenceAsync(destFolderName, destFileName);
var finalDestETag = finalDestFileReference.ContentId;
Assert.NotEqual(originalDestETag, finalDestETag);
}
private static async Task CopyFileWithNamesAsync(
CloudBlobCoreFileStorageService srcService,
string srcFolderName,
string srcFileName,
CloudBlobCoreFileStorageService destService,
string destFolderName,
string destFileName)
{
var destAccessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
await destService.CopyFileAsync(
srcFolderName,
srcFileName,
destFolderName,
destFileName,
destAccessCondition);
}
public async Task WillCopyTheFileIfDestinationHasFailedCopy()
{
// Arrange
AccessCondition srcAccessCondition = null;
AccessCondition destAccessCondition = null;
ISimpleCloudBlob srcBlob = null;
SetDestCopyStatus(CopyStatus.Failed);
_destBlobMock
.Setup(x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()))
.Returns(Task.FromResult(0))
.Callback <ISimpleCloudBlob, AccessCondition, AccessCondition>((b, s, d) =>
{
srcBlob = b;
srcAccessCondition = s;
destAccessCondition = d;
SetDestCopyStatus(CopyStatus.Pending);
});
_destBlobMock
.Setup(x => x.ExistsAsync())
.ReturnsAsync(true);
_destBlobMock
.Setup(x => x.FetchAttributesAsync())
.Returns(Task.FromResult(0))
.Callback(() => SetDestCopyStatus(CopyStatus.Success));
// Act
var srcETag = await _target.CopyFileAsync(
_srcFolderName,
_srcFileName,
_destFolderName,
_destFileName,
AccessConditionWrapper.GenerateIfNotExistsCondition());
// Assert
_destBlobMock.Verify(
x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()),
Times.Once);
Assert.Equal(_srcETag, srcETag);
Assert.Equal(_srcFileName, srcBlob.Name);
Assert.Equal(_srcETag, srcAccessCondition.IfMatchETag);
Assert.Equal(_destETag, destAccessCondition.IfMatchETag);
}
public async Task WillThrowInvalidOperationExceptionForConflict()
{
// Arrange
_destBlobMock
.Setup(x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()))
.Throws(new StorageException(new RequestResult {
HttpStatusCode = (int)HttpStatusCode.Conflict
}, "Conflict!", inner: null));
// Act & Assert
await Assert.ThrowsAsync <InvalidOperationException>(
() => _target.CopyFileAsync(
_srcFolderName,
_srcFileName,
_destFolderName,
_destFileName,
AccessConditionWrapper.GenerateIfNotExistsCondition()));
}
private static async Task CopyFileWithUriAsync(
CloudBlobCoreFileStorageService srcService,
string srcFolderName,
string srcFileName,
CloudBlobCoreFileStorageService destService,
string destFolderName,
string destFileName)
{
var endOfAccess = DateTimeOffset.UtcNow.AddHours(1);
var srcUri = await srcService.GetFileReadUriAsync(srcFolderName, srcFileName, endOfAccess);
var destAccessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
await destService.CopyFileAsync(
srcUri,
destFolderName,
destFileName,
destAccessCondition);
}
public async Task <RevalidationState> MaybeUpdateStateAsync(Func <RevalidationState, bool> updateAction)
{
var internalState = await GetInternalStateAsync();
var fileReference = internalState.FileReference;
var state = internalState.State;
// Only update the state if the update action returns true.
var originalState = new RevalidationState
{
IsInitialized = state.IsInitialized,
IsKillswitchActive = state.IsKillswitchActive,
DesiredPackageEventRate = state.DesiredPackageEventRate,
};
if (!updateAction(state))
{
return(originalState);
}
try
{
var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(fileReference.ContentId);
using (var stream = new MemoryStream())
using (var writer = new StreamWriter(stream))
using (var jsonWriter = new JsonTextWriter(writer))
{
_serializer.Serialize(jsonWriter, state);
jsonWriter.Flush();
stream.Position = 0;
await _storage.SaveFileAsync(CoreConstants.Folders.RevalidationFolderName, StateFileName, stream, accessCondition);
}
return(state);
}
catch (StorageException e) when(e.IsPreconditionFailedException())
{
throw new InvalidOperationException("Failed to update the state blob since the access condition failed", e);
}
}
/// <summary>
/// Asynchronously sets blob metadata.
/// </summary>
/// <param name="folderName">The folder (container) name.</param>
/// <param name="fileName">The blob file name.</param>
/// <param name="updateMetadataAsync">A function which updates a metadata dictionary and returns <c>true</c>
/// for changes to be persisted or <c>false</c> for changes to be discarded.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task SetMetadataAsync(
string folderName,
string fileName,
Func <Lazy <Task <Stream> >, IDictionary <string, string>, Task <bool> > updateMetadataAsync)
{
if (folderName == null)
{
throw new ArgumentNullException(nameof(folderName));
}
if (fileName == null)
{
throw new ArgumentNullException(nameof(fileName));
}
if (updateMetadataAsync == null)
{
throw new ArgumentNullException(nameof(updateMetadataAsync));
}
var container = await GetContainerAsync(folderName);
var blob = container.GetBlobReference(fileName);
await blob.FetchAttributesAsync();
var lazyStream = new Lazy <Task <Stream> >(() => GetFileAsync(folderName, fileName));
var wasUpdated = await updateMetadataAsync(lazyStream, blob.Metadata);
if (wasUpdated)
{
var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blob.ETag);
var mappedAccessCondition = new AccessCondition
{
IfNoneMatchETag = accessCondition.IfNoneMatchETag,
IfMatchETag = accessCondition.IfMatchETag
};
await blob.SetMetadataAsync(mappedAccessCondition);
}
}
public async Task ThrowsIfCopyOperationFails()
{
// Arrange
_destBlobMock
.Setup(x => x.StartCopyAsync(It.IsAny <ISimpleCloudBlob>(), It.IsAny <AccessCondition>(), It.IsAny <AccessCondition>()))
.Returns(Task.FromResult(0))
.Callback <ISimpleCloudBlob, AccessCondition, AccessCondition>((_, __, ___) =>
{
SetDestCopyStatus(CopyStatus.Failed);
});
// Act & Assert
var ex = await Assert.ThrowsAsync <StorageException>(
() => _target.CopyFileAsync(
_srcFolderName,
_srcFileName,
_destFolderName,
_destFileName,
AccessConditionWrapper.GenerateIfNotExistsCondition()));
Assert.Contains("The blob copy operation had copy status Failed", ex.Message);
}
private async Task <string> CopyFileAsync(
ISimpleCloudBlob srcBlob,
string destFolderName,
string destFileName,
IAccessCondition destAccessCondition)
{
if (destFolderName == null)
{
throw new ArgumentNullException(nameof(destFolderName));
}
if (destFileName == null)
{
throw new ArgumentNullException(nameof(destFileName));
}
var destContainer = await GetContainerAsync(destFolderName);
var destBlob = destContainer.GetBlobReference(destFileName);
destAccessCondition = destAccessCondition ?? AccessConditionWrapper.GenerateIfNotExistsCondition();
var mappedDestAccessCondition = new AccessCondition
{
IfNoneMatchETag = destAccessCondition.IfNoneMatchETag,
IfMatchETag = destAccessCondition.IfMatchETag,
};
if (!await srcBlob.ExistsAsync())
{
_trace.TraceEvent(
TraceEventType.Warning,
id: 0,
message: $"Before calling FetchAttributesAsync(), the source blob '{srcBlob.Name}' does not exist.");
}
// Determine the source blob etag.
await srcBlob.FetchAttributesAsync();
var srcAccessCondition = AccessCondition.GenerateIfMatchCondition(srcBlob.ETag);
// Check if the destination blob already exists and fetch attributes.
if (await destBlob.ExistsAsync())
{
if (destBlob.CopyState?.Status == CopyStatus.Failed)
{
// If the last copy failed, allow this copy to occur no matter what the caller's destination
// condition is. This is because the source blob is preferable over a failed copy. We use the etag
// of the failed blob to avoid inadvertently replacing a blob that is now valid (i.e. has a
// successful copy status).
_trace.TraceEvent(
TraceEventType.Information,
id: 0,
message: $"Destination blob '{destFolderName}/{destFileName}' already exists but has a " +
$"failed copy status. This blob will be replaced if the etag matches '{destBlob.ETag}'.");
mappedDestAccessCondition = AccessCondition.GenerateIfMatchCondition(destBlob.ETag);
}
else if ((srcBlob.Properties.ContentMD5 != null &&
srcBlob.Properties.ContentMD5 == destBlob.Properties.ContentMD5 &&
srcBlob.Properties.Length == destBlob.Properties.Length))
{
// If the blob hash is the same and the length is the same, no-op the copy.
_trace.TraceEvent(
TraceEventType.Information,
id: 0,
message: $"Destination blob '{destFolderName}/{destFileName}' already has hash " +
$"'{destBlob.Properties.ContentMD5}' and length '{destBlob.Properties.Length}'. The copy " +
$"will be skipped.");
return(srcBlob.ETag);
}
}
_trace.TraceEvent(
TraceEventType.Information,
id: 0,
message: $"Copying of source blob '{srcBlob.Uri}' to '{destFolderName}/{destFileName}' with source " +
$"access condition {Log(srcAccessCondition)} and destination access condition " +
$"{Log(mappedDestAccessCondition)}.");
// Start the server-side copy and wait for it to complete. If "If-None-Match: *" was specified and the
// destination already exists, HTTP 409 is thrown. If "If-Match: ETAG" was specified and the destination
// has changed, HTTP 412 is thrown.
try
{
await destBlob.StartCopyAsync(
srcBlob,
srcAccessCondition,
mappedDestAccessCondition);
}
catch (StorageException ex) when(ex.IsFileAlreadyExistsException())
{
throw new FileAlreadyExistsException(
String.Format(
CultureInfo.CurrentCulture,
"There is already a blob with name {0} in container {1}.",
destFileName,
destFolderName),
ex);
}
var stopwatch = Stopwatch.StartNew();
while (destBlob.CopyState.Status == CopyStatus.Pending &&
stopwatch.Elapsed < MaxCopyDuration)
{
if (!await destBlob.ExistsAsync())
{
_trace.TraceEvent(
TraceEventType.Warning,
id: 0,
message: $"Before calling FetchAttributesAsync(), the destination blob '{destBlob.Name}' does not exist.");
}
await destBlob.FetchAttributesAsync();
await Task.Delay(CopyPollFrequency);
}
if (destBlob.CopyState.Status == CopyStatus.Pending)
{
throw new TimeoutException($"Waiting for the blob copy operation to complete timed out after {MaxCopyDuration.TotalSeconds} seconds.");
}
else if (destBlob.CopyState.Status != CopyStatus.Success)
{
throw new StorageException($"The blob copy operation had copy status {destBlob.CopyState.Status} ({destBlob.CopyState.StatusDescription}).");
}
return(srcBlob.ETag);
}
