/// <summary>
/// The <see cref="CreateIfNotExistsInternal"/> operation creates a new 0-length
/// append blob. If the append blob already exists, the content of
/// the existing append blob will remain unchanged. To add content to the append
/// blob, call the <see cref="AppendBlockAsync"/> operation.
///
/// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />.
/// </summary>
/// <param name="httpHeaders">
/// Optional standard HTTP header properties that can be set for the
/// new append blob.
/// </param>
/// <param name="metadata">
/// Optional custom metadata to set for this append blob.
/// </param>
/// <param name="async">
/// Whether to invoke the operation asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{BlobContentInfo}"/> describing the
/// newly created append blob.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
private async Task <Response <BlobContentInfo> > CreateIfNotExistsInternal(
BlobHttpHeaders httpHeaders,
Metadata metadata,
bool async,
CancellationToken cancellationToken)
{
var conditions = new AppendBlobRequestConditions {
IfNoneMatch = new ETag(Constants.Wildcard)
};
try
{
return(await CreateInternal(
httpHeaders,
metadata,
conditions,
async,
cancellationToken,
Constants.Blob.Append.CreateIfNotExistsOperationName)
.ConfigureAwait(false));
}
catch (RequestFailedException storageRequestFailedException)
when(storageRequestFailedException.ErrorCode == Constants.Blob.AlreadyExists)
{
return(default);
/// <summary>
/// The <see cref="CreateIfNotExistsInternal"/> operation creates a new 0-length
/// append blob. If the append blob already exists, the content of
/// the existing append blob will remain unchanged. To add content to the append
/// blob, call the <see cref="AppendBlockAsync"/> operation.
///
/// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />.
/// </summary>
/// <param name="httpHeaders">
/// Optional standard HTTP header properties that can be set for the
/// new append blob.
/// </param>
/// <param name="metadata">
/// Optional custom metadata to set for this append blob.
/// </param>
/// <param name="async">
/// Whether to invoke the operation asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// If the append blob does not already exist, a <see cref="Response{BlobContentInfo}"/>
/// describing the newly created append blob. Otherwise, <c>null</c>.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
private async Task <Response <BlobContentInfo> > CreateIfNotExistsInternal(
BlobHttpHeaders httpHeaders,
Metadata metadata,
bool async,
CancellationToken cancellationToken)
{
using (Pipeline.BeginLoggingScope(nameof(AppendBlobClient)))
{
Pipeline.LogMethodEnter(
nameof(AppendBlobClient),
message:
$"{nameof(Uri)}: {Uri}\n" +
$"{nameof(httpHeaders)}: {httpHeaders}");
var conditions = new AppendBlobRequestConditions {
IfNoneMatch = new ETag(Constants.Wildcard)
};
try
{
Response <BlobContentInfo> response = await CreateInternal(
httpHeaders,
metadata,
conditions,
async,
cancellationToken,
$"{nameof(AppendBlobClient)}.{nameof(CreateIfNotExists)}")
.ConfigureAwait(false);
return(response);
}
catch (RequestFailedException storageRequestFailedException)
when(storageRequestFailedException.ErrorCode == BlobErrorCode.BlobAlreadyExists)
{
return(default);
private AppendBlobRequestConditions BuildDestinationAccessConditions(
AccessConditionParameters parameters,
bool lease = false,
bool appendPosAndMaxSize = false)
{
var accessConditions = new AppendBlobRequestConditions
{
IfMatch = parameters.Match != null ? new ETag(parameters.Match) : default(ETag?),
IfNoneMatch = parameters.NoneMatch != null ? new ETag(parameters.NoneMatch) : default(ETag?),
IfModifiedSince = parameters.IfModifiedSince,
IfUnmodifiedSince = parameters.IfUnmodifiedSince
};
if (lease)
{
accessConditions.LeaseId = parameters.LeaseId;
}
if (appendPosAndMaxSize)
{
accessConditions.IfAppendPositionEqual = parameters.AppendPosE;
accessConditions.IfMaxSizeLessThanOrEqual = parameters.MaxSizeLTE;
}
return(accessConditions);
}
public async Task AppendBlockAsync_AccessConditions()
{
var garbageLeaseId = GetGarbageLeaseId();
AccessConditionParameters[] testCases = new[]
{
new AccessConditionParameters(),
new AccessConditionParameters {
IfModifiedSince = OldDate
},
new AccessConditionParameters {
IfUnmodifiedSince = NewDate
},
new AccessConditionParameters {
Match = ReceivedETag
},
new AccessConditionParameters {
NoneMatch = GarbageETag
},
new AccessConditionParameters {
LeaseId = ReceivedLeaseId
},
new AccessConditionParameters {
AppendPosE = 0
},
new AccessConditionParameters {
MaxSizeLTE = 100
}
};
foreach (AccessConditionParameters parameters in testCases)
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
await blob.CreateAsync();
var data = GetRandomBuffer(7);
parameters.Match = await SetupBlobMatchCondition(blob, parameters.Match);
parameters.LeaseId = await SetupBlobLeaseCondition(blob, parameters.LeaseId, garbageLeaseId);
AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions(
parameters: parameters,
lease: true,
appendPosAndMaxSize: true);
// Act
using (var stream = new MemoryStream(data))
{
Response <BlobAppendInfo> response = await blob.AppendBlockAsync(
content : stream,
conditions : accessConditions);
// Assert
Assert.IsNotNull(response.GetRawResponse().Headers.RequestId);
}
}
}
public async Task AppendBlockAsync_AccessConditionsFail()
{
var garbageLeaseId = GetGarbageLeaseId();
AccessConditionParameters[] testCases = new[]
{
new AccessConditionParameters {
IfModifiedSince = NewDate
},
new AccessConditionParameters {
IfUnmodifiedSince = OldDate
},
new AccessConditionParameters {
Match = GarbageETag
},
new AccessConditionParameters {
NoneMatch = ReceivedETag
},
new AccessConditionParameters {
LeaseId = garbageLeaseId
},
new AccessConditionParameters {
AppendPosE = 1
},
new AccessConditionParameters {
MaxSizeLTE = 1
}
};
foreach (AccessConditionParameters parameters in testCases)
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
var data = GetRandomBuffer(7);
// AppendBlob needs to exists for us to test CreateAsync() with access conditions
await blob.CreateAsync();
parameters.NoneMatch = await SetupBlobMatchCondition(blob, parameters.NoneMatch);
AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions(
parameters: parameters,
lease: true,
appendPosAndMaxSize: true);
// Act
using (var stream = new MemoryStream(data))
{
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blob.AppendBlockAsync(
content: stream,
conditions: accessConditions),
e => { });
}
}
}
/// <summary>
/// The <see cref="CreateAsync"/> operation creates a new 0-length
/// append blob. The content of any existing blob is overwritten with
/// the newly initialized append blob. To add content to the append
/// blob, call the <see cref="AppendBlockAsync"/> operation.
///
/// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />.
/// </summary>
/// <param name="httpHeaders">
/// Optional standard HTTP header properties that can be set for the
/// new append blob.
/// </param>
/// <param name="metadata">
/// Optional custom metadata to set for this append blob.
/// </param>
/// <param name="conditions">
/// Optional <see cref="AppendBlobRequestConditions"/> to add
/// conditions on the creation of this new append blob.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{BlobContentInfo}"/> describing the
/// newly created append blob.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
public virtual async Task <Response <BlobContentInfo> > CreateAsync(
BlobHttpHeaders httpHeaders = default,
Metadata metadata = default,
AppendBlobRequestConditions conditions = default,
CancellationToken cancellationToken = default) =>
await CreateInternal(
httpHeaders,
metadata,
conditions,
true, // async
cancellationToken)
.ConfigureAwait(false);
/// <summary>
/// The <see cref="Create"/> operation creates a new 0-length
/// append blob. The content of any existing blob is overwritten with
/// the newly initialized append blob. To add content to the append
/// blob, call the <see cref="AppendBlock"/> operation.
///
/// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />.
/// </summary>
/// <param name="httpHeaders">
/// Optional standard HTTP header properties that can be set for the
/// new append blob.
/// </param>
/// <param name="metadata">
/// Optional custom metadata to set for this append blob.
/// </param>
/// <param name="conditions">
/// Optional <see cref="AppendBlobRequestConditions"/> to add
/// conditions on the creation of this new append blob.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{BlobContentInfo}"/> describing the
/// newly created append blob.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
public virtual Response <BlobContentInfo> Create(
BlobHttpHeaders httpHeaders = default,
Metadata metadata = default,
AppendBlobRequestConditions conditions = default,
CancellationToken cancellationToken = default) =>
CreateInternal(
httpHeaders,
metadata,
conditions,
false, // async
cancellationToken)
.EnsureCompleted();
public AppendBlobWriteStream(
AppendBlobClient appendBlobClient,
long bufferSize,
long position,
AppendBlobRequestConditions conditions,
IProgress <long> progressHandler) : base(
position,
bufferSize,
progressHandler)
{
ValidateBufferSize(bufferSize);
_appendBlobClient = appendBlobClient;
_conditions = conditions ?? new AppendBlobRequestConditions();
}
public AppendBlobWriteStream(
AppendBlobClient appendBlobClient,
long bufferSize,
long position,
AppendBlobRequestConditions conditions,
IProgress <long> progressHandler,
UploadTransactionalHashingOptions hashingOptions) : base(
position,
bufferSize,
progressHandler,
hashingOptions)
{
ValidateBufferSize(bufferSize);
_appendBlobClient = appendBlobClient;
_conditions = conditions ?? new AppendBlobRequestConditions();
}
public async Task CreateAsync_AccessConditions()
{
var garbageLeaseId = GetGarbageLeaseId();
AccessConditionParameters[] data = new[]
{
new AccessConditionParameters(),
new AccessConditionParameters {
IfModifiedSince = OldDate
},
new AccessConditionParameters {
IfUnmodifiedSince = NewDate
},
new AccessConditionParameters {
Match = ReceivedETag
},
new AccessConditionParameters {
NoneMatch = GarbageETag
},
new AccessConditionParameters {
LeaseId = ReceivedLeaseId
}
};
foreach (AccessConditionParameters parameters in data)
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
// AppendBlob needs to exists for us to test CreateAsync() with access conditions
await blob.CreateAsync();
parameters.Match = await SetupBlobMatchCondition(blob, parameters.Match);
parameters.LeaseId = await SetupBlobLeaseCondition(blob, parameters.LeaseId, garbageLeaseId);
AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions(
parameters: parameters,
lease: true);
// Act
Response <BlobContentInfo> response = await blob.CreateAsync(conditions : accessConditions);
// Assert
Assert.IsNotNull(response.GetRawResponse().Headers.RequestId);
}
}
public async Task CreateAsync_AccessConditionsFail()
{
var garbageLeaseId = GetGarbageLeaseId();
AccessConditionParameters[] data = new[]
{
new AccessConditionParameters {
IfModifiedSince = NewDate
},
new AccessConditionParameters {
IfUnmodifiedSince = OldDate
},
new AccessConditionParameters {
Match = GarbageETag
},
new AccessConditionParameters {
NoneMatch = ReceivedETag
},
new AccessConditionParameters {
LeaseId = garbageLeaseId
}
};
foreach (AccessConditionParameters parameters in data)
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
// AppendBlob needs to exists for us to test CreateAsync() with access conditions
await blob.CreateAsync();
parameters.NoneMatch = await SetupBlobMatchCondition(blob, parameters.NoneMatch);
AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions(
parameters: parameters,
lease: true);
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blob.CreateAsync(conditions: accessConditions),
e => { });
}
}
protected override async Task <Stream> OpenWriteAsync(
AppendBlobClient client,
bool overwrite,
long?maxDataSize,
Dictionary <string, string> tags,
int?bufferSize = default,
BlobRequestConditions conditions = default,
Dictionary <string, string> metadata = default,
HttpHeaderParameters httpHeaders = default,
IProgress <long> progressHandler = default)
{
if (metadata != default)
{
TestHelper.AssertInconclusiveRecordingFriendly(Recording.Mode, "AppendBlobClient.OpenWriteAsync() does not support metadata.");
}
if (tags != default)
{
TestHelper.AssertInconclusiveRecordingFriendly(Recording.Mode, "AppendBlobClient.OpenWriteAsync() does not support tags.");
}
if (httpHeaders != default)
{
TestHelper.AssertInconclusiveRecordingFriendly(Recording.Mode, "AppendBlobClient.OpenWriteAsync() does not support httpHeaders.");
}
AppendBlobRequestConditions appendConditions = conditions == default ? default : new AppendBlobRequestConditions
{
IfModifiedSince = conditions.IfModifiedSince,
IfUnmodifiedSince = conditions.IfUnmodifiedSince,
IfMatch = conditions.IfMatch,
IfNoneMatch = conditions.IfNoneMatch,
LeaseId = conditions.LeaseId,
};
return(await client.OpenWriteAsync(overwrite, new AppendBlobOpenWriteOptions
{
BufferSize = bufferSize,
OpenConditions = appendConditions,
ProgressHandler = progressHandler
}));
}
/// <summary>
/// The <see cref="CreateInternal"/> operation creates a new 0-length
/// append blob. The content of any existing blob is overwritten with
/// the newly initialized append blob. To add content to the append
/// blob, call the <see cref="AppendBlockAsync"/> operation.
///
/// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />.
/// </summary>
/// <param name="httpHeaders">
/// Optional standard HTTP header properties that can be set for the
/// new append blob.
/// </param>
/// <param name="metadata">
/// Optional custom metadata to set for this append blob.
/// </param>
/// <param name="conditions">
/// Optional <see cref="AppendBlobRequestConditions"/> to add
/// conditions on the creation of this new append blob.
/// </param>
/// <param name="async">
/// Whether to invoke the operation asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <param name="operationName">
/// Optional. To indicate if the name of the operation.
/// </param>
/// <returns>
/// A <see cref="Response{BlobContentInfo}"/> describing the
/// newly created append blob.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
private async Task<Response<BlobContentInfo>> CreateInternal(
BlobHttpHeaders httpHeaders,
Metadata metadata,
AppendBlobRequestConditions conditions,
bool async,
CancellationToken cancellationToken,
string operationName = null)
{
using (Pipeline.BeginLoggingScope(nameof(AppendBlobClient)))
{
Pipeline.LogMethodEnter(
nameof(AppendBlobClient),
message:
$"{nameof(Uri)}: {Uri}\n" +
$"{nameof(httpHeaders)}: {httpHeaders}\n" +
$"{nameof(conditions)}: {conditions}");
try
{
return await BlobRestClient.AppendBlob.CreateAsync(
ClientDiagnostics,
Pipeline,
Uri,
contentLength: default,
public async Task AppendBlockFromUriAsync_AccessConditionsFail()
{
var garbageLeaseId = GetGarbageLeaseId();
AccessConditionParameters[] testCases = new[]
{
new AccessConditionParameters {
IfModifiedSince = NewDate
},
new AccessConditionParameters {
IfUnmodifiedSince = OldDate
},
new AccessConditionParameters {
Match = GarbageETag
},
new AccessConditionParameters {
NoneMatch = ReceivedETag
},
new AccessConditionParameters {
LeaseId = garbageLeaseId
},
new AccessConditionParameters {
AppendPosE = 1
},
new AccessConditionParameters {
MaxSizeLTE = 1
},
new AccessConditionParameters {
SourceIfModifiedSince = NewDate
},
new AccessConditionParameters {
SourceIfUnmodifiedSince = OldDate
},
new AccessConditionParameters {
SourceIfMatch = GarbageETag
},
new AccessConditionParameters {
SourceIfNoneMatch = ReceivedETag
}
};
foreach (AccessConditionParameters parameters in testCases)
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
await test.Container.SetAccessPolicyAsync(PublicAccessType.BlobContainer);
var data = GetRandomBuffer(7);
using (var stream = new MemoryStream(data))
{
AppendBlobClient sourceBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
await sourceBlob.CreateAsync();
await sourceBlob.AppendBlockAsync(stream);
AppendBlobClient destBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
await destBlob.CreateAsync();
parameters.NoneMatch = await SetupBlobMatchCondition(destBlob, parameters.NoneMatch);
parameters.SourceIfNoneMatch = await SetupBlobMatchCondition(sourceBlob, parameters.SourceIfNoneMatch);
AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions(
parameters: parameters,
lease: true,
appendPosAndMaxSize: true);
AppendBlobRequestConditions sourceAccessConditions = BuildSourceAccessConditions(parameters);
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
destBlob.AppendBlockFromUriAsync(
sourceUri: sourceBlob.Uri,
conditions: accessConditions,
sourceConditions: sourceAccessConditions),
actualException => Assert.IsTrue(true)
);
}
}
}
public AzureBlobsLogWriter(BlobContainerClient blobsContainerClient,
string fileName,
bool appendOpen = false)
{
fileName = AzureBlobsLogsInterface.PathFixer(fileName);
_blobsContainerClient = blobsContainerClient;
_logClient = _blobsContainerClient.GetAppendBlobClient(fileName);
ETag currentETag;
if (_previousOpenAttempts.ContainsKey(fileName) && appendOpen)
{
// We've opened this blob before and want to be non-destructive. We don't need to CreateIfNotExists, which could be VERY slow.
currentETag = _logClient.GetProperties().Value.ETag;
}
else
{
try
{
// Create the file non-destructively if needed, guaranteeing write continuity on creation by grabbing the etag of the create, if needed
if (appendOpen)
{
var response = _logClient.CreateIfNotExists();
if (response != null)
{
currentETag = response.Value.ETag;
}
else
{
currentETag = _logClient.GetProperties().Value.ETag;
}
}
else
{
currentETag = _logClient.Create().Value.ETag;
}
}
catch { currentETag = _logClient.GetProperties().Value.ETag; }
}
// Try to grab the blob lease
_leaseClient = _logClient.GetBlobLeaseClient();
// The blob hasn't be touched since the last time. This is a candidate for breaking the lease.
if (_previousOpenAttempts.ContainsKey(fileName) && (_previousOpenAttempts[fileName].ToString().Equals(currentETag.ToString())))
{
_previousOpenAttempts[fileName] = currentETag;
// The blob hasn't been updated. Try to break the lease and reacquire
var requestConditions = new BlobRequestConditions();
requestConditions = new BlobRequestConditions();
requestConditions.IfMatch = currentETag;
// If the condition fails in the break, it's because someone else managed to touch the file, so give up
ETag newETag;
try
{
newETag = _leaseClient.Break(null, requestConditions).Value.ETag;
}
catch (Exception e) { newETag = currentETag; }
var etagCondition = new RequestConditions();
etagCondition.IfMatch = newETag;
// If the condition fails, someone snuck in and grabbed the lock before we could. Give up.
_curLease = _leaseClient.Acquire(TimeSpan.FromSeconds(-1), etagCondition).Value;
}
else
{
// Not a candidate for breaking the lease. Just try to acquire.
_previousOpenAttempts[fileName] = currentETag;
_curLease = _leaseClient.Acquire(TimeSpan.FromSeconds(-1)).Value;
}
_leaseCondition = new AppendBlobRequestConditions();
_leaseCondition.LeaseId = _curLease.LeaseId;
// We got the lease! Set up thread to periodically touch the blob to prevent others from breaking the lease.
_blobMetadata = _logClient.GetProperties().Value.Metadata;
_stopRelockThread = false;
_relockThreadStopped = false;
_leaseRenewThread = new Thread(() =>
{
while (!_stopRelockThread)
{
Thread.Sleep(100);
var response = _logClient.SetMetadata(_blobMetadata, _leaseCondition);
}
_relockThreadStopped = true;
})
{
IsBackground = true
};
_leaseRenewThread.Start();
_bytesToSend = new MemoryStream();
Debug.Assert(_logClient.Exists());
}
