/// <summary> /// The <see cref="CreateIfNotExistsInternal"/> operation creates a new 0-length /// append blob. If the append blob already exists, the content of /// the existing append blob will remain unchanged. To add content to the append /// blob, call the <see cref="AppendBlockAsync"/> operation. /// /// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />. /// </summary> /// <param name="httpHeaders"> /// Optional standard HTTP header properties that can be set for the /// new append blob. /// </param> /// <param name="metadata"> /// Optional custom metadata to set for this append blob. /// </param> /// <param name="async"> /// Whether to invoke the operation asynchronously. /// </param> /// <param name="cancellationToken"> /// Optional <see cref="CancellationToken"/> to propagate /// notifications that the operation should be cancelled. /// </param> /// <returns> /// A <see cref="Response{BlobContentInfo}"/> describing the /// newly created append blob. /// </returns> /// <remarks> /// A <see cref="RequestFailedException"/> will be thrown if /// a failure occurs. /// </remarks> private async Task <Response <BlobContentInfo> > CreateIfNotExistsInternal( BlobHttpHeaders httpHeaders, Metadata metadata, bool async, CancellationToken cancellationToken) { var conditions = new AppendBlobRequestConditions { IfNoneMatch = new ETag(Constants.Wildcard) }; try { return(await CreateInternal( httpHeaders, metadata, conditions, async, cancellationToken, Constants.Blob.Append.CreateIfNotExistsOperationName) .ConfigureAwait(false)); } catch (RequestFailedException storageRequestFailedException) when(storageRequestFailedException.ErrorCode == Constants.Blob.AlreadyExists) { return(default);
/// <summary> /// The <see cref="CreateIfNotExistsInternal"/> operation creates a new 0-length /// append blob. If the append blob already exists, the content of /// the existing append blob will remain unchanged. To add content to the append /// blob, call the <see cref="AppendBlockAsync"/> operation. /// /// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />. /// </summary> /// <param name="httpHeaders"> /// Optional standard HTTP header properties that can be set for the /// new append blob. /// </param> /// <param name="metadata"> /// Optional custom metadata to set for this append blob. /// </param> /// <param name="async"> /// Whether to invoke the operation asynchronously. /// </param> /// <param name="cancellationToken"> /// Optional <see cref="CancellationToken"/> to propagate /// notifications that the operation should be cancelled. /// </param> /// <returns> /// If the append blob does not already exist, a <see cref="Response{BlobContentInfo}"/> /// describing the newly created append blob. Otherwise, <c>null</c>. /// </returns> /// <remarks> /// A <see cref="RequestFailedException"/> will be thrown if /// a failure occurs. /// </remarks> private async Task <Response <BlobContentInfo> > CreateIfNotExistsInternal( BlobHttpHeaders httpHeaders, Metadata metadata, bool async, CancellationToken cancellationToken) { using (Pipeline.BeginLoggingScope(nameof(AppendBlobClient))) { Pipeline.LogMethodEnter( nameof(AppendBlobClient), message: $"{nameof(Uri)}: {Uri}\n" + $"{nameof(httpHeaders)}: {httpHeaders}"); var conditions = new AppendBlobRequestConditions { IfNoneMatch = new ETag(Constants.Wildcard) }; try { Response <BlobContentInfo> response = await CreateInternal( httpHeaders, metadata, conditions, async, cancellationToken, $"{nameof(AppendBlobClient)}.{nameof(CreateIfNotExists)}") .ConfigureAwait(false); return(response); } catch (RequestFailedException storageRequestFailedException) when(storageRequestFailedException.ErrorCode == BlobErrorCode.BlobAlreadyExists) { return(default);
private AppendBlobRequestConditions BuildDestinationAccessConditions( AccessConditionParameters parameters, bool lease = false, bool appendPosAndMaxSize = false) { var accessConditions = new AppendBlobRequestConditions { IfMatch = parameters.Match != null ? new ETag(parameters.Match) : default(ETag?), IfNoneMatch = parameters.NoneMatch != null ? new ETag(parameters.NoneMatch) : default(ETag?), IfModifiedSince = parameters.IfModifiedSince, IfUnmodifiedSince = parameters.IfUnmodifiedSince }; if (lease) { accessConditions.LeaseId = parameters.LeaseId; } if (appendPosAndMaxSize) { accessConditions.IfAppendPositionEqual = parameters.AppendPosE; accessConditions.IfMaxSizeLessThanOrEqual = parameters.MaxSizeLTE; } return(accessConditions); }
public async Task AppendBlockAsync_AccessConditions() { var garbageLeaseId = GetGarbageLeaseId(); AccessConditionParameters[] testCases = new[] { new AccessConditionParameters(), new AccessConditionParameters { IfModifiedSince = OldDate }, new AccessConditionParameters { IfUnmodifiedSince = NewDate }, new AccessConditionParameters { Match = ReceivedETag }, new AccessConditionParameters { NoneMatch = GarbageETag }, new AccessConditionParameters { LeaseId = ReceivedLeaseId }, new AccessConditionParameters { AppendPosE = 0 }, new AccessConditionParameters { MaxSizeLTE = 100 } }; foreach (AccessConditionParameters parameters in testCases) { await using DisposingContainer test = await GetTestContainerAsync(); // Arrange AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); await blob.CreateAsync(); var data = GetRandomBuffer(7); parameters.Match = await SetupBlobMatchCondition(blob, parameters.Match); parameters.LeaseId = await SetupBlobLeaseCondition(blob, parameters.LeaseId, garbageLeaseId); AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions( parameters: parameters, lease: true, appendPosAndMaxSize: true); // Act using (var stream = new MemoryStream(data)) { Response <BlobAppendInfo> response = await blob.AppendBlockAsync( content : stream, conditions : accessConditions); // Assert Assert.IsNotNull(response.GetRawResponse().Headers.RequestId); } } }
public async Task AppendBlockAsync_AccessConditionsFail() { var garbageLeaseId = GetGarbageLeaseId(); AccessConditionParameters[] testCases = new[] { new AccessConditionParameters { IfModifiedSince = NewDate }, new AccessConditionParameters { IfUnmodifiedSince = OldDate }, new AccessConditionParameters { Match = GarbageETag }, new AccessConditionParameters { NoneMatch = ReceivedETag }, new AccessConditionParameters { LeaseId = garbageLeaseId }, new AccessConditionParameters { AppendPosE = 1 }, new AccessConditionParameters { MaxSizeLTE = 1 } }; foreach (AccessConditionParameters parameters in testCases) { await using DisposingContainer test = await GetTestContainerAsync(); // Arrange AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); var data = GetRandomBuffer(7); // AppendBlob needs to exists for us to test CreateAsync() with access conditions await blob.CreateAsync(); parameters.NoneMatch = await SetupBlobMatchCondition(blob, parameters.NoneMatch); AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions( parameters: parameters, lease: true, appendPosAndMaxSize: true); // Act using (var stream = new MemoryStream(data)) { await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>( blob.AppendBlockAsync( content: stream, conditions: accessConditions), e => { }); } } }
/// <summary> /// The <see cref="CreateAsync"/> operation creates a new 0-length /// append blob. The content of any existing blob is overwritten with /// the newly initialized append blob. To add content to the append /// blob, call the <see cref="AppendBlockAsync"/> operation. /// /// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />. /// </summary> /// <param name="httpHeaders"> /// Optional standard HTTP header properties that can be set for the /// new append blob. /// </param> /// <param name="metadata"> /// Optional custom metadata to set for this append blob. /// </param> /// <param name="conditions"> /// Optional <see cref="AppendBlobRequestConditions"/> to add /// conditions on the creation of this new append blob. /// </param> /// <param name="cancellationToken"> /// Optional <see cref="CancellationToken"/> to propagate /// notifications that the operation should be cancelled. /// </param> /// <returns> /// A <see cref="Response{BlobContentInfo}"/> describing the /// newly created append blob. /// </returns> /// <remarks> /// A <see cref="RequestFailedException"/> will be thrown if /// a failure occurs. /// </remarks> public virtual async Task <Response <BlobContentInfo> > CreateAsync( BlobHttpHeaders httpHeaders = default, Metadata metadata = default, AppendBlobRequestConditions conditions = default, CancellationToken cancellationToken = default) => await CreateInternal( httpHeaders, metadata, conditions, true, // async cancellationToken) .ConfigureAwait(false);
/// <summary> /// The <see cref="Create"/> operation creates a new 0-length /// append blob. The content of any existing blob is overwritten with /// the newly initialized append blob. To add content to the append /// blob, call the <see cref="AppendBlock"/> operation. /// /// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />. /// </summary> /// <param name="httpHeaders"> /// Optional standard HTTP header properties that can be set for the /// new append blob. /// </param> /// <param name="metadata"> /// Optional custom metadata to set for this append blob. /// </param> /// <param name="conditions"> /// Optional <see cref="AppendBlobRequestConditions"/> to add /// conditions on the creation of this new append blob. /// </param> /// <param name="cancellationToken"> /// Optional <see cref="CancellationToken"/> to propagate /// notifications that the operation should be cancelled. /// </param> /// <returns> /// A <see cref="Response{BlobContentInfo}"/> describing the /// newly created append blob. /// </returns> /// <remarks> /// A <see cref="RequestFailedException"/> will be thrown if /// a failure occurs. /// </remarks> public virtual Response <BlobContentInfo> Create( BlobHttpHeaders httpHeaders = default, Metadata metadata = default, AppendBlobRequestConditions conditions = default, CancellationToken cancellationToken = default) => CreateInternal( httpHeaders, metadata, conditions, false, // async cancellationToken) .EnsureCompleted();
public AppendBlobWriteStream( AppendBlobClient appendBlobClient, long bufferSize, long position, AppendBlobRequestConditions conditions, IProgress <long> progressHandler) : base( position, bufferSize, progressHandler) { ValidateBufferSize(bufferSize); _appendBlobClient = appendBlobClient; _conditions = conditions ?? new AppendBlobRequestConditions(); }
public AppendBlobWriteStream( AppendBlobClient appendBlobClient, long bufferSize, long position, AppendBlobRequestConditions conditions, IProgress <long> progressHandler, UploadTransactionalHashingOptions hashingOptions) : base( position, bufferSize, progressHandler, hashingOptions) { ValidateBufferSize(bufferSize); _appendBlobClient = appendBlobClient; _conditions = conditions ?? new AppendBlobRequestConditions(); }
public async Task CreateAsync_AccessConditions() { var garbageLeaseId = GetGarbageLeaseId(); AccessConditionParameters[] data = new[] { new AccessConditionParameters(), new AccessConditionParameters { IfModifiedSince = OldDate }, new AccessConditionParameters { IfUnmodifiedSince = NewDate }, new AccessConditionParameters { Match = ReceivedETag }, new AccessConditionParameters { NoneMatch = GarbageETag }, new AccessConditionParameters { LeaseId = ReceivedLeaseId } }; foreach (AccessConditionParameters parameters in data) { await using DisposingContainer test = await GetTestContainerAsync(); // Arrange AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); // AppendBlob needs to exists for us to test CreateAsync() with access conditions await blob.CreateAsync(); parameters.Match = await SetupBlobMatchCondition(blob, parameters.Match); parameters.LeaseId = await SetupBlobLeaseCondition(blob, parameters.LeaseId, garbageLeaseId); AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions( parameters: parameters, lease: true); // Act Response <BlobContentInfo> response = await blob.CreateAsync(conditions : accessConditions); // Assert Assert.IsNotNull(response.GetRawResponse().Headers.RequestId); } }
public async Task CreateAsync_AccessConditionsFail() { var garbageLeaseId = GetGarbageLeaseId(); AccessConditionParameters[] data = new[] { new AccessConditionParameters { IfModifiedSince = NewDate }, new AccessConditionParameters { IfUnmodifiedSince = OldDate }, new AccessConditionParameters { Match = GarbageETag }, new AccessConditionParameters { NoneMatch = ReceivedETag }, new AccessConditionParameters { LeaseId = garbageLeaseId } }; foreach (AccessConditionParameters parameters in data) { await using DisposingContainer test = await GetTestContainerAsync(); // Arrange AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); // AppendBlob needs to exists for us to test CreateAsync() with access conditions await blob.CreateAsync(); parameters.NoneMatch = await SetupBlobMatchCondition(blob, parameters.NoneMatch); AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions( parameters: parameters, lease: true); // Act await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>( blob.CreateAsync(conditions: accessConditions), e => { }); } }
protected override async Task <Stream> OpenWriteAsync( AppendBlobClient client, bool overwrite, long?maxDataSize, Dictionary <string, string> tags, int?bufferSize = default, BlobRequestConditions conditions = default, Dictionary <string, string> metadata = default, HttpHeaderParameters httpHeaders = default, IProgress <long> progressHandler = default) { if (metadata != default) { TestHelper.AssertInconclusiveRecordingFriendly(Recording.Mode, "AppendBlobClient.OpenWriteAsync() does not support metadata."); } if (tags != default) { TestHelper.AssertInconclusiveRecordingFriendly(Recording.Mode, "AppendBlobClient.OpenWriteAsync() does not support tags."); } if (httpHeaders != default) { TestHelper.AssertInconclusiveRecordingFriendly(Recording.Mode, "AppendBlobClient.OpenWriteAsync() does not support httpHeaders."); } AppendBlobRequestConditions appendConditions = conditions == default ? default : new AppendBlobRequestConditions { IfModifiedSince = conditions.IfModifiedSince, IfUnmodifiedSince = conditions.IfUnmodifiedSince, IfMatch = conditions.IfMatch, IfNoneMatch = conditions.IfNoneMatch, LeaseId = conditions.LeaseId, }; return(await client.OpenWriteAsync(overwrite, new AppendBlobOpenWriteOptions { BufferSize = bufferSize, OpenConditions = appendConditions, ProgressHandler = progressHandler })); }
/// <summary> /// The <see cref="CreateInternal"/> operation creates a new 0-length /// append blob. The content of any existing blob is overwritten with /// the newly initialized append blob. To add content to the append /// blob, call the <see cref="AppendBlockAsync"/> operation. /// /// For more information, see <see href="https://docs.microsoft.com/rest/api/storageservices/put-blob" />. /// </summary> /// <param name="httpHeaders"> /// Optional standard HTTP header properties that can be set for the /// new append blob. /// </param> /// <param name="metadata"> /// Optional custom metadata to set for this append blob. /// </param> /// <param name="conditions"> /// Optional <see cref="AppendBlobRequestConditions"/> to add /// conditions on the creation of this new append blob. /// </param> /// <param name="async"> /// Whether to invoke the operation asynchronously. /// </param> /// <param name="cancellationToken"> /// Optional <see cref="CancellationToken"/> to propagate /// notifications that the operation should be cancelled. /// </param> /// <param name="operationName"> /// Optional. To indicate if the name of the operation. /// </param> /// <returns> /// A <see cref="Response{BlobContentInfo}"/> describing the /// newly created append blob. /// </returns> /// <remarks> /// A <see cref="RequestFailedException"/> will be thrown if /// a failure occurs. /// </remarks> private async Task<Response<BlobContentInfo>> CreateInternal( BlobHttpHeaders httpHeaders, Metadata metadata, AppendBlobRequestConditions conditions, bool async, CancellationToken cancellationToken, string operationName = null) { using (Pipeline.BeginLoggingScope(nameof(AppendBlobClient))) { Pipeline.LogMethodEnter( nameof(AppendBlobClient), message: $"{nameof(Uri)}: {Uri}\n" + $"{nameof(httpHeaders)}: {httpHeaders}\n" + $"{nameof(conditions)}: {conditions}"); try { return await BlobRestClient.AppendBlob.CreateAsync( ClientDiagnostics, Pipeline, Uri, contentLength: default,
public async Task AppendBlockFromUriAsync_AccessConditionsFail() { var garbageLeaseId = GetGarbageLeaseId(); AccessConditionParameters[] testCases = new[] { new AccessConditionParameters { IfModifiedSince = NewDate }, new AccessConditionParameters { IfUnmodifiedSince = OldDate }, new AccessConditionParameters { Match = GarbageETag }, new AccessConditionParameters { NoneMatch = ReceivedETag }, new AccessConditionParameters { LeaseId = garbageLeaseId }, new AccessConditionParameters { AppendPosE = 1 }, new AccessConditionParameters { MaxSizeLTE = 1 }, new AccessConditionParameters { SourceIfModifiedSince = NewDate }, new AccessConditionParameters { SourceIfUnmodifiedSince = OldDate }, new AccessConditionParameters { SourceIfMatch = GarbageETag }, new AccessConditionParameters { SourceIfNoneMatch = ReceivedETag } }; foreach (AccessConditionParameters parameters in testCases) { await using DisposingContainer test = await GetTestContainerAsync(); // Arrange await test.Container.SetAccessPolicyAsync(PublicAccessType.BlobContainer); var data = GetRandomBuffer(7); using (var stream = new MemoryStream(data)) { AppendBlobClient sourceBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); await sourceBlob.CreateAsync(); await sourceBlob.AppendBlockAsync(stream); AppendBlobClient destBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); await destBlob.CreateAsync(); parameters.NoneMatch = await SetupBlobMatchCondition(destBlob, parameters.NoneMatch); parameters.SourceIfNoneMatch = await SetupBlobMatchCondition(sourceBlob, parameters.SourceIfNoneMatch); AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions( parameters: parameters, lease: true, appendPosAndMaxSize: true); AppendBlobRequestConditions sourceAccessConditions = BuildSourceAccessConditions(parameters); // Act await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>( destBlob.AppendBlockFromUriAsync( sourceUri: sourceBlob.Uri, conditions: accessConditions, sourceConditions: sourceAccessConditions), actualException => Assert.IsTrue(true) ); } } }
public AzureBlobsLogWriter(BlobContainerClient blobsContainerClient, string fileName, bool appendOpen = false) { fileName = AzureBlobsLogsInterface.PathFixer(fileName); _blobsContainerClient = blobsContainerClient; _logClient = _blobsContainerClient.GetAppendBlobClient(fileName); ETag currentETag; if (_previousOpenAttempts.ContainsKey(fileName) && appendOpen) { // We've opened this blob before and want to be non-destructive. We don't need to CreateIfNotExists, which could be VERY slow. currentETag = _logClient.GetProperties().Value.ETag; } else { try { // Create the file non-destructively if needed, guaranteeing write continuity on creation by grabbing the etag of the create, if needed if (appendOpen) { var response = _logClient.CreateIfNotExists(); if (response != null) { currentETag = response.Value.ETag; } else { currentETag = _logClient.GetProperties().Value.ETag; } } else { currentETag = _logClient.Create().Value.ETag; } } catch { currentETag = _logClient.GetProperties().Value.ETag; } } // Try to grab the blob lease _leaseClient = _logClient.GetBlobLeaseClient(); // The blob hasn't be touched since the last time. This is a candidate for breaking the lease. if (_previousOpenAttempts.ContainsKey(fileName) && (_previousOpenAttempts[fileName].ToString().Equals(currentETag.ToString()))) { _previousOpenAttempts[fileName] = currentETag; // The blob hasn't been updated. Try to break the lease and reacquire var requestConditions = new BlobRequestConditions(); requestConditions = new BlobRequestConditions(); requestConditions.IfMatch = currentETag; // If the condition fails in the break, it's because someone else managed to touch the file, so give up ETag newETag; try { newETag = _leaseClient.Break(null, requestConditions).Value.ETag; } catch (Exception e) { newETag = currentETag; } var etagCondition = new RequestConditions(); etagCondition.IfMatch = newETag; // If the condition fails, someone snuck in and grabbed the lock before we could. Give up. _curLease = _leaseClient.Acquire(TimeSpan.FromSeconds(-1), etagCondition).Value; } else { // Not a candidate for breaking the lease. Just try to acquire. _previousOpenAttempts[fileName] = currentETag; _curLease = _leaseClient.Acquire(TimeSpan.FromSeconds(-1)).Value; } _leaseCondition = new AppendBlobRequestConditions(); _leaseCondition.LeaseId = _curLease.LeaseId; // We got the lease! Set up thread to periodically touch the blob to prevent others from breaking the lease. _blobMetadata = _logClient.GetProperties().Value.Metadata; _stopRelockThread = false; _relockThreadStopped = false; _leaseRenewThread = new Thread(() => { while (!_stopRelockThread) { Thread.Sleep(100); var response = _logClient.SetMetadata(_blobMetadata, _leaseCondition); } _relockThreadStopped = true; }) { IsBackground = true }; _leaseRenewThread.Start(); _bytesToSend = new MemoryStream(); Debug.Assert(_logClient.Exists()); }