protected override async Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState) { await this.Location.AzureFile.DownloadRangeToStreamAsync( asyncState.DownloadStream, asyncState.StartOffset, asyncState.Length, null, Utils.GenerateFileRequestOptions(this.Location.FileRequestOptions), Utils.GenerateOperationContext(this.Controller.TransferContext), this.CancellationToken); }
private async Task DownloadRangeAsync(RangeBasedDownloadState asyncState) { Debug.Assert(null != asyncState, "asyncState object expected"); Debug.Assert( this.state == State.Download || this.state == State.Error, "DownloadRangeAsync called, but state isn't Download or Error"); // If a parallel operation caused the controller to be placed in // error state exit early to avoid unnecessary I/O. if (this.state == State.Error) { return; } if (asyncState.Range.HasData) { await Utils.ExecuteXsclApiCallAsync( async() => await this.DoDownloadRangeToStreamAsync(asyncState), this.CancellationToken); } else { // Zero memory buffer. asyncState.DownloadStream.SetAllZero(); } asyncState.DownloadStream.FinishWrite(); asyncState.DownloadStream.ReserveBuffer = true; foreach (var buffer in asyncState.DownloadStream.GetBuffers()) { // Two download streams can refer to the same download buffer instance. It may cause the download // buffer be added into shared transfer data twice if only buffer.Finished is checked here: // Thread A: FinishedWrite() // Thread B: FinishedWrite(), buffer.Finished is true now // Thread A: Check buffer.Finished // Thread B: Check buffer.Finished // Thread A: Add buffer into sharedTransferData // Thread C: Writer remove buffer from sharedTransferData // Thread B: Add buffer into sharedTransferData again // So call MarkAsProcessed to make sure buffer is added exactly once. if (buffer.Finished && buffer.MarkAsProcessed()) { TransferData transferData = new TransferData(this.Scheduler.MemoryManager) { StartOffset = buffer.StartOffset, Length = buffer.Length, MemoryBuffer = buffer.MemoryBuffer }; this.SharedTransferData.AvailableData.TryAdd(buffer.StartOffset, transferData); } } }
protected override async Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState) { AccessCondition accessCondition = Utils.GenerateIfMatchConditionWithCustomerCondition( this.sourceLocation.Blob.Properties.ETag, this.sourceLocation.AccessCondition); await this.sourceLocation.Blob.DownloadRangeToStreamAsync( asyncState.DownloadStream, asyncState.StartOffset, asyncState.Length, accessCondition, Utils.GenerateBlobRequestOptions(this.sourceLocation.BlobRequestOptions), Utils.GenerateOperationContext(this.Controller.TransferContext), this.CancellationToken); }
protected abstract Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState);
private async Task DownloadRangeAsync() { Debug.Assert( this.state == State.Error || this.state == State.Download, "DownloadRangeAsync called, but state isn't Download or Error"); if (Interlocked.CompareExchange(ref workToken, 0, 1) == 0) { return; } if (State.Error == this.state) { // Some thread has set error message, just return here. return; } if (this.nextDownloadIndex < this.rangeList.Count) { Range rangeData = this.rangeList[this.nextDownloadIndex]; int blockSize = this.SharedTransferData.BlockSize; long blockStartOffset = (rangeData.StartOffset / blockSize) * blockSize; long nextBlockStartOffset = Math.Min(blockStartOffset + blockSize, this.SharedTransferData.TotalLength); TransferDownloadStream downloadStream = null; if ((rangeData.StartOffset > blockStartOffset) && (rangeData.EndOffset < nextBlockStartOffset)) { Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned"); downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, (int)(rangeData.StartOffset - blockStartOffset), (int)(rangeData.EndOffset + 1 - rangeData.StartOffset)); } else { // Attempt to reserve memory. If none available we'll // retry some time later. byte[][] memoryBuffer = this.Scheduler.MemoryManager.RequireBuffers(this.SharedTransferData.MemoryChunksRequiredEachTime); if (null == memoryBuffer) { this.SetRangeDownloadHasWork(); return; } if (rangeData.EndOffset >= this.lastTransferOffset) { bool canRead = true; lock (this.transferJob.CheckPoint.TransferWindowLock) { if (this.transferJob.CheckPoint.TransferWindow.Count >= Constants.MaxCountInTransferWindow) { canRead = false; } else { if (this.transferJob.CheckPoint.EntryTransferOffset < this.SharedTransferData.TotalLength) { this.transferJob.CheckPoint.TransferWindow.Add(this.transferJob.CheckPoint.EntryTransferOffset); this.transferJob.CheckPoint.EntryTransferOffset = Math.Min(this.transferJob.CheckPoint.EntryTransferOffset + blockSize, this.SharedTransferData.TotalLength); } } } if (!canRead) { this.Scheduler.MemoryManager.ReleaseBuffers(memoryBuffer); this.SetRangeDownloadHasWork(); return; } } if (rangeData.StartOffset == blockStartOffset) { this.currentDownloadBuffer = new TransferDownloadBuffer(blockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - blockStartOffset), memoryBuffer); downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, 0, (int)(rangeData.EndOffset + 1 - rangeData.StartOffset)); } else { Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned"); TransferDownloadBuffer nextBuffer = new TransferDownloadBuffer(nextBlockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - nextBlockStartOffset), memoryBuffer); downloadStream = new TransferDownloadStream( this.Scheduler.MemoryManager, this.currentDownloadBuffer, (int)(rangeData.StartOffset - blockStartOffset), (int)(nextBlockStartOffset - rangeData.StartOffset), nextBuffer, 0, (int)(rangeData.EndOffset + 1 - nextBlockStartOffset)); this.currentDownloadBuffer = nextBuffer; } } using (downloadStream) { this.nextDownloadIndex++; this.SetRangeDownloadHasWork(); RangeBasedDownloadState rangeBasedDownloadState = new RangeBasedDownloadState { Range = rangeData, DownloadStream = downloadStream }; await this.DownloadRangeAsync(rangeBasedDownloadState); } this.SetChunkFinish(); return; } this.SetRangeDownloadHasWork(); }
protected override async Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState) { AccessCondition accessCondition = Utils.GenerateIfMatchConditionWithCustomerCondition( this.Location.Blob.Properties.ETag, this.Location.AccessCondition); await this.Location.Blob.DownloadRangeToStreamAsync( asyncState.DownloadStream, asyncState.StartOffset, asyncState.Length, accessCondition, Utils.GenerateBlobRequestOptions(this.Location.BlobRequestOptions), Utils.GenerateOperationContext(this.Controller.TransferContext), this.CancellationToken); }
protected override async Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState) { await this.sourceLocation.AzureFile.DownloadRangeToStreamAsync( asyncState.DownloadStream, asyncState.StartOffset, asyncState.Length, null, Utils.GenerateFileRequestOptions(this.sourceLocation.FileRequestOptions), Utils.GenerateOperationContext(this.Controller.TransferContext), this.CancellationToken); }
private async Task DownloadRangeAsync(RangeBasedDownloadState asyncState) { Debug.Assert(null != asyncState, "asyncState object expected"); Debug.Assert( this.state == State.Download || this.state == State.Error, "DownloadRangeAsync called, but state isn't Download or Error"); // If a parallel operation caused the controller to be placed in // error state exit early to avoid unnecessary I/O. if (this.state == State.Error) { return; } if (asyncState.Range.HasData) { await this.DoDownloadRangeToStreamAsync(asyncState); } else { // Zero memory buffer. asyncState.DownloadStream.SetAllZero(); } asyncState.DownloadStream.FinishWrite(); asyncState.DownloadStream.ReserveBuffer = true; foreach (var buffer in asyncState.DownloadStream.GetBuffers()) { // Two download streams can refer to the same download buffer instance. It may cause the download // buffer be added into shared transfer data twice if only buffer.Finished is checked here: // Thread A: FinishedWrite() // Thread B: FinishedWrite(), buffer.Finished is true now // Thread A: Check buffer.Finished // Thread B: Check buffer.Finished // Thread A: Add buffer into sharedTransferData // Thread C: Writer remove buffer from sharedTransferData // Thread B: Add buffer into sharedTransferData again // So call MarkAsProcessed to make sure buffer is added exactly once. if (buffer.Finished && buffer.MarkAsProcessed()) { TransferData transferData = new TransferData(this.Scheduler.MemoryManager) { StartOffset = buffer.StartOffset, Length = buffer.Length, MemoryBuffer = buffer.MemoryBuffer }; this.SharedTransferData.AvailableData.TryAdd(buffer.StartOffset, transferData); } } }
private async Task DownloadRangeAsync() { Debug.Assert( this.state == State.Error || this.state == State.Download, "DownloadRangeAsync called, but state isn't Download or Error"); this.hasWork = false; if (State.Error == this.state) { // Some thread has set error message, just return here. return; } if (this.nextDownloadIndex < this.rangeList.Count) { Range rangeData = this.rangeList[this.nextDownloadIndex]; int blockSize = this.Scheduler.TransferOptions.BlockSize; long blockStartOffset = (rangeData.StartOffset / blockSize) * blockSize; long nextBlockStartOffset = Math.Min(blockStartOffset + blockSize, this.SharedTransferData.TotalLength); TransferDownloadStream downloadStream = null; if ((rangeData.StartOffset > blockStartOffset) && (rangeData.EndOffset < nextBlockStartOffset)) { Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned"); downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, (int)(rangeData.StartOffset - blockStartOffset), (int)(rangeData.EndOffset + 1 - rangeData.StartOffset)); } else { // Attempt to reserve memory. If none available we'll // retry some time later. byte[] memoryBuffer = this.Scheduler.MemoryManager.RequireBuffer(); if (null == memoryBuffer) { this.SetRangeDownloadHasWork(); return; } if (rangeData.EndOffset >= this.lastTransferOffset) { bool canRead = true; lock (this.transferJob.CheckPoint.TransferWindowLock) { if (this.transferJob.CheckPoint.TransferWindow.Count >= Constants.MaxCountInTransferWindow) { canRead = false; } else { if (this.transferJob.CheckPoint.EntryTransferOffset < this.SharedTransferData.TotalLength) { this.transferJob.CheckPoint.TransferWindow.Add(this.transferJob.CheckPoint.EntryTransferOffset); this.transferJob.CheckPoint.EntryTransferOffset = Math.Min(this.transferJob.CheckPoint.EntryTransferOffset + this.Scheduler.TransferOptions.BlockSize, this.SharedTransferData.TotalLength); } } } if (!canRead) { this.Scheduler.MemoryManager.ReleaseBuffer(memoryBuffer); this.SetRangeDownloadHasWork(); return; } } if (rangeData.StartOffset == blockStartOffset) { this.currentDownloadBuffer = new TransferDownloadBuffer(blockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - blockStartOffset), memoryBuffer); downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, 0, (int)(rangeData.EndOffset + 1 - rangeData.StartOffset)); } else { Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned"); TransferDownloadBuffer nextBuffer = new TransferDownloadBuffer(nextBlockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - nextBlockStartOffset), memoryBuffer); downloadStream = new TransferDownloadStream( this.Scheduler.MemoryManager, this.currentDownloadBuffer, (int)(rangeData.StartOffset - blockStartOffset), (int)(nextBlockStartOffset - rangeData.StartOffset), nextBuffer, 0, (int)(rangeData.EndOffset + 1 - nextBlockStartOffset)); this.currentDownloadBuffer = nextBuffer; } } using (downloadStream) { this.nextDownloadIndex++; this.SetRangeDownloadHasWork(); RangeBasedDownloadState rangeBasedDownloadState = new RangeBasedDownloadState { Range = rangeData, DownloadStream = downloadStream }; await this.DownloadRangeAsync(rangeBasedDownloadState); } this.SetChunkFinish(); return; } this.SetRangeDownloadHasWork(); }