protected override async Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState)
{
await this.Location.AzureFile.DownloadRangeToStreamAsync(
asyncState.DownloadStream,
asyncState.StartOffset,
asyncState.Length,
null,
Utils.GenerateFileRequestOptions(this.Location.FileRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken);
}
private async Task DownloadRangeAsync(RangeBasedDownloadState asyncState)
{
Debug.Assert(null != asyncState, "asyncState object expected");
Debug.Assert(
this.state == State.Download || this.state == State.Error,
"DownloadRangeAsync called, but state isn't Download or Error");
// If a parallel operation caused the controller to be placed in
// error state exit early to avoid unnecessary I/O.
if (this.state == State.Error)
{
return;
}
if (asyncState.Range.HasData)
{
await Utils.ExecuteXsclApiCallAsync(
async() => await this.DoDownloadRangeToStreamAsync(asyncState),
this.CancellationToken);
}
else
{
// Zero memory buffer.
asyncState.DownloadStream.SetAllZero();
}
asyncState.DownloadStream.FinishWrite();
asyncState.DownloadStream.ReserveBuffer = true;
foreach (var buffer in asyncState.DownloadStream.GetBuffers())
{
// Two download streams can refer to the same download buffer instance. It may cause the download
// buffer be added into shared transfer data twice if only buffer.Finished is checked here:
// Thread A: FinishedWrite()
// Thread B: FinishedWrite(), buffer.Finished is true now
// Thread A: Check buffer.Finished
// Thread B: Check buffer.Finished
// Thread A: Add buffer into sharedTransferData
// Thread C: Writer remove buffer from sharedTransferData
// Thread B: Add buffer into sharedTransferData again
// So call MarkAsProcessed to make sure buffer is added exactly once.
if (buffer.Finished && buffer.MarkAsProcessed())
{
TransferData transferData = new TransferData(this.Scheduler.MemoryManager)
{
StartOffset = buffer.StartOffset,
Length = buffer.Length,
MemoryBuffer = buffer.MemoryBuffer
};
this.SharedTransferData.AvailableData.TryAdd(buffer.StartOffset, transferData);
}
}
}
protected override async Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState)
{
AccessCondition accessCondition = Utils.GenerateIfMatchConditionWithCustomerCondition(
this.sourceLocation.Blob.Properties.ETag,
this.sourceLocation.AccessCondition);
await this.sourceLocation.Blob.DownloadRangeToStreamAsync(
asyncState.DownloadStream,
asyncState.StartOffset,
asyncState.Length,
accessCondition,
Utils.GenerateBlobRequestOptions(this.sourceLocation.BlobRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken);
}
protected abstract Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState);
private async Task DownloadRangeAsync()
{
Debug.Assert(
this.state == State.Error || this.state == State.Download,
"DownloadRangeAsync called, but state isn't Download or Error");
if (Interlocked.CompareExchange(ref workToken, 0, 1) == 0)
{
return;
}
if (State.Error == this.state)
{
// Some thread has set error message, just return here.
return;
}
if (this.nextDownloadIndex < this.rangeList.Count)
{
Range rangeData = this.rangeList[this.nextDownloadIndex];
int blockSize = this.SharedTransferData.BlockSize;
long blockStartOffset = (rangeData.StartOffset / blockSize) * blockSize;
long nextBlockStartOffset = Math.Min(blockStartOffset + blockSize, this.SharedTransferData.TotalLength);
TransferDownloadStream downloadStream = null;
if ((rangeData.StartOffset > blockStartOffset) && (rangeData.EndOffset < nextBlockStartOffset))
{
Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned");
downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, (int)(rangeData.StartOffset - blockStartOffset), (int)(rangeData.EndOffset + 1 - rangeData.StartOffset));
}
else
{
// Attempt to reserve memory. If none available we'll
// retry some time later.
byte[][] memoryBuffer = this.Scheduler.MemoryManager.RequireBuffers(this.SharedTransferData.MemoryChunksRequiredEachTime);
if (null == memoryBuffer)
{
this.SetRangeDownloadHasWork();
return;
}
if (rangeData.EndOffset >= this.lastTransferOffset)
{
bool canRead = true;
lock (this.transferJob.CheckPoint.TransferWindowLock)
{
if (this.transferJob.CheckPoint.TransferWindow.Count >= Constants.MaxCountInTransferWindow)
{
canRead = false;
}
else
{
if (this.transferJob.CheckPoint.EntryTransferOffset < this.SharedTransferData.TotalLength)
{
this.transferJob.CheckPoint.TransferWindow.Add(this.transferJob.CheckPoint.EntryTransferOffset);
this.transferJob.CheckPoint.EntryTransferOffset = Math.Min(this.transferJob.CheckPoint.EntryTransferOffset + blockSize, this.SharedTransferData.TotalLength);
}
}
}
if (!canRead)
{
this.Scheduler.MemoryManager.ReleaseBuffers(memoryBuffer);
this.SetRangeDownloadHasWork();
return;
}
}
if (rangeData.StartOffset == blockStartOffset)
{
this.currentDownloadBuffer = new TransferDownloadBuffer(blockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - blockStartOffset), memoryBuffer);
downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, 0, (int)(rangeData.EndOffset + 1 - rangeData.StartOffset));
}
else
{
Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned");
TransferDownloadBuffer nextBuffer = new TransferDownloadBuffer(nextBlockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - nextBlockStartOffset), memoryBuffer);
downloadStream = new TransferDownloadStream(
this.Scheduler.MemoryManager,
this.currentDownloadBuffer,
(int)(rangeData.StartOffset - blockStartOffset),
(int)(nextBlockStartOffset - rangeData.StartOffset),
nextBuffer,
0,
(int)(rangeData.EndOffset + 1 - nextBlockStartOffset));
this.currentDownloadBuffer = nextBuffer;
}
}
using (downloadStream)
{
this.nextDownloadIndex++;
this.SetRangeDownloadHasWork();
RangeBasedDownloadState rangeBasedDownloadState = new RangeBasedDownloadState
{
Range = rangeData,
DownloadStream = downloadStream
};
await this.DownloadRangeAsync(rangeBasedDownloadState);
}
this.SetChunkFinish();
return;
}
this.SetRangeDownloadHasWork();
}
protected override async Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState)
{
AccessCondition accessCondition = Utils.GenerateIfMatchConditionWithCustomerCondition(
this.Location.Blob.Properties.ETag,
this.Location.AccessCondition);
await this.Location.Blob.DownloadRangeToStreamAsync(
asyncState.DownloadStream,
asyncState.StartOffset,
asyncState.Length,
accessCondition,
Utils.GenerateBlobRequestOptions(this.Location.BlobRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken);
}
protected override async Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState)
{
await this.sourceLocation.AzureFile.DownloadRangeToStreamAsync(
asyncState.DownloadStream,
asyncState.StartOffset,
asyncState.Length,
null,
Utils.GenerateFileRequestOptions(this.sourceLocation.FileRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken);
}
protected abstract Task DoDownloadRangeToStreamAsync(RangeBasedDownloadState asyncState);
private async Task DownloadRangeAsync(RangeBasedDownloadState asyncState)
{
Debug.Assert(null != asyncState, "asyncState object expected");
Debug.Assert(
this.state == State.Download || this.state == State.Error,
"DownloadRangeAsync called, but state isn't Download or Error");
// If a parallel operation caused the controller to be placed in
// error state exit early to avoid unnecessary I/O.
if (this.state == State.Error)
{
return;
}
if (asyncState.Range.HasData)
{
await this.DoDownloadRangeToStreamAsync(asyncState);
}
else
{
// Zero memory buffer.
asyncState.DownloadStream.SetAllZero();
}
asyncState.DownloadStream.FinishWrite();
asyncState.DownloadStream.ReserveBuffer = true;
foreach (var buffer in asyncState.DownloadStream.GetBuffers())
{
// Two download streams can refer to the same download buffer instance. It may cause the download
// buffer be added into shared transfer data twice if only buffer.Finished is checked here:
// Thread A: FinishedWrite()
// Thread B: FinishedWrite(), buffer.Finished is true now
// Thread A: Check buffer.Finished
// Thread B: Check buffer.Finished
// Thread A: Add buffer into sharedTransferData
// Thread C: Writer remove buffer from sharedTransferData
// Thread B: Add buffer into sharedTransferData again
// So call MarkAsProcessed to make sure buffer is added exactly once.
if (buffer.Finished && buffer.MarkAsProcessed())
{
TransferData transferData = new TransferData(this.Scheduler.MemoryManager)
{
StartOffset = buffer.StartOffset,
Length = buffer.Length,
MemoryBuffer = buffer.MemoryBuffer
};
this.SharedTransferData.AvailableData.TryAdd(buffer.StartOffset, transferData);
}
}
}
private async Task DownloadRangeAsync()
{
Debug.Assert(
this.state == State.Error || this.state == State.Download,
"DownloadRangeAsync called, but state isn't Download or Error");
this.hasWork = false;
if (State.Error == this.state)
{
// Some thread has set error message, just return here.
return;
}
if (this.nextDownloadIndex < this.rangeList.Count)
{
Range rangeData = this.rangeList[this.nextDownloadIndex];
int blockSize = this.Scheduler.TransferOptions.BlockSize;
long blockStartOffset = (rangeData.StartOffset / blockSize) * blockSize;
long nextBlockStartOffset = Math.Min(blockStartOffset + blockSize, this.SharedTransferData.TotalLength);
TransferDownloadStream downloadStream = null;
if ((rangeData.StartOffset > blockStartOffset) && (rangeData.EndOffset < nextBlockStartOffset))
{
Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned");
downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, (int)(rangeData.StartOffset - blockStartOffset), (int)(rangeData.EndOffset + 1 - rangeData.StartOffset));
}
else
{
// Attempt to reserve memory. If none available we'll
// retry some time later.
byte[] memoryBuffer = this.Scheduler.MemoryManager.RequireBuffer();
if (null == memoryBuffer)
{
this.SetRangeDownloadHasWork();
return;
}
if (rangeData.EndOffset >= this.lastTransferOffset)
{
bool canRead = true;
lock (this.transferJob.CheckPoint.TransferWindowLock)
{
if (this.transferJob.CheckPoint.TransferWindow.Count >= Constants.MaxCountInTransferWindow)
{
canRead = false;
}
else
{
if (this.transferJob.CheckPoint.EntryTransferOffset < this.SharedTransferData.TotalLength)
{
this.transferJob.CheckPoint.TransferWindow.Add(this.transferJob.CheckPoint.EntryTransferOffset);
this.transferJob.CheckPoint.EntryTransferOffset = Math.Min(this.transferJob.CheckPoint.EntryTransferOffset + this.Scheduler.TransferOptions.BlockSize, this.SharedTransferData.TotalLength);
}
}
}
if (!canRead)
{
this.Scheduler.MemoryManager.ReleaseBuffer(memoryBuffer);
this.SetRangeDownloadHasWork();
return;
}
}
if (rangeData.StartOffset == blockStartOffset)
{
this.currentDownloadBuffer = new TransferDownloadBuffer(blockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - blockStartOffset), memoryBuffer);
downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, 0, (int)(rangeData.EndOffset + 1 - rangeData.StartOffset));
}
else
{
Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned");
TransferDownloadBuffer nextBuffer = new TransferDownloadBuffer(nextBlockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - nextBlockStartOffset), memoryBuffer);
downloadStream = new TransferDownloadStream(
this.Scheduler.MemoryManager,
this.currentDownloadBuffer,
(int)(rangeData.StartOffset - blockStartOffset),
(int)(nextBlockStartOffset - rangeData.StartOffset),
nextBuffer,
0,
(int)(rangeData.EndOffset + 1 - nextBlockStartOffset));
this.currentDownloadBuffer = nextBuffer;
}
}
using (downloadStream)
{
this.nextDownloadIndex++;
this.SetRangeDownloadHasWork();
RangeBasedDownloadState rangeBasedDownloadState = new RangeBasedDownloadState
{
Range = rangeData,
DownloadStream = downloadStream
};
await this.DownloadRangeAsync(rangeBasedDownloadState);
}
this.SetChunkFinish();
return;
}
this.SetRangeDownloadHasWork();
}
