private void AddRangesByCheckPoint(long startOffset, long endOffset, bool hasData, ref bool reachLastTransferOffset, ref int lastTransferWindowIndex)
{
SingleObjectCheckpoint checkpoint = this.transferJob.CheckPoint;
if (reachLastTransferOffset)
{
this.rangeList.AddRange(
new Utils.Range
{
StartOffset = startOffset,
EndOffset = endOffset,
HasData = hasData,
}.SplitRanges(Constants.DefaultTransferChunkSize));
}
else
{
Utils.Range range = new Utils.Range()
{
StartOffset = -1,
HasData = hasData
};
while (lastTransferWindowIndex < checkpoint.TransferWindow.Count)
{
long lastTransferWindowStart = checkpoint.TransferWindow[lastTransferWindowIndex];
long lastTransferWindowEnd = Math.Min(checkpoint.TransferWindow[lastTransferWindowIndex] + this.SharedTransferData.BlockSize - 1, this.SharedTransferData.TotalLength);
if (lastTransferWindowStart <= endOffset)
{
if (-1 == range.StartOffset)
{
// New range
range.StartOffset = Math.Max(lastTransferWindowStart, startOffset);
range.EndOffset = Math.Min(lastTransferWindowEnd, endOffset);
}
else
{
if (range.EndOffset != lastTransferWindowStart - 1)
{
// Store the previous range and create a new one
this.rangeList.AddRange(range.SplitRanges(Constants.DefaultTransferChunkSize));
range = new Utils.Range()
{
StartOffset = Math.Max(lastTransferWindowStart, startOffset),
HasData = hasData
};
}
range.EndOffset = Math.Min(lastTransferWindowEnd, endOffset);
}
if (range.EndOffset == lastTransferWindowEnd)
{
// Reach the end of transfer window, move to next
++lastTransferWindowIndex;
continue;
}
}
break;
}
if (-1 != range.StartOffset)
{
this.rangeList.AddRange(range.SplitRanges(Constants.DefaultTransferChunkSize));
}
if (checkpoint.EntryTransferOffset <= endOffset + 1)
{
reachLastTransferOffset = true;
if (checkpoint.EntryTransferOffset <= endOffset)
{
this.rangeList.AddRange(new Utils.Range()
{
StartOffset = checkpoint.EntryTransferOffset,
EndOffset = endOffset,
HasData = hasData,
}.SplitRanges(Constants.DefaultTransferChunkSize));
}
}
}
}
private async Task DownloadRangeAsync()
{
Debug.Assert(
this.state == State.Error || this.state == State.Download,
"DownloadRangeAsync called, but state isn't Download or Error");
this.hasWork = false;
if (State.Error == this.state)
{
// Some thread has set error message, just return here.
return;
}
if (this.nextDownloadIndex < this.rangeList.Count)
{
Utils.Range rangeData = this.rangeList[this.nextDownloadIndex];
int blockSize = this.SharedTransferData.BlockSize;
long blockStartOffset = (rangeData.StartOffset / blockSize) * blockSize;
long nextBlockStartOffset = Math.Min(blockStartOffset + blockSize, this.SharedTransferData.TotalLength);
TransferDownloadStream downloadStream = null;
if ((rangeData.StartOffset > blockStartOffset) && (rangeData.EndOffset < nextBlockStartOffset))
{
Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned");
downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, (int)(rangeData.StartOffset - blockStartOffset), (int)(rangeData.EndOffset + 1 - rangeData.StartOffset));
}
else
{
// Attempt to reserve memory. If none available we'll
// retry some time later.
byte[][] memoryBuffer = this.Scheduler.MemoryManager.RequireBuffers(this.SharedTransferData.MemoryChunksRequiredEachTime);
if (null == memoryBuffer)
{
this.SetRangeDownloadHasWork();
return;
}
if (rangeData.EndOffset >= this.lastTransferOffset)
{
bool canRead = true;
lock (this.transferJob.CheckPoint.TransferWindowLock)
{
if (this.transferJob.CheckPoint.TransferWindow.Count >= Constants.MaxCountInTransferWindow)
{
canRead = false;
}
else
{
if (this.transferJob.CheckPoint.EntryTransferOffset < this.SharedTransferData.TotalLength)
{
this.transferJob.CheckPoint.TransferWindow.Add(this.transferJob.CheckPoint.EntryTransferOffset);
this.transferJob.CheckPoint.EntryTransferOffset = Math.Min(this.transferJob.CheckPoint.EntryTransferOffset + blockSize, this.SharedTransferData.TotalLength);
}
}
}
if (!canRead)
{
this.Scheduler.MemoryManager.ReleaseBuffers(memoryBuffer);
this.SetRangeDownloadHasWork();
return;
}
}
if (rangeData.StartOffset == blockStartOffset)
{
this.currentDownloadBuffer = new TransferDownloadBuffer(blockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - blockStartOffset), memoryBuffer);
downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, 0, (int)(rangeData.EndOffset + 1 - rangeData.StartOffset));
}
else
{
Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned");
TransferDownloadBuffer nextBuffer = new TransferDownloadBuffer(nextBlockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - nextBlockStartOffset), memoryBuffer);
downloadStream = new TransferDownloadStream(
this.Scheduler.MemoryManager,
this.currentDownloadBuffer,
(int)(rangeData.StartOffset - blockStartOffset),
(int)(nextBlockStartOffset - rangeData.StartOffset),
nextBuffer,
0,
(int)(rangeData.EndOffset + 1 - nextBlockStartOffset));
this.currentDownloadBuffer = nextBuffer;
}
}
using (downloadStream)
{
this.nextDownloadIndex++;
this.SetRangeDownloadHasWork();
RangeBasedDownloadState rangeBasedDownloadState = new RangeBasedDownloadState
{
Range = rangeData,
DownloadStream = downloadStream
};
await this.DownloadRangeAsync(rangeBasedDownloadState);
}
this.SetChunkFinish();
return;
}
this.SetRangeDownloadHasWork();
}
/// <summary>
/// Turn raw ranges get from Azure Storage in rangesSpanList
/// into list of Range.
/// </summary>
private void ArrangeRanges()
{
long currentEndOffset = -1;
// 1st RangesSpan (148MB)
IEnumerator <Utils.RangesSpan> enumerator = this.rangesSpanList.GetEnumerator();
bool hasValue = enumerator.MoveNext();
bool reachLastTransferOffset = false;
int lastTransferWindowIndex = 0;
Utils.RangesSpan current;
Utils.RangesSpan next;
if (hasValue)
{
// 1st 148MB
current = enumerator.Current;
while (hasValue)
{
hasValue = enumerator.MoveNext();
// 1st 148MB doesn't have any data
if (!current.Ranges.Any())
{
// 2nd 148MB
current = enumerator.Current;
continue;
}
if (hasValue)
{
// 2nd 148MB
next = enumerator.Current;
Debug.Assert(
current.EndOffset < this.transferJob.CheckPoint.EntryTransferOffset ||
((current.EndOffset + 1) == next.StartOffset),
"Something wrong with ranges list.");
// Both 1st 148MB & 2nd 148MB has data
if (next.Ranges.Any())
{
// They are connected, merge the range
if ((current.Ranges.Last().EndOffset + 1) == next.Ranges.First().StartOffset)
{
Utils.Range mergedRange = new Utils.Range()
{
StartOffset = current.Ranges.Last().StartOffset,
EndOffset = next.Ranges.First().EndOffset,
HasData = true
};
// Remove the last range in 1st 148MB and first range in 2nd 148MB
current.Ranges.RemoveAt(current.Ranges.Count - 1);
next.Ranges.RemoveAt(0);
// Add the merged range to 1st *148MB* (not 148MB anymore)
current.Ranges.Add(mergedRange);
current.EndOffset = mergedRange.EndOffset;
next.StartOffset = mergedRange.EndOffset + 1;
if (next.EndOffset == mergedRange.EndOffset)
{
continue;
}
}
}
}
foreach (Utils.Range range in current.Ranges)
{
// Check if we have a gap before the current range.
// If so we'll generate a range with HasData = false.
if (currentEndOffset != range.StartOffset - 1)
{
// Add empty ranges based on gaps
this.AddRangesByCheckPoint(
currentEndOffset + 1,
range.StartOffset - 1,
false,
ref reachLastTransferOffset,
ref lastTransferWindowIndex);
}
this.AddRangesByCheckPoint(
range.StartOffset,
range.EndOffset,
true,
ref reachLastTransferOffset,
ref lastTransferWindowIndex);
currentEndOffset = range.EndOffset;
}
current = enumerator.Current;
}
}
if (currentEndOffset < this.SharedTransferData.TotalLength - 1)
{
this.AddRangesByCheckPoint(
currentEndOffset + 1,
this.SharedTransferData.TotalLength - 1,
false,
ref reachLastTransferOffset,
ref lastTransferWindowIndex);
}
}
private async Task GetRangesAsync()
{
Debug.Assert(
(this.state == State.GetRanges) || (this.state == State.Error),
"GetRangesAsync called, but state isn't GetRanges or Error");
this.hasWork = false;
this.lastTransferOffset = this.SharedTransferData.TransferJob.CheckPoint.EntryTransferOffset;
int spanIndex = Interlocked.Increment(ref this.getRangesSpanIndex);
this.hasWork = spanIndex < (this.rangesSpanList.Count - 1);
Utils.RangesSpan rangesSpan = this.rangesSpanList[spanIndex];
rangesSpan.Ranges = await this.DoGetRangesAsync(rangesSpan);
List <Utils.Range> ranges = new List <Utils.Range>();
Utils.Range currentRange = null;
long currentStartOffset = rangesSpan.StartOffset;
foreach (var range in rangesSpan.Ranges)
{
long emptySize = range.StartOffset - currentStartOffset;
if (emptySize > 0 && emptySize < MinimumNoDataRangeSize)
{
// There is empty range which size is smaller than MinimumNoDataRangeSize
// merge it to the adjacent data range.
if (null == currentRange)
{
currentRange = new Utils.Range()
{
StartOffset = currentStartOffset,
EndOffset = range.EndOffset,
HasData = range.HasData
};
}
else
{
currentRange.EndOffset = range.EndOffset;
}
}
else
{
// Empty range size is larger than MinimumNoDataRangeSize
// put current data range in list and start to deal with the next data range.
if (null != currentRange)
{
ranges.Add(currentRange);
}
currentRange = new Utils.Range
{
StartOffset = range.StartOffset,
EndOffset = range.EndOffset,
HasData = range.HasData
};
}
currentStartOffset = range.EndOffset + 1;
}
if (null != currentRange)
{
ranges.Add(currentRange);
}
rangesSpan.Ranges = ranges;
if (this.getRangesCountDownEvent.Signal())
{
this.ArrangeRanges();
// Don't call CallFinish here, InitDownloadInfo will call it.
this.InitDownloadInfo();
}
}
