protected override async Task<List<Range>> DoGetRangesAsync(RangesSpan rangesSpan)
{
AccessCondition accessCondition = Utils.GenerateIfMatchConditionWithCustomerCondition(
this.Location.Blob.Properties.ETag,
this.Location.AccessCondition);
List<Range> rangeList = new List<Range>();
foreach (var pageRange in await this.pageBlob.GetPageRangesAsync(
rangesSpan.StartOffset,
rangesSpan.EndOffset - rangesSpan.StartOffset + 1,
accessCondition,
Utils.GenerateBlobRequestOptions(this.Location.BlobRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken))
{
rangeList.Add(new Range()
{
StartOffset = pageRange.StartOffset,
EndOffset = pageRange.EndOffset,
HasData = true
});
}
return rangeList;
}
protected override async Task <List <Range> > DoGetRangesAsync(RangesSpan rangesSpan)
{
AccessCondition accessCondition = Utils.GenerateIfMatchConditionWithCustomerCondition(
this.sourceLocation.Blob.Properties.ETag,
this.sourceLocation.AccessCondition);
List <Range> rangeList = new List <Range>();
foreach (var pageRange in await this.pageBlob.GetPageRangesAsync(
rangesSpan.StartOffset,
rangesSpan.EndOffset - rangesSpan.StartOffset + 1,
accessCondition,
Utils.GenerateBlobRequestOptions(this.sourceLocation.BlobRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken))
{
rangeList.Add(new Range()
{
StartOffset = pageRange.StartOffset,
EndOffset = pageRange.EndOffset,
HasData = true
});
}
return(rangeList);
}
protected override async Task <List <Range> > DoGetRangesAsync(RangesSpan rangesSpan)
{
List <Range> rangeList = new List <Range>();
foreach (var fileRange in await this.file.ListRangesAsync(
rangesSpan.StartOffset,
rangesSpan.EndOffset - rangesSpan.StartOffset + 1,
null,
Utils.GenerateFileRequestOptions(this.Location.FileRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken))
{
rangeList.Add(new Range()
{
StartOffset = fileRange.StartOffset,
EndOffset = fileRange.EndOffset,
HasData = true
});
}
return(rangeList);
}
protected override async Task<List<Range>> DoGetRangesAsync(RangesSpan rangesSpan)
{
List<Range> rangeList = new List<Range>();
foreach (var fileRange in await this.cloudFile.ListRangesAsync(
rangesSpan.StartOffset,
rangesSpan.EndOffset - rangesSpan.StartOffset + 1,
null,
Utils.GenerateFileRequestOptions(this.sourceLocation.FileRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken))
{
rangeList.Add(new Range()
{
StartOffset = fileRange.StartOffset,
EndOffset = fileRange.EndOffset,
HasData = true
});
}
return rangeList;
}
protected abstract Task <List <Range> > DoGetRangesAsync(RangesSpan rangesSpan);
/// <summary>
/// It might fail to get large ranges list from storage. This method is to split the whole file to spans of 148MB to get ranges.
/// In restartable, we only need to get ranges for chunks in TransferWindow and after TransferEntryOffset in check point.
/// In TransferWindow, there might be some chunks adjacent to TransferEntryOffset, so this method will first merge these chunks into TransferEntryOffset;
/// Then in remained chunks in the TransferWindow, it's very possible that ranges of several chunks can be got in one 148MB span.
/// To avoid sending too many get ranges requests, this method will merge the chunks to 148MB spans.
/// </summary>
private void PrepareToGetRanges()
{
this.getRangesSpanIndex = -1;
this.rangesSpanList = new List <RangesSpan>();
this.rangeList = new List <Range>();
this.nextDownloadIndex = 0;
SingleObjectCheckpoint checkpoint = this.transferJob.CheckPoint;
int blockSize = this.SharedTransferData.BlockSize;
RangesSpan rangesSpan = null;
if ((null != checkpoint.TransferWindow) &&
(checkpoint.TransferWindow.Any()))
{
checkpoint.TransferWindow.Sort();
long lastOffset = 0;
if (checkpoint.EntryTransferOffset == this.SharedTransferData.TotalLength)
{
long lengthBeforeLastChunk = checkpoint.EntryTransferOffset % blockSize;
lastOffset = 0 == lengthBeforeLastChunk ?
checkpoint.EntryTransferOffset - blockSize :
checkpoint.EntryTransferOffset - lengthBeforeLastChunk;
}
else
{
lastOffset = checkpoint.EntryTransferOffset - blockSize;
}
for (int i = checkpoint.TransferWindow.Count - 1; i >= 0; i--)
{
if (lastOffset == checkpoint.TransferWindow[i])
{
checkpoint.TransferWindow.RemoveAt(i);
checkpoint.EntryTransferOffset = lastOffset;
}
else if (lastOffset < checkpoint.TransferWindow[i])
{
throw new FormatException(Resources.RestartableInfoCorruptedException);
}
else
{
break;
}
lastOffset = checkpoint.EntryTransferOffset - blockSize;
}
if (this.transferJob.CheckPoint.TransferWindow.Any())
{
rangesSpan = new RangesSpan();
rangesSpan.StartOffset = checkpoint.TransferWindow[0];
rangesSpan.EndOffset = Math.Min(rangesSpan.StartOffset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1;
for (int i = 1; i < checkpoint.TransferWindow.Count; ++i)
{
if (checkpoint.TransferWindow[i] + blockSize > rangesSpan.EndOffset)
{
long lastEndOffset = rangesSpan.EndOffset;
this.rangesSpanList.Add(rangesSpan);
rangesSpan = new RangesSpan();
rangesSpan.StartOffset = checkpoint.TransferWindow[i] > lastEndOffset ? checkpoint.TransferWindow[i] : lastEndOffset + 1;
rangesSpan.EndOffset = Math.Min(rangesSpan.StartOffset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1;
}
}
this.rangesSpanList.Add(rangesSpan);
}
}
long offset = null != rangesSpan ?
rangesSpan.EndOffset > checkpoint.EntryTransferOffset ?
rangesSpan.EndOffset + 1 :
checkpoint.EntryTransferOffset :
checkpoint.EntryTransferOffset;
while (offset < this.SharedTransferData.TotalLength)
{
rangesSpan = new RangesSpan()
{
StartOffset = offset,
EndOffset = Math.Min(offset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1
};
this.rangesSpanList.Add(rangesSpan);
offset = rangesSpan.EndOffset + 1;
}
if (this.rangesSpanList.Any())
{
this.getRangesCountDownEvent = new CountdownEvent(this.rangesSpanList.Count);
}
}
private async Task GetRangesAsync()
{
Debug.Assert(
(this.state == State.GetRanges) || (this.state == State.Error),
"GetRangesAsync called, but state isn't GetRanges or Error");
if (Interlocked.CompareExchange(ref workToken, 0, 1) == 0)
{
return;
}
this.lastTransferOffset = this.SharedTransferData.TransferJob.CheckPoint.EntryTransferOffset;
int spanIndex = Interlocked.Increment(ref this.getRangesSpanIndex);
this.workToken = (spanIndex < (this.rangesSpanList.Count - 1)) ? 1 : 0;
RangesSpan rangesSpan = this.rangesSpanList[spanIndex];
rangesSpan.Ranges = await this.DoGetRangesAsync(rangesSpan);
List <Range> ranges = new List <Range>();
Range currentRange = null;
long currentStartOffset = rangesSpan.StartOffset;
foreach (var range in rangesSpan.Ranges)
{
long emptySize = range.StartOffset - currentStartOffset;
if (emptySize > 0 && emptySize < MinimumNoDataRangeSize)
{
// There is empty range which size is smaller than MinimumNoDataRangeSize
// merge it to the adjacent data range.
if (null == currentRange)
{
currentRange = new Range()
{
StartOffset = currentStartOffset,
EndOffset = range.EndOffset,
HasData = range.HasData
};
}
else
{
currentRange.EndOffset = range.EndOffset;
}
}
else
{
// Empty range size is larger than MinimumNoDataRangeSize
// put current data range in list and start to deal with the next data range.
if (null != currentRange)
{
ranges.Add(currentRange);
}
currentRange = new Range
{
StartOffset = range.StartOffset,
EndOffset = range.EndOffset,
HasData = range.HasData
};
}
currentStartOffset = range.EndOffset + 1;
}
if (null != currentRange)
{
ranges.Add(currentRange);
}
rangesSpan.Ranges = ranges;
if (this.getRangesCountDownEvent.Signal())
{
this.ArrangeRanges();
// Don't call CallFinish here, InitDownloadInfo will call it.
this.InitDownloadInfo();
}
}
protected abstract Task<List<Range>> DoGetRangesAsync(RangesSpan rangesSpan);
/// <summary>
/// It might fail to get large ranges list from storage. This method is to split the whole file to spans of 148MB to get ranges.
/// In restartable, we only need to get ranges for chunks in TransferWindow and after TransferEntryOffset in check point.
/// In TransferWindow, there might be some chunks adjacent to TransferEntryOffset, so this method will first merge these chunks into TransferEntryOffset;
/// Then in remained chunks in the TransferWindow, it's very possible that ranges of several chunks can be got in one 148MB span.
/// To avoid sending too many get ranges requests, this method will merge the chunks to 148MB spans.
/// </summary>
private void PrepareToGetRanges()
{
this.getRangesSpanIndex = -1;
this.rangesSpanList = new List<RangesSpan>();
this.rangeList = new List<Range>();
this.nextDownloadIndex = 0;
SingleObjectCheckpoint checkpoint = this.transferJob.CheckPoint;
int blockSize = this.Scheduler.TransferOptions.BlockSize;
RangesSpan rangesSpan = null;
if ((null != checkpoint.TransferWindow)
&& (checkpoint.TransferWindow.Any()))
{
checkpoint.TransferWindow.Sort();
long lastOffset = 0;
if (checkpoint.EntryTransferOffset == this.SharedTransferData.TotalLength)
{
long lengthBeforeLastChunk = checkpoint.EntryTransferOffset % blockSize;
lastOffset = 0 == lengthBeforeLastChunk ?
checkpoint.EntryTransferOffset - blockSize :
checkpoint.EntryTransferOffset - lengthBeforeLastChunk;
}
else
{
lastOffset = checkpoint.EntryTransferOffset - blockSize;
}
for (int i = checkpoint.TransferWindow.Count - 1; i >= 0; i--)
{
if (lastOffset == checkpoint.TransferWindow[i])
{
checkpoint.TransferWindow.RemoveAt(i);
checkpoint.EntryTransferOffset = lastOffset;
}
else if (lastOffset < checkpoint.TransferWindow[i])
{
throw new FormatException(Resources.RestartableInfoCorruptedException);
}
else
{
break;
}
lastOffset = checkpoint.EntryTransferOffset - blockSize;
}
if (this.transferJob.CheckPoint.TransferWindow.Any())
{
rangesSpan = new RangesSpan();
rangesSpan.StartOffset = checkpoint.TransferWindow[0];
rangesSpan.EndOffset = Math.Min(rangesSpan.StartOffset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1;
for (int i = 1; i < checkpoint.TransferWindow.Count; ++i )
{
if (checkpoint.TransferWindow[i] + blockSize > rangesSpan.EndOffset)
{
long lastEndOffset = rangesSpan.EndOffset;
this.rangesSpanList.Add(rangesSpan);
rangesSpan = new RangesSpan();
rangesSpan.StartOffset = checkpoint.TransferWindow[i] > lastEndOffset ? checkpoint.TransferWindow[i] : lastEndOffset + 1;
rangesSpan.EndOffset = Math.Min(rangesSpan.StartOffset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1;
}
}
this.rangesSpanList.Add(rangesSpan);
}
}
long offset = null != rangesSpan ?
rangesSpan.EndOffset > checkpoint.EntryTransferOffset ?
rangesSpan.EndOffset + 1 :
checkpoint.EntryTransferOffset :
checkpoint.EntryTransferOffset;
while (offset < this.SharedTransferData.TotalLength)
{
rangesSpan = new RangesSpan()
{
StartOffset = offset,
EndOffset = Math.Min(offset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1
};
this.rangesSpanList.Add(rangesSpan);
offset = rangesSpan.EndOffset + 1;
}
if (this.rangesSpanList.Any())
{
this.getRangesCountDownEvent = new CountdownEvent(this.rangesSpanList.Count);
}
}
