protected override async Task <List <Utils.Range> > DoGetRangesAsync(Utils.RangesSpan rangesSpan)
{
AccessCondition accessCondition = Utils.GenerateIfMatchConditionWithCustomerCondition(
this.sourceLocation.Blob.Properties.ETag,
this.sourceLocation.AccessCondition);
List <Utils.Range> rangeList = new List <Utils.Range>();
foreach (var pageRange in await this.pageBlob.GetPageRangesAsync(
rangesSpan.StartOffset,
rangesSpan.EndOffset - rangesSpan.StartOffset + 1,
accessCondition,
Utils.GenerateBlobRequestOptions(this.sourceLocation.BlobRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken))
{
rangeList.Add(new Utils.Range()
{
StartOffset = pageRange.StartOffset,
EndOffset = pageRange.EndOffset,
HasData = true
});
}
return(rangeList);
}
protected override async Task <List <Utils.Range> > DoGetRangesAsync(Utils.RangesSpan rangesSpan)
{
List <Utils.Range> rangeList = new List <Utils.Range>();
foreach (var fileRange in await this.cloudFile.ListRangesAsync(
rangesSpan.StartOffset,
rangesSpan.EndOffset - rangesSpan.StartOffset + 1,
null,
Utils.GenerateFileRequestOptions(this.sourceLocation.FileRequestOptions),
Utils.GenerateOperationContext(this.Controller.TransferContext),
this.CancellationToken))
{
rangeList.Add(new Utils.Range()
{
StartOffset = fileRange.StartOffset,
EndOffset = fileRange.EndOffset,
HasData = true
});
}
return(rangeList);
}
protected abstract Task <List <Utils.Range> > DoGetRangesAsync(Utils.RangesSpan rangesSpan);
/// <summary>
/// It might fail to get large ranges list from storage. This method is to split the whole file to spans of 148MB to get ranges.
/// In restartable, we only need to get ranges for chunks in TransferWindow and after TransferEntryOffset in check point.
/// In TransferWindow, there might be some chunks adjacent to TransferEntryOffset, so this method will first merge these chunks into TransferEntryOffset;
/// Then in remained chunks in the TransferWindow, it's very possible that ranges of several chunks can be got in one 148MB span.
/// To avoid sending too many get ranges requests, this method will merge the chunks to 148MB spans.
/// </summary>
private void PrepareToGetRanges()
{
this.getRangesSpanIndex = -1;
this.rangesSpanList = new List <Utils.RangesSpan>();
this.rangeList = new List <Utils.Range>();
this.nextDownloadIndex = 0;
SingleObjectCheckpoint checkpoint = this.transferJob.CheckPoint;
int blockSize = this.SharedTransferData.BlockSize;
Utils.RangesSpan rangesSpan = null;
if ((null != checkpoint.TransferWindow) &&
(checkpoint.TransferWindow.Any()))
{
checkpoint.TransferWindow.Sort();
long lastOffset = 0;
if (checkpoint.EntryTransferOffset == this.SharedTransferData.TotalLength)
{
long lengthBeforeLastChunk = checkpoint.EntryTransferOffset % blockSize;
lastOffset = 0 == lengthBeforeLastChunk ?
checkpoint.EntryTransferOffset - blockSize :
checkpoint.EntryTransferOffset - lengthBeforeLastChunk;
}
else
{
lastOffset = checkpoint.EntryTransferOffset - blockSize;
}
for (int i = checkpoint.TransferWindow.Count - 1; i >= 0; i--)
{
if (lastOffset == checkpoint.TransferWindow[i])
{
checkpoint.TransferWindow.RemoveAt(i);
checkpoint.EntryTransferOffset = lastOffset;
}
else if (lastOffset < checkpoint.TransferWindow[i])
{
throw new FormatException(Resources.RestartableInfoCorruptedException);
}
else
{
break;
}
lastOffset = checkpoint.EntryTransferOffset - blockSize;
}
if (this.transferJob.CheckPoint.TransferWindow.Any())
{
rangesSpan = new Utils.RangesSpan();
rangesSpan.StartOffset = checkpoint.TransferWindow[0];
rangesSpan.EndOffset = Math.Min(rangesSpan.StartOffset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1;
for (int i = 1; i < checkpoint.TransferWindow.Count; ++i)
{
if (checkpoint.TransferWindow[i] + blockSize > rangesSpan.EndOffset)
{
long lastEndOffset = rangesSpan.EndOffset;
this.rangesSpanList.Add(rangesSpan);
rangesSpan = new Utils.RangesSpan();
rangesSpan.StartOffset = checkpoint.TransferWindow[i] > lastEndOffset ? checkpoint.TransferWindow[i] : lastEndOffset + 1;
rangesSpan.EndOffset = Math.Min(rangesSpan.StartOffset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1;
}
}
this.rangesSpanList.Add(rangesSpan);
}
}
long offset = null != rangesSpan ?
rangesSpan.EndOffset > checkpoint.EntryTransferOffset ?
rangesSpan.EndOffset + 1 :
checkpoint.EntryTransferOffset :
checkpoint.EntryTransferOffset;
while (offset < this.SharedTransferData.TotalLength)
{
rangesSpan = new Utils.RangesSpan()
{
StartOffset = offset,
EndOffset = Math.Min(offset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1
};
this.rangesSpanList.Add(rangesSpan);
offset = rangesSpan.EndOffset + 1;
}
if (!this.rangesSpanList.Any())
{
return;
}
else if (this.rangesSpanList.Count == 1)
{
if (this.rangesSpanList[0].EndOffset - this.rangesSpanList[0].StartOffset < blockSize)
{
this.rangeList.Add(new Utils.Range()
{
StartOffset = this.rangesSpanList[0].StartOffset,
EndOffset = this.rangesSpanList[0].EndOffset,
HasData = true
});
this.rangesSpanList.Clear();
return;
}
}
this.getRangesCountDownEvent = new CountdownEvent(this.rangesSpanList.Count);
}
private async Task GetRangesAsync()
{
Debug.Assert(
(this.state == State.GetRanges) || (this.state == State.Error),
"GetRangesAsync called, but state isn't GetRanges or Error");
this.hasWork = false;
this.lastTransferOffset = this.SharedTransferData.TransferJob.CheckPoint.EntryTransferOffset;
int spanIndex = Interlocked.Increment(ref this.getRangesSpanIndex);
this.hasWork = spanIndex < (this.rangesSpanList.Count - 1);
Utils.RangesSpan rangesSpan = this.rangesSpanList[spanIndex];
rangesSpan.Ranges = await this.DoGetRangesAsync(rangesSpan);
List <Utils.Range> ranges = new List <Utils.Range>();
Utils.Range currentRange = null;
long currentStartOffset = rangesSpan.StartOffset;
foreach (var range in rangesSpan.Ranges)
{
long emptySize = range.StartOffset - currentStartOffset;
if (emptySize > 0 && emptySize < MinimumNoDataRangeSize)
{
// There is empty range which size is smaller than MinimumNoDataRangeSize
// merge it to the adjacent data range.
if (null == currentRange)
{
currentRange = new Utils.Range()
{
StartOffset = currentStartOffset,
EndOffset = range.EndOffset,
HasData = range.HasData
};
}
else
{
currentRange.EndOffset = range.EndOffset;
}
}
else
{
// Empty range size is larger than MinimumNoDataRangeSize
// put current data range in list and start to deal with the next data range.
if (null != currentRange)
{
ranges.Add(currentRange);
}
currentRange = new Utils.Range
{
StartOffset = range.StartOffset,
EndOffset = range.EndOffset,
HasData = range.HasData
};
}
currentStartOffset = range.EndOffset + 1;
}
if (null != currentRange)
{
ranges.Add(currentRange);
}
rangesSpan.Ranges = ranges;
if (this.getRangesCountDownEvent.Signal())
{
this.ArrangeRanges();
// Don't call CallFinish here, InitDownloadInfo will call it.
this.InitDownloadInfo();
}
}
