private void AddRangesByCheckPoint(long startOffset, long endOffset, bool hasData, ref bool reachLastTransferOffset, ref int lastTransferWindowIndex) { SingleObjectCheckpoint checkpoint = this.transferJob.CheckPoint; if (reachLastTransferOffset) { this.rangeList.AddRange( new Utils.Range { StartOffset = startOffset, EndOffset = endOffset, HasData = hasData, }.SplitRanges(Constants.DefaultTransferChunkSize)); } else { Utils.Range range = new Utils.Range() { StartOffset = -1, HasData = hasData }; while (lastTransferWindowIndex < checkpoint.TransferWindow.Count) { long lastTransferWindowStart = checkpoint.TransferWindow[lastTransferWindowIndex]; long lastTransferWindowEnd = Math.Min(checkpoint.TransferWindow[lastTransferWindowIndex] + this.SharedTransferData.BlockSize - 1, this.SharedTransferData.TotalLength); if (lastTransferWindowStart <= endOffset) { if (-1 == range.StartOffset) { // New range range.StartOffset = Math.Max(lastTransferWindowStart, startOffset); range.EndOffset = Math.Min(lastTransferWindowEnd, endOffset); } else { if (range.EndOffset != lastTransferWindowStart - 1) { // Store the previous range and create a new one this.rangeList.AddRange(range.SplitRanges(Constants.DefaultTransferChunkSize)); range = new Utils.Range() { StartOffset = Math.Max(lastTransferWindowStart, startOffset), HasData = hasData }; } range.EndOffset = Math.Min(lastTransferWindowEnd, endOffset); } if (range.EndOffset == lastTransferWindowEnd) { // Reach the end of transfer window, move to next ++lastTransferWindowIndex; continue; } } break; } if (-1 != range.StartOffset) { this.rangeList.AddRange(range.SplitRanges(Constants.DefaultTransferChunkSize)); } if (checkpoint.EntryTransferOffset <= endOffset + 1) { reachLastTransferOffset = true; if (checkpoint.EntryTransferOffset <= endOffset) { this.rangeList.AddRange(new Utils.Range() { StartOffset = checkpoint.EntryTransferOffset, EndOffset = endOffset, HasData = hasData, }.SplitRanges(Constants.DefaultTransferChunkSize)); } } } }
private async Task DownloadRangeAsync() { Debug.Assert( this.state == State.Error || this.state == State.Download, "DownloadRangeAsync called, but state isn't Download or Error"); this.hasWork = false; if (State.Error == this.state) { // Some thread has set error message, just return here. return; } if (this.nextDownloadIndex < this.rangeList.Count) { Utils.Range rangeData = this.rangeList[this.nextDownloadIndex]; int blockSize = this.SharedTransferData.BlockSize; long blockStartOffset = (rangeData.StartOffset / blockSize) * blockSize; long nextBlockStartOffset = Math.Min(blockStartOffset + blockSize, this.SharedTransferData.TotalLength); TransferDownloadStream downloadStream = null; if ((rangeData.StartOffset > blockStartOffset) && (rangeData.EndOffset < nextBlockStartOffset)) { Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned"); downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, (int)(rangeData.StartOffset - blockStartOffset), (int)(rangeData.EndOffset + 1 - rangeData.StartOffset)); } else { // Attempt to reserve memory. If none available we'll // retry some time later. byte[][] memoryBuffer = this.Scheduler.MemoryManager.RequireBuffers(this.SharedTransferData.MemoryChunksRequiredEachTime); if (null == memoryBuffer) { this.SetRangeDownloadHasWork(); return; } if (rangeData.EndOffset >= this.lastTransferOffset) { bool canRead = true; lock (this.transferJob.CheckPoint.TransferWindowLock) { if (this.transferJob.CheckPoint.TransferWindow.Count >= Constants.MaxCountInTransferWindow) { canRead = false; } else { if (this.transferJob.CheckPoint.EntryTransferOffset < this.SharedTransferData.TotalLength) { this.transferJob.CheckPoint.TransferWindow.Add(this.transferJob.CheckPoint.EntryTransferOffset); this.transferJob.CheckPoint.EntryTransferOffset = Math.Min(this.transferJob.CheckPoint.EntryTransferOffset + blockSize, this.SharedTransferData.TotalLength); } } } if (!canRead) { this.Scheduler.MemoryManager.ReleaseBuffers(memoryBuffer); this.SetRangeDownloadHasWork(); return; } } if (rangeData.StartOffset == blockStartOffset) { this.currentDownloadBuffer = new TransferDownloadBuffer(blockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - blockStartOffset), memoryBuffer); downloadStream = new TransferDownloadStream(this.Scheduler.MemoryManager, this.currentDownloadBuffer, 0, (int)(rangeData.EndOffset + 1 - rangeData.StartOffset)); } else { Debug.Assert(null != this.currentDownloadBuffer, "Download buffer should have been allocated when range start offset is not block size aligned"); TransferDownloadBuffer nextBuffer = new TransferDownloadBuffer(nextBlockStartOffset, (int)Math.Min(blockSize, this.SharedTransferData.TotalLength - nextBlockStartOffset), memoryBuffer); downloadStream = new TransferDownloadStream( this.Scheduler.MemoryManager, this.currentDownloadBuffer, (int)(rangeData.StartOffset - blockStartOffset), (int)(nextBlockStartOffset - rangeData.StartOffset), nextBuffer, 0, (int)(rangeData.EndOffset + 1 - nextBlockStartOffset)); this.currentDownloadBuffer = nextBuffer; } } using (downloadStream) { this.nextDownloadIndex++; this.SetRangeDownloadHasWork(); RangeBasedDownloadState rangeBasedDownloadState = new RangeBasedDownloadState { Range = rangeData, DownloadStream = downloadStream }; await this.DownloadRangeAsync(rangeBasedDownloadState); } this.SetChunkFinish(); return; } this.SetRangeDownloadHasWork(); }
/// <summary> /// Turn raw ranges get from Azure Storage in rangesSpanList /// into list of Range. /// </summary> private void ArrangeRanges() { long currentEndOffset = -1; // 1st RangesSpan (148MB) IEnumerator <Utils.RangesSpan> enumerator = this.rangesSpanList.GetEnumerator(); bool hasValue = enumerator.MoveNext(); bool reachLastTransferOffset = false; int lastTransferWindowIndex = 0; Utils.RangesSpan current; Utils.RangesSpan next; if (hasValue) { // 1st 148MB current = enumerator.Current; while (hasValue) { hasValue = enumerator.MoveNext(); // 1st 148MB doesn't have any data if (!current.Ranges.Any()) { // 2nd 148MB current = enumerator.Current; continue; } if (hasValue) { // 2nd 148MB next = enumerator.Current; Debug.Assert( current.EndOffset < this.transferJob.CheckPoint.EntryTransferOffset || ((current.EndOffset + 1) == next.StartOffset), "Something wrong with ranges list."); // Both 1st 148MB & 2nd 148MB has data if (next.Ranges.Any()) { // They are connected, merge the range if ((current.Ranges.Last().EndOffset + 1) == next.Ranges.First().StartOffset) { Utils.Range mergedRange = new Utils.Range() { StartOffset = current.Ranges.Last().StartOffset, EndOffset = next.Ranges.First().EndOffset, HasData = true }; // Remove the last range in 1st 148MB and first range in 2nd 148MB current.Ranges.RemoveAt(current.Ranges.Count - 1); next.Ranges.RemoveAt(0); // Add the merged range to 1st *148MB* (not 148MB anymore) current.Ranges.Add(mergedRange); current.EndOffset = mergedRange.EndOffset; next.StartOffset = mergedRange.EndOffset + 1; if (next.EndOffset == mergedRange.EndOffset) { continue; } } } } foreach (Utils.Range range in current.Ranges) { // Check if we have a gap before the current range. // If so we'll generate a range with HasData = false. if (currentEndOffset != range.StartOffset - 1) { // Add empty ranges based on gaps this.AddRangesByCheckPoint( currentEndOffset + 1, range.StartOffset - 1, false, ref reachLastTransferOffset, ref lastTransferWindowIndex); } this.AddRangesByCheckPoint( range.StartOffset, range.EndOffset, true, ref reachLastTransferOffset, ref lastTransferWindowIndex); currentEndOffset = range.EndOffset; } current = enumerator.Current; } } if (currentEndOffset < this.SharedTransferData.TotalLength - 1) { this.AddRangesByCheckPoint( currentEndOffset + 1, this.SharedTransferData.TotalLength - 1, false, ref reachLastTransferOffset, ref lastTransferWindowIndex); } }
private async Task GetRangesAsync() { Debug.Assert( (this.state == State.GetRanges) || (this.state == State.Error), "GetRangesAsync called, but state isn't GetRanges or Error"); this.hasWork = false; this.lastTransferOffset = this.SharedTransferData.TransferJob.CheckPoint.EntryTransferOffset; int spanIndex = Interlocked.Increment(ref this.getRangesSpanIndex); this.hasWork = spanIndex < (this.rangesSpanList.Count - 1); Utils.RangesSpan rangesSpan = this.rangesSpanList[spanIndex]; rangesSpan.Ranges = await this.DoGetRangesAsync(rangesSpan); List <Utils.Range> ranges = new List <Utils.Range>(); Utils.Range currentRange = null; long currentStartOffset = rangesSpan.StartOffset; foreach (var range in rangesSpan.Ranges) { long emptySize = range.StartOffset - currentStartOffset; if (emptySize > 0 && emptySize < MinimumNoDataRangeSize) { // There is empty range which size is smaller than MinimumNoDataRangeSize // merge it to the adjacent data range. if (null == currentRange) { currentRange = new Utils.Range() { StartOffset = currentStartOffset, EndOffset = range.EndOffset, HasData = range.HasData }; } else { currentRange.EndOffset = range.EndOffset; } } else { // Empty range size is larger than MinimumNoDataRangeSize // put current data range in list and start to deal with the next data range. if (null != currentRange) { ranges.Add(currentRange); } currentRange = new Utils.Range { StartOffset = range.StartOffset, EndOffset = range.EndOffset, HasData = range.HasData }; } currentStartOffset = range.EndOffset + 1; } if (null != currentRange) { ranges.Add(currentRange); } rangesSpan.Ranges = ranges; if (this.getRangesCountDownEvent.Signal()) { this.ArrangeRanges(); // Don't call CallFinish here, InitDownloadInfo will call it. this.InitDownloadInfo(); } }