public void ByteRangesTests()
{
var fullRange = BytesRangeUtils.Full();
var outer = BytesRangeUtils.Make(0, 100);
var inner = BytesRangeUtils.Make(10, 90);
var innerInside = BytesRangeUtils.Make(50, 60);
var innerRightSideOverlap = BytesRangeUtils.Make(50, 95);
var innerLeftSideOverlap = BytesRangeUtils.Make(5, 15);
Assert.That(outer.Contains(inner));
var localizedRightOverlap = innerRightSideOverlap.LocalizeTo(inner);
var localizedLeftOverlap = innerLeftSideOverlap.LocalizeTo(inner);
Assert.That(localizedRightOverlap, Is.EqualTo(BytesRangeUtils.Make(40, -1)));
Assert.That(localizedLeftOverlap, Is.EqualTo(BytesRangeUtils.Make(0, 5)));
var localizedFull = fullRange.LocalizeTo(inner);
Assert.That(localizedFull, Is.EqualTo(BytesRangeUtils.Full()));
var localizedInner = innerInside.LocalizeTo(inner);
Assert.That(localizedInner, Is.EqualTo(BytesRangeUtils.Make(40, 50)));
}
public void JobQueuing_WithSpecifiedRangeAndOffset()
{
BytesRange range = BytesRangeUtils.Make(450, 830);
long offset = 512;
var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, offset, range, DataSize, _chunksData).ToList();
Assert.That(jobs.Count, Is.EqualTo(3));
var firstJob = jobs[0];
Assert.That(firstJob.Url, Is.EqualTo("test.com/someData.4"));
Assert.That(firstJob.Range.Start, Is.EqualTo(0));
Assert.That(firstJob.Range.End, Is.EqualTo(-1));
var middleJob = jobs[1];
Assert.That(middleJob.Url, Is.EqualTo("test.com/someData.5"));
Assert.That(middleJob.Range.Start, Is.EqualTo(0));
Assert.That(middleJob.Range.End, Is.EqualTo(-1));
var lastJob = jobs[2];
Assert.That(lastJob.Url, Is.EqualTo("test.com/someData.6"));
Assert.That(lastJob.Range.Start, Is.EqualTo(0));
Assert.That(lastJob.Range.End, Is.EqualTo(63));
}
public void ChunksCalculations_RangeIsExactWithChunks()
{
BytesRange range = BytesRangeUtils.Make(ChunkSize, (ChunkSize * 3) - 1); // Exactly 3 chunks
var chunksRange = range.Chunkify(_chunksData);
Assert.That(chunksRange, Is.EqualTo(range));
}
public void ChunksCalculations_WithRangeExactlyAsDataSize()
{
BytesRange range = BytesRangeUtils.Make(0, DataSize);
var chunksRange = range.Chunkify(_chunksData);
Assert.That(chunksRange.Start, Is.EqualTo(0));
Assert.That(chunksRange.End, Is.EqualTo(DataSize));
}
public void ChunksCalculations_WithSpecifiedRange_2()
{
BytesRange range = BytesRangeUtils.Make(450, 830);
var chunksRange = range.Chunkify(_chunksData);
Assert.That(chunksRange.Start, Is.EqualTo(ChunkSize * 7));
Assert.That(chunksRange.End, Is.EqualTo(ChunkSize * 13 - 1));
Assert.That(chunksRange.Contains(range));
}
public void ChunksCalculations_WithFullRange()
{
BytesRange range = BytesRangeUtils.Full();
var chunksRange = range.Chunkify(_chunksData);
Assert.That(chunksRange.Start, Is.EqualTo(0));
Assert.That(chunksRange.End, Is.EqualTo(-1));
Assert.That(chunksRange.Contains(range));
}
public void JobQueuing_SinglePartScenario()
{
BytesRange range = BytesRangeUtils.Make(315, 380);
var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, 0, range, DataSize, _chunksData).ToList();
Assert.That(jobs.Count, Is.EqualTo(1));
var job = jobs[0];
Assert.That(job.Url, Is.EqualTo("test.com/someData.2"));
Assert.That(job.Range.Start, Is.EqualTo(0));
Assert.That(job.Range.End, Is.EqualTo(-1).Or.EqualTo(127));
}
public void JobQueuing_WithFullRange()
{
BytesRange range = BytesRangeUtils.Full();
var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, 0, range, DataSize, _chunksData).ToList();
int expectedJobCount = (int)(DataSize / PartSize) + 1;
Assert.That(jobs.Count, Is.EqualTo(expectedJobCount));
var firstJob = jobs[0];
Assert.That(firstJob.Range.Start, Is.EqualTo(0));
Assert.That(firstJob.Range.End, Is.EqualTo(-1));
var lastJob = jobs[expectedJobCount - 1];
Assert.That(lastJob.Range.Start, Is.EqualTo(0));
Assert.That(lastJob.Range.End, Is.EqualTo(-1));
}
public static IEnumerable <DownloadJob> BuildDownloadJobQueue(ResourceUrl resourceUrl, long currentOffset, BytesRange range, long dataSize, ChunksData chunksData)
{
// The effective range is the original range contained within multiples of chunk size
BytesRange effectiveRange = range.Chunkify(chunksData);
var dataBounds = new BytesRange(currentOffset, -1);
BytesRange bounds = effectiveRange.ContainIn(dataBounds);
// An uncommon edge case might occur, in which bounds.Start is equal to dataSize,
// this would cause the download to continue forever, with every request crashing due to invalid range header
if (bounds.Start >= dataSize)
{
yield break;
}
if (resourceUrl.PartSize == 0)
{
yield return(new DownloadJob(resourceUrl.Url, bounds.Start, bounds.End));
yield break;
}
long partSize = resourceUrl.PartSize;
int firstPart = (int)(bounds.Start / partSize);
int totalPartCount = (int)(dataSize / partSize);
if (dataSize % partSize != 0)
{
totalPartCount += 1;
}
int lastPart = totalPartCount;
if (bounds.End != -1)
{
lastPart = (int)(bounds.End / partSize);
if (bounds.End % partSize != 0)
{
lastPart += 1;
}
}
long lastByte = dataSize - 1;
for (int i = firstPart; i < lastPart; i++)
{
string url = resourceUrl.Url;
if (i > 0)
{
// second and later indices should have index numebr at the end
url += "." + i;
}
BytesRange partRange = BytesRangeUtils.Make(i * partSize, (i + 1) * partSize - 1, lastByte);
BytesRange localBounds = bounds.LocalizeTo(partRange);
yield return(new DownloadJob(url, localBounds.Start, localBounds.End));
}
}
