public void SetUp()
{
_url = new ResourceUrl
{
Url = "test.com/someData",
MetaUrl = "test.com/someMetaData",
Country = "GB",
PartSize = PartSize
};
List <Chunk> chunks = new List <Chunk>();
for (int i = 0; i < (DataSize / ChunkSize); i++)
{
chunks.Add(new Chunk
{
Hash = new byte[] { 0x20 } // TODO: Generate random bytes here
});
}
_chunksData = new ChunksData
{
ChunkSize = ChunkSize,
Chunks = chunks.ToArray()
};
}
public ChunkedFileStream(string path, long fileSize, ChunksData chunksData, HashFunction hashFunction,
WorkFlags workFlags = WorkFlags.None)
{
Checks.ArgumentNotNullOrEmpty(path, "path");
Checks.ArgumentMoreThanZero(fileSize, "fileSize");
Checks.ArgumentNotNull(hashFunction, "hashFunction");
DebugLogger.LogConstructor();
DebugLogger.LogVariable(path, "path");
DebugLogger.LogVariable(fileSize, "fileSize");
_fileSize = fileSize;
_chunksData = chunksData;
_hashFunction = hashFunction;
_buffer = new byte[_chunksData.ChunkSize];
if ((workFlags | WorkFlags.PreservePreviousFile) != 0)
{
// Often you may want to continue downloading of a file if this exists
// It tries to open a file and re-download it from the verified position.
// It does not check the hash of the file. It trusts that the file is already valid up to that point.
// Because the only way to download the file should be using Chunked Downloader.
_fileStream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None);
_fileStream.Seek(0, SeekOrigin.End); // seek and stay at the end, so we can append
long currentFileSize = _fileStream.Position;
// Let's make sure that file size is a multiply of chunk size.
// If not, something is wrong with the file.
if (currentFileSize % chunksData.ChunkSize == 0)
{
_chunkIndex = (int)(currentFileSize / chunksData.ChunkSize);
}
else
{
DebugLogger.LogWarningFormat(
"File {0} size {1} is not a multiply of chunk size: {2}. Will recreate it.", path,
currentFileSize, chunksData.ChunkSize);
_fileStream.Close();
_fileStream.Dispose();
_fileStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None);
}
}
else
{
_fileStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None);
}
}
public void Setup()
{
_fileName = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
_invalidHash = new byte[] { 1, 2, 3, 4, 5 };
_chunksData = new ChunksData
{
ChunkSize = 2,
Chunks = new []
{
new Chunk
{
Hash = new byte[] { 1 }
},
new Chunk
{
Hash = new byte[] { 2 }
}
}
};
}
public ChunkedHttpDownloader([NotNull] string destinationFilePath, [NotNull] ResourceUrl[] urls, ChunksData chunksData,
long size)
{
if (string.IsNullOrEmpty(destinationFilePath))
{
throw new ArgumentException("Value cannot be null or empty.", "destinationFilePath");
}
if (urls == null)
{
throw new ArgumentNullException("urls");
}
if (size <= 0)
{
throw new ArgumentOutOfRangeException("size");
}
_logger = PatcherLogManager.DefaultLogger;
_destinationFilePath = destinationFilePath;
_urls = urls;
_chunksData = chunksData;
_size = size;
}
public static IEnumerable <DownloadJob> BuildDownloadJobQueue(ResourceUrl resourceUrl, long currentOffset, BytesRange range, long dataSize, ChunksData chunksData)
{
// The effective range is the original range contained within multiples of chunk size
BytesRange effectiveRange = range.Chunkify(chunksData);
var dataBounds = new BytesRange(currentOffset, -1);
BytesRange bounds = effectiveRange.ContainIn(dataBounds);
// An uncommon edge case might occur, in which bounds.Start is equal to dataSize,
// this would cause the download to continue forever, with every request crashing due to invalid range header
if (bounds.Start >= dataSize)
{
yield break;
}
if (resourceUrl.PartSize == 0)
{
yield return(new DownloadJob(resourceUrl.Url, bounds.Start, bounds.End));
yield break;
}
long partSize = resourceUrl.PartSize;
int firstPart = (int)(bounds.Start / partSize);
int totalPartCount = (int)(dataSize / partSize);
if (dataSize % partSize != 0)
{
totalPartCount += 1;
}
int lastPart = totalPartCount;
if (bounds.End != -1)
{
lastPart = (int)(bounds.End / partSize);
if (bounds.End % partSize != 0)
{
lastPart += 1;
}
}
long lastByte = dataSize - 1;
for (int i = firstPart; i < lastPart; i++)
{
string url = resourceUrl.Url;
if (i > 0)
{
// second and later indices should have index numebr at the end
url += "." + i;
}
BytesRange partRange = BytesRangeUtils.Make(i * partSize, (i + 1) * partSize - 1, lastByte);
BytesRange localBounds = bounds.LocalizeTo(partRange);
yield return(new DownloadJob(url, localBounds.Start, localBounds.End));
}
}
public ChunkedFileStream([NotNull] string path, long fileSize, ChunksData chunksData,
[NotNull] HashFunction hashFunction,
WorkFlags workFlags = WorkFlags.None)
{
if (path == null)
{
throw new ArgumentNullException("path");
}
if (fileSize <= 0)
{
throw new ArgumentOutOfRangeException("fileSize");
}
if (hashFunction == null)
{
throw new ArgumentNullException("hashFunction");
}
_logger = PatcherLogManager.DefaultLogger;
_fileSize = fileSize;
_chunksData = chunksData;
_hashFunction = hashFunction;
_buffer = new byte[_chunksData.ChunkSize];
_logger.LogTrace("path = " + path);
_logger.LogTrace("fileSize = " + fileSize);
_logger.LogTrace("chunksData.ChunkSize = " + chunksData.ChunkSize);
bool preservePreviousFile = (workFlags | WorkFlags.PreservePreviousFile) != 0;
_logger.LogTrace("preservePreviousFile = " + preservePreviousFile);
if (preservePreviousFile)
{
// Often you may want to continue downloading of a file if this exists
// It tries to open a file and re-download it from the verified position.
// It does not check the hash of the file. It trusts that the file is already valid up to that point.
// Because the only way to download the file should be using Chunked Downloader.
_logger.LogDebug("Opening file stream...");
_fileStream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None);
_fileStream.Seek(0, SeekOrigin.End); // seek and stay at the end, so we can append
long currentFileSize = _fileStream.Position;
_logger.LogDebug("File stream opened.");
_logger.LogTrace("currentFileSize = " + currentFileSize);
_logger.LogDebug("Checking whether stream can append to current file...");
if (currentFileSize == 0)
{
_logger.LogDebug("File is new. Append is not possible.");
}
// Let's make sure that file size is a multiply of chunk size.
// If not, something is wrong with the file.
else if (currentFileSize % chunksData.ChunkSize == 0)
{
_chunkIndex = (int)(currentFileSize / chunksData.ChunkSize);
_logger.LogDebug(string.Format("Append is possible - starting from {0} chunk index.", _chunkIndex));
}
else
{
_logger.LogDebug(string.Format(
"File size {0} is not a multiply of chunk size: {1}. Append is not possible - recreating file.",
currentFileSize,
chunksData.ChunkSize));
_logger.LogDebug("Closing previous file stream...");
_fileStream.Close();
_fileStream.Dispose();
_logger.LogDebug("Previous file stream closed.");
OpenNewFileStream(path);
}
}
else
{
OpenNewFileStream(path);
}
}
