private Pack1Unarchiver(string packagePath, Pack1Meta metaData, string destinationDirPath, byte[] key, string suffix, BytesRange range)
{
Checks.ArgumentFileExists(packagePath, "packagePath");
Checks.ArgumentDirectoryExists(destinationDirPath, "destinationDirPath");
Checks.ArgumentNotNull(suffix, "suffix");
if (range.Start == 0)
{
Assert.AreEqual(MagicBytes.Pack1, MagicBytes.ReadFileType(packagePath), "Is not Pack1 format");
}
DebugLogger.LogConstructor();
DebugLogger.LogVariable(packagePath, "packagePath");
DebugLogger.LogVariable(destinationDirPath, "destinationDirPath");
DebugLogger.LogVariable(suffix, "suffix");
_packagePath = packagePath;
_metaData = metaData;
_destinationDirPath = destinationDirPath;
_suffix = suffix;
_range = range;
using (var sha256 = SHA256.Create())
{
_key = sha256.ComputeHash(key);
}
_iv = Convert.FromBase64String(_metaData.Iv);
}
public Task DownloadAsync(string appId, Media media, Stream stream, BytesRange range,
CancellationToken ct = default)
{
var fileName = CreateFileName(appId, media);
return(assetStore.DownloadAsync(fileName, stream, range, ct));
}
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
{
Guard.NotNull(stream, nameof(stream));
var key = GetKey(fileName, nameof(fileName));
try
{
var request = new GetObjectRequest
{
BucketName = options.Bucket,
Key = key
};
if (range.IsDefined)
{
request.ByteRange = new ByteRange(range.ToString());
}
using (var response = await s3Client.GetObjectAsync(request, ct))
{
await response.ResponseStream.CopyToAsync(stream, BufferSize, ct);
}
}
catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
throw new AssetNotFoundException(fileName, ex);
}
}
public void DownloadStreamsCertainData_For_SpecifiedRange()
{
var bytesRange = new BytesRange
{
Start = 100,
End = 200
};
var inputData = CreateRandomData(1024);
var inputDataStream = new MemoryStream(inputData, false);
var baseHttpDownloader = new BaseHttpDownloader("http://test_url.com", 10000,
MockHttpClient(inputDataStream, HttpStatusCode.OK), Substitute.For <ILogger>());
var outputDataStream = new MemoryStream(inputData.Length);
baseHttpDownloader.DataAvailable += (data, length) =>
{
Assert.IsTrue(length > 0, "Data length passed in DataAvailable event is not more than zero.");
outputDataStream.Write(data, 0, length);
};
baseHttpDownloader.SetBytesRange(bytesRange);
baseHttpDownloader.Download(CancellationToken.Empty);
outputDataStream.Seek(0, SeekOrigin.Begin);
ValidateOutput(inputData, outputDataStream, bytesRange);
}
public void JobQueuing_WithSpecifiedRangeAndOffset()
{
BytesRange range = BytesRangeUtils.Make(450, 830);
long offset = 512;
var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, offset, range, DataSize, _chunksData).ToList();
Assert.That(jobs.Count, Is.EqualTo(3));
var firstJob = jobs[0];
Assert.That(firstJob.Url, Is.EqualTo("test.com/someData.4"));
Assert.That(firstJob.Range.Start, Is.EqualTo(0));
Assert.That(firstJob.Range.End, Is.EqualTo(-1));
var middleJob = jobs[1];
Assert.That(middleJob.Url, Is.EqualTo("test.com/someData.5"));
Assert.That(middleJob.Range.Start, Is.EqualTo(0));
Assert.That(middleJob.Range.End, Is.EqualTo(-1));
var lastJob = jobs[2];
Assert.That(lastJob.Url, Is.EqualTo("test.com/someData.6"));
Assert.That(lastJob.Range.Start, Is.EqualTo(0));
Assert.That(lastJob.Range.End, Is.EqualTo(63));
}
public void ChunksCalculations_RangeIsExactWithChunks()
{
BytesRange range = BytesRangeUtils.Make(ChunkSize, (ChunkSize * 3) - 1); // Exactly 3 chunks
var chunksRange = range.Chunkify(_chunksData);
Assert.That(chunksRange, Is.EqualTo(range));
}
public void ChunksCalculations_WithRangeExactlyAsDataSize()
{
BytesRange range = BytesRangeUtils.Make(0, DataSize);
var chunksRange = range.Chunkify(_chunksData);
Assert.That(chunksRange.Start, Is.EqualTo(0));
Assert.That(chunksRange.End, Is.EqualTo(DataSize));
}
public void ChunksCalculations_WithFullRange()
{
BytesRange range = BytesRangeUtils.Full();
var chunksRange = range.Chunkify(_chunksData);
Assert.That(chunksRange.Start, Is.EqualTo(0));
Assert.That(chunksRange.End, Is.EqualTo(-1));
Assert.That(chunksRange.Contains(range));
}
public void ChunksCalculations_WithSpecifiedRange_2()
{
BytesRange range = BytesRangeUtils.Make(450, 830);
var chunksRange = range.Chunkify(_chunksData);
Assert.That(chunksRange.Start, Is.EqualTo(ChunkSize * 7));
Assert.That(chunksRange.End, Is.EqualTo(ChunkSize * 13 - 1));
Assert.That(chunksRange.Contains(range));
}
public void JobQueuing_SinglePartScenario()
{
BytesRange range = BytesRangeUtils.Make(315, 380);
var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, 0, range, DataSize, _chunksData).ToList();
Assert.That(jobs.Count, Is.EqualTo(1));
var job = jobs[0];
Assert.That(job.Url, Is.EqualTo("test.com/someData.2"));
Assert.That(job.Range.Start, Is.EqualTo(0));
Assert.That(job.Range.End, Is.EqualTo(-1).Or.EqualTo(127));
}
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range, CancellationToken ct = default)
{
Guard.NotNull(stream, nameof(stream));
var file = GetFile(fileName, nameof(fileName));
try
{
using (var fileStream = file.OpenRead())
{
await fileStream.CopyToAsync(stream, range, ct);
}
}
catch (FileNotFoundException ex)
{
throw new AssetNotFoundException(fileName, ex);
}
}
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
{
Guard.NotNullOrEmpty(fileName, nameof(fileName));
Guard.NotNull(stream, nameof(stream));
try
{
var blob = blobContainer.GetBlockBlobReference(fileName);
await using var blobStream = await blob.OpenReadAsync(null, null, null, ct);
await blobStream.CopyToAsync(stream, range, ct);
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == 404)
{
throw new AssetNotFoundException(fileName, ex);
}
}
/// <summary>
/// Makes the Start and End values of a given range correspond to provided chunk sizes.
/// </summary>
public static BytesRange Chunkify(this BytesRange range, ChunksData chunksData)
{
long chunkSize = chunksData.ChunkSize;
long bottom = (range.Start / chunkSize) * chunkSize;
if (range.End == -1)
{
return(new BytesRange(bottom, -1));
}
long top = ((range.End / chunkSize) + 1) * (chunkSize) - 1;
if (top > chunksData.Chunks.Length * chunkSize)
{
top = range.End;
}
return(new BytesRange(bottom, top));
}
public static async Task CopyToAsync(this Stream source, Stream target, BytesRange range, CancellationToken ct, bool skip = true)
{
var buffer = Pool.Rent(8192);
try
{
if (skip && range.From > 0)
{
source.Seek(range.From.Value, SeekOrigin.Begin);
}
var bytesLeft = range.Length;
while (true)
{
if (bytesLeft <= 0)
{
return;
}
ct.ThrowIfCancellationRequested();
var readLength = (int)Math.Min(buffer.Length, bytesLeft);
var read = await source.ReadAsync(buffer, 0, readLength, ct);
bytesLeft -= read;
if (read == 0)
{
return;
}
ct.ThrowIfCancellationRequested();
await target.WriteAsync(buffer, 0, read, ct);
}
}
finally
{
Pool.Return(buffer);
}
}
public void JobQueuing_WithFullRange()
{
BytesRange range = BytesRangeUtils.Full();
var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, 0, range, DataSize, _chunksData).ToList();
int expectedJobCount = (int)(DataSize / PartSize) + 1;
Assert.That(jobs.Count, Is.EqualTo(expectedJobCount));
var firstJob = jobs[0];
Assert.That(firstJob.Range.Start, Is.EqualTo(0));
Assert.That(firstJob.Range.End, Is.EqualTo(-1));
var lastJob = jobs[expectedJobCount - 1];
Assert.That(lastJob.Range.Start, Is.EqualTo(0));
Assert.That(lastJob.Range.End, Is.EqualTo(-1));
}
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range, CancellationToken ct = default)
{
Guard.NotNull(stream);
try
{
var name = GetFileName(fileName, nameof(fileName));
var options = range.IsDefined ? DownloadSeekable : DownloadDefault;
using (var readStream = await bucket.OpenDownloadStreamAsync(name, options, ct))
{
await readStream.CopyToAsync(stream, range, ct);
}
}
catch (GridFSFileNotFoundException ex)
{
throw new AssetNotFoundException(fileName, ex);
}
}
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
{
Guard.NotNullOrEmpty(fileName);
try
{
var downloadOptions = new DownloadObjectOptions();
if (range.IsDefined)
{
downloadOptions.Range = new RangeHeaderValue(range.From, range.To);
}
await storageClient.DownloadObjectAsync(bucketName, fileName, stream, downloadOptions, ct);
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound)
{
throw new AssetNotFoundException(fileName, ex);
}
}
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
{
Guard.NotNullOrEmpty(fileName, nameof(fileName));
Guard.NotNull(stream, nameof(stream));
using (var client = GetFtpClient())
{
try
{
using (var ftpStream = await client.OpenReadAsync(fileName, range.From ?? 0, ct))
{
await ftpStream.CopyToAsync(stream, range, ct, false);
}
}
catch (FtpException ex) when(IsNotFound(ex))
{
throw new AssetNotFoundException(fileName, ex);
}
}
}
public async Task ExecuteAsync(ActionContext context, FileCallbackResult result)
{
try
{
var(range, _, serveBody) = SetHeadersAndLog(context, result, result.FileSize, result.FileSize != null);
if (!string.IsNullOrWhiteSpace(result.FileDownloadName) && result.SendInline)
{
var headerValue = new ContentDispositionHeaderValue("inline");
headerValue.SetHttpFileName(result.FileDownloadName);
context.HttpContext.Response.Headers[HeaderNames.ContentDisposition] = headerValue.ToString();
}
if (serveBody)
{
var bytesRange = new BytesRange(range?.From, range?.To);
await result.Callback(context.HttpContext.Response.Body, bytesRange, context.HttpContext.RequestAborted);
}
}
catch (OperationCanceledException)
{
return;
}
catch (Exception e)
{
if (!context.HttpContext.Response.HasStarted && result.ErrorAs404)
{
context.HttpContext.Response.Headers.Clear();
context.HttpContext.Response.StatusCode = 404;
Logger.LogCritical(new EventId(99), e, "Failed to send result.");
}
else
{
throw;
}
}
}
public virtual async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
{
Guard.NotNullOrEmpty(fileName);
Guard.NotNull(stream);
if (!streams.TryGetValue(fileName, out var sourceStream))
{
throw new AssetNotFoundException(fileName);
}
using (await readerLock.LockAsync())
{
try
{
await sourceStream.CopyToAsync(stream, range, ct);
}
finally
{
sourceStream.Position = 0;
}
}
}
public void SetRange(BytesRange range)
{
_range = range;
}
private static void ValidateOutput(byte[] inputData, MemoryStream outputDataStream, BytesRange bytesRange)
{
byte[] buffer = new byte[1];
for (long i = bytesRange.Start; i < bytesRange.End; i++)
{
Assert.AreEqual(1, outputDataStream.Read(buffer, 0, 1),
string.Format("Cannot read output data stream at byte {0}.", i));
Assert.AreEqual(inputData[i], buffer[0], string.Format("Output data is different at byte {0}.", i));
}
}
public Pack1Unarchiver(string packagePath, Pack1Meta metaData, string destinationDirPath, string key, string suffix, BytesRange range)
: this(packagePath, metaData, destinationDirPath, Encoding.ASCII.GetBytes(key), suffix, range)
{
// do nothing
}
public Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
{
throw new NotSupportedException();
}
public async Task DownloadAsync(DomainId appId, DomainId id, long fileVersion, Stream stream, BytesRange range = default, CancellationToken ct = default)
{
try
{
var fileNameNew = GetFileName(appId, id, fileVersion);
await assetStore.DownloadAsync(fileNameNew, stream, range, ct);
}
catch (AssetNotFoundException)
{
var fileNameOld = GetFileName(id, fileVersion);
await assetStore.DownloadAsync(fileNameOld, stream, range, ct);
}
}
public static IEnumerable <DownloadJob> BuildDownloadJobQueue(ResourceUrl resourceUrl, long currentOffset, BytesRange range, long dataSize, ChunksData chunksData)
{
// The effective range is the original range contained within multiples of chunk size
BytesRange effectiveRange = range.Chunkify(chunksData);
var dataBounds = new BytesRange(currentOffset, -1);
BytesRange bounds = effectiveRange.ContainIn(dataBounds);
// An uncommon edge case might occur, in which bounds.Start is equal to dataSize,
// this would cause the download to continue forever, with every request crashing due to invalid range header
if (bounds.Start >= dataSize)
{
yield break;
}
if (resourceUrl.PartSize == 0)
{
yield return(new DownloadJob(resourceUrl.Url, bounds.Start, bounds.End));
yield break;
}
long partSize = resourceUrl.PartSize;
int firstPart = (int)(bounds.Start / partSize);
int totalPartCount = (int)(dataSize / partSize);
if (dataSize % partSize != 0)
{
totalPartCount += 1;
}
int lastPart = totalPartCount;
if (bounds.End != -1)
{
lastPart = (int)(bounds.End / partSize);
if (bounds.End % partSize != 0)
{
lastPart += 1;
}
}
long lastByte = dataSize - 1;
for (int i = firstPart; i < lastPart; i++)
{
string url = resourceUrl.Url;
if (i > 0)
{
// second and later indices should have index numebr at the end
url += "." + i;
}
BytesRange partRange = BytesRangeUtils.Make(i * partSize, (i + 1) * partSize - 1, lastByte);
BytesRange localBounds = bounds.LocalizeTo(partRange);
yield return(new DownloadJob(url, localBounds.Start, localBounds.End));
}
}
public DownloadJob(string url, long start = 0, long end = -1)
{
Url = url;
Range = new BytesRange(start, end);
}
public Task DownloadAsync(Guid id, long fileVersion, Stream stream, BytesRange range = default, CancellationToken ct = default)
{
var fileName = GetFileName(id, fileVersion);
return(assetStore.DownloadAsync(fileName, stream, range, ct));
}
public override void Execute(CancellationToken cancellationToken)
{
base.Execute(cancellationToken);
foreach (var entry in _entries)
{
var tempDirName = _packagePath + string.Format("{0}_{1}_{2}", entry.Name, entry.Offset, entry.Size);
TemporaryDirectory.ExecuteIn(tempDirName, (tempDir) =>
{
_logger.LogDebug(string.Format("Repairing the file {0}", entry.Name));
string packagePath = Path.Combine(tempDir.Path, ".pack" + Path.GetRandomFileName());
string unarchivePath = Path.Combine(tempDir.Path, Path.GetRandomFileName());
if (!Directory.Exists(unarchivePath))
{
DirectoryOperations.CreateDirectory(unarchivePath, cancellationToken);
}
var downloader = new ChunkedHttpDownloader(packagePath, _resource.ResourceUrls, _resource.ChunksData, _resource.Size);
long start = entry.Offset.GetValueOrDefault();
long end = (start + entry.Size.GetValueOrDefault()) - 1; // Offset by 1 to denote a byte index
var range = new BytesRange(start, end);
downloader.SetRange(range);
var effectiveRange = downloader.CalculateContainingChunksRange(range);
long totalData = effectiveRange.End == -1 ? _resource.Size - effectiveRange.Start : effectiveRange.End - effectiveRange.Start;
var downloadStatus = _entryStatus[entry].DownloadStatus;
var repairStatus = _entryStatus[entry].RepairStatus;
downloadStatus.IsActive.Value = true;
downloadStatus.TotalBytes.Value = totalData;
downloadStatus.Description.Value = "Downloading broken file...";
downloadStatus.Bytes.Value = 0;
downloader.DownloadProgressChanged += downloadedBytes =>
{
downloadStatus.Bytes.Value = downloadedBytes;
};
_logger.LogDebug(string.Format("Downloading the partial package with range {0}-{1}", start, end));
downloader.Download(cancellationToken);
downloadStatus.IsActive.Value = false;
repairStatus.IsActive.Value = true;
repairStatus.Description.Value = "Reparing broken file...";
repairStatus.Progress.Value = 0.0;
_logger.LogDebug("Unarchiving the package.");
var unarchiver = new Pack1Unarchiver(packagePath, _meta, unarchivePath, _packagePassword, _unpackingSuffix, effectiveRange);
unarchiver.UnarchiveProgressChanged += (name, isFile, unarchiveEntry, amount, entryProgress) =>
{
repairStatus.Progress.Value = entryProgress;
};
unarchiver.UnarchiveSingleFile(entry, cancellationToken);
EmplaceFile(Path.Combine(unarchivePath, entry.Name + _unpackingSuffix), Path.Combine(_localData.Path, entry.Name), cancellationToken);
repairStatus.IsActive.Value = false;
});
}
}
public BytesRange CalculateContainingChunksRange(BytesRange range)
{
return(range.Chunkify(_chunksData));
}
