private Pack1Unarchiver(string packagePath, Pack1Meta metaData, string destinationDirPath, byte[] key, string suffix, BytesRange range)
        {
            Checks.ArgumentFileExists(packagePath, "packagePath");
            Checks.ArgumentDirectoryExists(destinationDirPath, "destinationDirPath");
            Checks.ArgumentNotNull(suffix, "suffix");

            if (range.Start == 0)
            {
                Assert.AreEqual(MagicBytes.Pack1, MagicBytes.ReadFileType(packagePath), "Is not Pack1 format");
            }

            DebugLogger.LogConstructor();
            DebugLogger.LogVariable(packagePath, "packagePath");
            DebugLogger.LogVariable(destinationDirPath, "destinationDirPath");
            DebugLogger.LogVariable(suffix, "suffix");

            _packagePath        = packagePath;
            _metaData           = metaData;
            _destinationDirPath = destinationDirPath;
            _suffix             = suffix;
            _range = range;

            using (var sha256 = SHA256.Create())
            {
                _key = sha256.ComputeHash(key);
            }

            _iv = Convert.FromBase64String(_metaData.Iv);
        }
Exemplo n.º 2
0
        public Task DownloadAsync(string appId, Media media, Stream stream, BytesRange range,
                                  CancellationToken ct = default)
        {
            var fileName = CreateFileName(appId, media);

            return(assetStore.DownloadAsync(fileName, stream, range, ct));
        }
Exemplo n.º 3
0
        public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
        {
            Guard.NotNull(stream, nameof(stream));

            var key = GetKey(fileName, nameof(fileName));

            try
            {
                var request = new GetObjectRequest
                {
                    BucketName = options.Bucket,
                    Key        = key
                };

                if (range.IsDefined)
                {
                    request.ByteRange = new ByteRange(range.ToString());
                }

                using (var response = await s3Client.GetObjectAsync(request, ct))
                {
                    await response.ResponseStream.CopyToAsync(stream, BufferSize, ct);
                }
            }
            catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound)
            {
                throw new AssetNotFoundException(fileName, ex);
            }
        }
Exemplo n.º 4
0
    public void DownloadStreamsCertainData_For_SpecifiedRange()
    {
        var bytesRange = new BytesRange
        {
            Start = 100,
            End   = 200
        };

        var inputData       = CreateRandomData(1024);
        var inputDataStream = new MemoryStream(inputData, false);

        var baseHttpDownloader = new BaseHttpDownloader("http://test_url.com", 10000,
                                                        MockHttpClient(inputDataStream, HttpStatusCode.OK), Substitute.For <ILogger>());


        var outputDataStream = new MemoryStream(inputData.Length);

        baseHttpDownloader.DataAvailable += (data, length) =>
        {
            Assert.IsTrue(length > 0, "Data length passed in DataAvailable event is not more than zero.");

            outputDataStream.Write(data, 0, length);
        };

        baseHttpDownloader.SetBytesRange(bytesRange);

        baseHttpDownloader.Download(CancellationToken.Empty);

        outputDataStream.Seek(0, SeekOrigin.Begin);

        ValidateOutput(inputData, outputDataStream, bytesRange);
    }
    public void JobQueuing_WithSpecifiedRangeAndOffset()
    {
        BytesRange range = BytesRangeUtils.Make(450, 830);

        long offset = 512;

        var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, offset, range, DataSize, _chunksData).ToList();

        Assert.That(jobs.Count, Is.EqualTo(3));

        var firstJob = jobs[0];

        Assert.That(firstJob.Url, Is.EqualTo("test.com/someData.4"));
        Assert.That(firstJob.Range.Start, Is.EqualTo(0));
        Assert.That(firstJob.Range.End, Is.EqualTo(-1));

        var middleJob = jobs[1];

        Assert.That(middleJob.Url, Is.EqualTo("test.com/someData.5"));
        Assert.That(middleJob.Range.Start, Is.EqualTo(0));
        Assert.That(middleJob.Range.End, Is.EqualTo(-1));

        var lastJob = jobs[2];

        Assert.That(lastJob.Url, Is.EqualTo("test.com/someData.6"));

        Assert.That(lastJob.Range.Start, Is.EqualTo(0));
        Assert.That(lastJob.Range.End, Is.EqualTo(63));
    }
    public void ChunksCalculations_RangeIsExactWithChunks()
    {
        BytesRange range = BytesRangeUtils.Make(ChunkSize, (ChunkSize * 3) - 1); // Exactly 3 chunks

        var chunksRange = range.Chunkify(_chunksData);

        Assert.That(chunksRange, Is.EqualTo(range));
    }
    public void ChunksCalculations_WithRangeExactlyAsDataSize()
    {
        BytesRange range = BytesRangeUtils.Make(0, DataSize);

        var chunksRange = range.Chunkify(_chunksData);

        Assert.That(chunksRange.Start, Is.EqualTo(0));
        Assert.That(chunksRange.End, Is.EqualTo(DataSize));
    }
    public void ChunksCalculations_WithFullRange()
    {
        BytesRange range = BytesRangeUtils.Full();

        var chunksRange = range.Chunkify(_chunksData);

        Assert.That(chunksRange.Start, Is.EqualTo(0));
        Assert.That(chunksRange.End, Is.EqualTo(-1));

        Assert.That(chunksRange.Contains(range));
    }
    public void ChunksCalculations_WithSpecifiedRange_2()
    {
        BytesRange range = BytesRangeUtils.Make(450, 830);

        var chunksRange = range.Chunkify(_chunksData);

        Assert.That(chunksRange.Start, Is.EqualTo(ChunkSize * 7));
        Assert.That(chunksRange.End, Is.EqualTo(ChunkSize * 13 - 1));

        Assert.That(chunksRange.Contains(range));
    }
    public void JobQueuing_SinglePartScenario()
    {
        BytesRange range = BytesRangeUtils.Make(315, 380);

        var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, 0, range, DataSize, _chunksData).ToList();

        Assert.That(jobs.Count, Is.EqualTo(1));

        var job = jobs[0];

        Assert.That(job.Url, Is.EqualTo("test.com/someData.2"));
        Assert.That(job.Range.Start, Is.EqualTo(0));
        Assert.That(job.Range.End, Is.EqualTo(-1).Or.EqualTo(127));
    }
Exemplo n.º 11
0
        public async Task DownloadAsync(string fileName, Stream stream, BytesRange range, CancellationToken ct = default)
        {
            Guard.NotNull(stream, nameof(stream));

            var file = GetFile(fileName, nameof(fileName));

            try
            {
                using (var fileStream = file.OpenRead())
                {
                    await fileStream.CopyToAsync(stream, range, ct);
                }
            }
            catch (FileNotFoundException ex)
            {
                throw new AssetNotFoundException(fileName, ex);
            }
        }
Exemplo n.º 12
0
        public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
        {
            Guard.NotNullOrEmpty(fileName, nameof(fileName));
            Guard.NotNull(stream, nameof(stream));

            try
            {
                var blob = blobContainer.GetBlockBlobReference(fileName);

                await using var blobStream = await blob.OpenReadAsync(null, null, null, ct);

                await blobStream.CopyToAsync(stream, range, ct);
            }
            catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == 404)
            {
                throw new AssetNotFoundException(fileName, ex);
            }
        }
        /// <summary>
        /// Makes the Start and End values of a given range correspond to provided chunk sizes.
        /// </summary>
        public static BytesRange Chunkify(this BytesRange range, ChunksData chunksData)
        {
            long chunkSize = chunksData.ChunkSize;
            long bottom    = (range.Start / chunkSize) * chunkSize;

            if (range.End == -1)
            {
                return(new BytesRange(bottom, -1));
            }

            long top = ((range.End / chunkSize) + 1) * (chunkSize) - 1;

            if (top > chunksData.Chunks.Length * chunkSize)
            {
                top = range.End;
            }

            return(new BytesRange(bottom, top));
        }
Exemplo n.º 14
0
        public static async Task CopyToAsync(this Stream source, Stream target, BytesRange range, CancellationToken ct, bool skip = true)
        {
            var buffer = Pool.Rent(8192);

            try
            {
                if (skip && range.From > 0)
                {
                    source.Seek(range.From.Value, SeekOrigin.Begin);
                }

                var bytesLeft = range.Length;

                while (true)
                {
                    if (bytesLeft <= 0)
                    {
                        return;
                    }

                    ct.ThrowIfCancellationRequested();

                    var readLength = (int)Math.Min(buffer.Length, bytesLeft);

                    var read = await source.ReadAsync(buffer, 0, readLength, ct);

                    bytesLeft -= read;

                    if (read == 0)
                    {
                        return;
                    }

                    ct.ThrowIfCancellationRequested();

                    await target.WriteAsync(buffer, 0, read, ct);
                }
            }
            finally
            {
                Pool.Return(buffer);
            }
        }
    public void JobQueuing_WithFullRange()
    {
        BytesRange range = BytesRangeUtils.Full();

        var jobs             = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, 0, range, DataSize, _chunksData).ToList();
        int expectedJobCount = (int)(DataSize / PartSize) + 1;

        Assert.That(jobs.Count, Is.EqualTo(expectedJobCount));

        var firstJob = jobs[0];

        Assert.That(firstJob.Range.Start, Is.EqualTo(0));
        Assert.That(firstJob.Range.End, Is.EqualTo(-1));

        var lastJob = jobs[expectedJobCount - 1];

        Assert.That(lastJob.Range.Start, Is.EqualTo(0));
        Assert.That(lastJob.Range.End, Is.EqualTo(-1));
    }
Exemplo n.º 16
0
        public async Task DownloadAsync(string fileName, Stream stream, BytesRange range, CancellationToken ct = default)
        {
            Guard.NotNull(stream);

            try
            {
                var name = GetFileName(fileName, nameof(fileName));

                var options = range.IsDefined ? DownloadSeekable : DownloadDefault;

                using (var readStream = await bucket.OpenDownloadStreamAsync(name, options, ct))
                {
                    await readStream.CopyToAsync(stream, range, ct);
                }
            }
            catch (GridFSFileNotFoundException ex)
            {
                throw new AssetNotFoundException(fileName, ex);
            }
        }
Exemplo n.º 17
0
        public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
        {
            Guard.NotNullOrEmpty(fileName);

            try
            {
                var downloadOptions = new DownloadObjectOptions();

                if (range.IsDefined)
                {
                    downloadOptions.Range = new RangeHeaderValue(range.From, range.To);
                }

                await storageClient.DownloadObjectAsync(bucketName, fileName, stream, downloadOptions, ct);
            }
            catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound)
            {
                throw new AssetNotFoundException(fileName, ex);
            }
        }
Exemplo n.º 18
0
        public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
        {
            Guard.NotNullOrEmpty(fileName, nameof(fileName));
            Guard.NotNull(stream, nameof(stream));

            using (var client = GetFtpClient())
            {
                try
                {
                    using (var ftpStream = await client.OpenReadAsync(fileName, range.From ?? 0, ct))
                    {
                        await ftpStream.CopyToAsync(stream, range, ct, false);
                    }
                }
                catch (FtpException ex) when(IsNotFound(ex))
                {
                    throw new AssetNotFoundException(fileName, ex);
                }
            }
        }
Exemplo n.º 19
0
        public async Task ExecuteAsync(ActionContext context, FileCallbackResult result)
        {
            try
            {
                var(range, _, serveBody) = SetHeadersAndLog(context, result, result.FileSize, result.FileSize != null);

                if (!string.IsNullOrWhiteSpace(result.FileDownloadName) && result.SendInline)
                {
                    var headerValue = new ContentDispositionHeaderValue("inline");

                    headerValue.SetHttpFileName(result.FileDownloadName);

                    context.HttpContext.Response.Headers[HeaderNames.ContentDisposition] = headerValue.ToString();
                }

                if (serveBody)
                {
                    var bytesRange = new BytesRange(range?.From, range?.To);

                    await result.Callback(context.HttpContext.Response.Body, bytesRange, context.HttpContext.RequestAborted);
                }
            }
            catch (OperationCanceledException)
            {
                return;
            }
            catch (Exception e)
            {
                if (!context.HttpContext.Response.HasStarted && result.ErrorAs404)
                {
                    context.HttpContext.Response.Headers.Clear();
                    context.HttpContext.Response.StatusCode = 404;

                    Logger.LogCritical(new EventId(99), e, "Failed to send result.");
                }
                else
                {
                    throw;
                }
            }
        }
Exemplo n.º 20
0
        public virtual async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
        {
            Guard.NotNullOrEmpty(fileName);
            Guard.NotNull(stream);

            if (!streams.TryGetValue(fileName, out var sourceStream))
            {
                throw new AssetNotFoundException(fileName);
            }

            using (await readerLock.LockAsync())
            {
                try
                {
                    await sourceStream.CopyToAsync(stream, range, ct);
                }
                finally
                {
                    sourceStream.Position = 0;
                }
            }
        }
Exemplo n.º 21
0
 public void SetRange(BytesRange range)
 {
     _range = range;
 }
Exemplo n.º 22
0
    private static void ValidateOutput(byte[] inputData, MemoryStream outputDataStream, BytesRange bytesRange)
    {
        byte[] buffer = new byte[1];

        for (long i = bytesRange.Start; i < bytesRange.End; i++)
        {
            Assert.AreEqual(1, outputDataStream.Read(buffer, 0, 1),
                            string.Format("Cannot read output data stream at byte {0}.", i));
            Assert.AreEqual(inputData[i], buffer[0], string.Format("Output data is different at byte {0}.", i));
        }
    }
 public Pack1Unarchiver(string packagePath, Pack1Meta metaData, string destinationDirPath, string key, string suffix, BytesRange range)
     : this(packagePath, metaData, destinationDirPath, Encoding.ASCII.GetBytes(key), suffix, range)
 {
     // do nothing
 }
Exemplo n.º 24
0
 public Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default)
 {
     throw new NotSupportedException();
 }
Exemplo n.º 25
0
        public async Task DownloadAsync(DomainId appId, DomainId id, long fileVersion, Stream stream, BytesRange range = default, CancellationToken ct = default)
        {
            try
            {
                var fileNameNew = GetFileName(appId, id, fileVersion);

                await assetStore.DownloadAsync(fileNameNew, stream, range, ct);
            }
            catch (AssetNotFoundException)
            {
                var fileNameOld = GetFileName(id, fileVersion);

                await assetStore.DownloadAsync(fileNameOld, stream, range, ct);
            }
        }
Exemplo n.º 26
0
        public static IEnumerable <DownloadJob> BuildDownloadJobQueue(ResourceUrl resourceUrl, long currentOffset, BytesRange range, long dataSize, ChunksData chunksData)
        {
            // The effective range is the original range contained within multiples of chunk size
            BytesRange effectiveRange = range.Chunkify(chunksData);
            var        dataBounds     = new BytesRange(currentOffset, -1);

            BytesRange bounds = effectiveRange.ContainIn(dataBounds);

            // An uncommon edge case might occur, in which bounds.Start is equal to dataSize,
            // this would cause the download to continue forever, with every request crashing due to invalid range header
            if (bounds.Start >= dataSize)
            {
                yield break;
            }

            if (resourceUrl.PartSize == 0)
            {
                yield return(new DownloadJob(resourceUrl.Url, bounds.Start, bounds.End));

                yield break;
            }

            long partSize = resourceUrl.PartSize;

            int firstPart      = (int)(bounds.Start / partSize);
            int totalPartCount = (int)(dataSize / partSize);

            if (dataSize % partSize != 0)
            {
                totalPartCount += 1;
            }


            int lastPart = totalPartCount;

            if (bounds.End != -1)
            {
                lastPart = (int)(bounds.End / partSize);
                if (bounds.End % partSize != 0)
                {
                    lastPart += 1;
                }
            }

            long lastByte = dataSize - 1;

            for (int i = firstPart; i < lastPart; i++)
            {
                string url = resourceUrl.Url;
                if (i > 0)
                {
                    // second and later indices should have index numebr at the end
                    url += "." + i;
                }

                BytesRange partRange   = BytesRangeUtils.Make(i * partSize, (i + 1) * partSize - 1, lastByte);
                BytesRange localBounds = bounds.LocalizeTo(partRange);

                yield return(new DownloadJob(url, localBounds.Start, localBounds.End));
            }
        }
Exemplo n.º 27
0
 public DownloadJob(string url, long start = 0, long end = -1)
 {
     Url   = url;
     Range = new BytesRange(start, end);
 }
Exemplo n.º 28
0
        public Task DownloadAsync(Guid id, long fileVersion, Stream stream, BytesRange range = default, CancellationToken ct = default)
        {
            var fileName = GetFileName(id, fileVersion);

            return(assetStore.DownloadAsync(fileName, stream, range, ct));
        }
Exemplo n.º 29
0
        public override void Execute(CancellationToken cancellationToken)
        {
            base.Execute(cancellationToken);

            foreach (var entry in _entries)
            {
                var tempDirName = _packagePath + string.Format("{0}_{1}_{2}", entry.Name, entry.Offset, entry.Size);
                TemporaryDirectory.ExecuteIn(tempDirName, (tempDir) =>
                {
                    _logger.LogDebug(string.Format("Repairing the file {0}", entry.Name));
                    string packagePath   = Path.Combine(tempDir.Path, ".pack" + Path.GetRandomFileName());
                    string unarchivePath = Path.Combine(tempDir.Path, Path.GetRandomFileName());

                    if (!Directory.Exists(unarchivePath))
                    {
                        DirectoryOperations.CreateDirectory(unarchivePath, cancellationToken);
                    }

                    var downloader = new ChunkedHttpDownloader(packagePath, _resource.ResourceUrls, _resource.ChunksData, _resource.Size);

                    long start = entry.Offset.GetValueOrDefault();
                    long end   = (start + entry.Size.GetValueOrDefault()) - 1; // Offset by 1 to denote a byte index

                    var range = new BytesRange(start, end);

                    downloader.SetRange(range);
                    var effectiveRange = downloader.CalculateContainingChunksRange(range);

                    long totalData = effectiveRange.End == -1 ? _resource.Size - effectiveRange.Start : effectiveRange.End - effectiveRange.Start;

                    var downloadStatus = _entryStatus[entry].DownloadStatus;
                    var repairStatus   = _entryStatus[entry].RepairStatus;

                    downloadStatus.IsActive.Value    = true;
                    downloadStatus.TotalBytes.Value  = totalData;
                    downloadStatus.Description.Value = "Downloading broken file...";
                    downloadStatus.Bytes.Value       = 0;

                    downloader.DownloadProgressChanged += downloadedBytes =>
                    {
                        downloadStatus.Bytes.Value = downloadedBytes;
                    };

                    _logger.LogDebug(string.Format("Downloading the partial package with range {0}-{1}", start, end));
                    downloader.Download(cancellationToken);

                    downloadStatus.IsActive.Value = false;

                    repairStatus.IsActive.Value    = true;
                    repairStatus.Description.Value = "Reparing broken file...";
                    repairStatus.Progress.Value    = 0.0;

                    _logger.LogDebug("Unarchiving the package.");
                    var unarchiver = new Pack1Unarchiver(packagePath, _meta, unarchivePath, _packagePassword, _unpackingSuffix, effectiveRange);
                    unarchiver.UnarchiveProgressChanged += (name, isFile, unarchiveEntry, amount, entryProgress) =>
                    {
                        repairStatus.Progress.Value = entryProgress;
                    };

                    unarchiver.UnarchiveSingleFile(entry, cancellationToken);

                    EmplaceFile(Path.Combine(unarchivePath, entry.Name + _unpackingSuffix), Path.Combine(_localData.Path, entry.Name), cancellationToken);

                    repairStatus.IsActive.Value = false;
                });
            }
        }
Exemplo n.º 30
0
 public BytesRange CalculateContainingChunksRange(BytesRange range)
 {
     return(range.Chunkify(_chunksData));
 }