private Pack1Unarchiver(string packagePath, Pack1Meta metaData, string destinationDirPath, byte[] key, string suffix, BytesRange range) { Checks.ArgumentFileExists(packagePath, "packagePath"); Checks.ArgumentDirectoryExists(destinationDirPath, "destinationDirPath"); Checks.ArgumentNotNull(suffix, "suffix"); if (range.Start == 0) { Assert.AreEqual(MagicBytes.Pack1, MagicBytes.ReadFileType(packagePath), "Is not Pack1 format"); } DebugLogger.LogConstructor(); DebugLogger.LogVariable(packagePath, "packagePath"); DebugLogger.LogVariable(destinationDirPath, "destinationDirPath"); DebugLogger.LogVariable(suffix, "suffix"); _packagePath = packagePath; _metaData = metaData; _destinationDirPath = destinationDirPath; _suffix = suffix; _range = range; using (var sha256 = SHA256.Create()) { _key = sha256.ComputeHash(key); } _iv = Convert.FromBase64String(_metaData.Iv); }
public Task DownloadAsync(string appId, Media media, Stream stream, BytesRange range, CancellationToken ct = default) { var fileName = CreateFileName(appId, media); return(assetStore.DownloadAsync(fileName, stream, range, ct)); }
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default) { Guard.NotNull(stream, nameof(stream)); var key = GetKey(fileName, nameof(fileName)); try { var request = new GetObjectRequest { BucketName = options.Bucket, Key = key }; if (range.IsDefined) { request.ByteRange = new ByteRange(range.ToString()); } using (var response = await s3Client.GetObjectAsync(request, ct)) { await response.ResponseStream.CopyToAsync(stream, BufferSize, ct); } } catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound) { throw new AssetNotFoundException(fileName, ex); } }
public void DownloadStreamsCertainData_For_SpecifiedRange() { var bytesRange = new BytesRange { Start = 100, End = 200 }; var inputData = CreateRandomData(1024); var inputDataStream = new MemoryStream(inputData, false); var baseHttpDownloader = new BaseHttpDownloader("http://test_url.com", 10000, MockHttpClient(inputDataStream, HttpStatusCode.OK), Substitute.For <ILogger>()); var outputDataStream = new MemoryStream(inputData.Length); baseHttpDownloader.DataAvailable += (data, length) => { Assert.IsTrue(length > 0, "Data length passed in DataAvailable event is not more than zero."); outputDataStream.Write(data, 0, length); }; baseHttpDownloader.SetBytesRange(bytesRange); baseHttpDownloader.Download(CancellationToken.Empty); outputDataStream.Seek(0, SeekOrigin.Begin); ValidateOutput(inputData, outputDataStream, bytesRange); }
public void JobQueuing_WithSpecifiedRangeAndOffset() { BytesRange range = BytesRangeUtils.Make(450, 830); long offset = 512; var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, offset, range, DataSize, _chunksData).ToList(); Assert.That(jobs.Count, Is.EqualTo(3)); var firstJob = jobs[0]; Assert.That(firstJob.Url, Is.EqualTo("test.com/someData.4")); Assert.That(firstJob.Range.Start, Is.EqualTo(0)); Assert.That(firstJob.Range.End, Is.EqualTo(-1)); var middleJob = jobs[1]; Assert.That(middleJob.Url, Is.EqualTo("test.com/someData.5")); Assert.That(middleJob.Range.Start, Is.EqualTo(0)); Assert.That(middleJob.Range.End, Is.EqualTo(-1)); var lastJob = jobs[2]; Assert.That(lastJob.Url, Is.EqualTo("test.com/someData.6")); Assert.That(lastJob.Range.Start, Is.EqualTo(0)); Assert.That(lastJob.Range.End, Is.EqualTo(63)); }
public void ChunksCalculations_RangeIsExactWithChunks() { BytesRange range = BytesRangeUtils.Make(ChunkSize, (ChunkSize * 3) - 1); // Exactly 3 chunks var chunksRange = range.Chunkify(_chunksData); Assert.That(chunksRange, Is.EqualTo(range)); }
public void ChunksCalculations_WithRangeExactlyAsDataSize() { BytesRange range = BytesRangeUtils.Make(0, DataSize); var chunksRange = range.Chunkify(_chunksData); Assert.That(chunksRange.Start, Is.EqualTo(0)); Assert.That(chunksRange.End, Is.EqualTo(DataSize)); }
public void ChunksCalculations_WithFullRange() { BytesRange range = BytesRangeUtils.Full(); var chunksRange = range.Chunkify(_chunksData); Assert.That(chunksRange.Start, Is.EqualTo(0)); Assert.That(chunksRange.End, Is.EqualTo(-1)); Assert.That(chunksRange.Contains(range)); }
public void ChunksCalculations_WithSpecifiedRange_2() { BytesRange range = BytesRangeUtils.Make(450, 830); var chunksRange = range.Chunkify(_chunksData); Assert.That(chunksRange.Start, Is.EqualTo(ChunkSize * 7)); Assert.That(chunksRange.End, Is.EqualTo(ChunkSize * 13 - 1)); Assert.That(chunksRange.Contains(range)); }
public void JobQueuing_SinglePartScenario() { BytesRange range = BytesRangeUtils.Make(315, 380); var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, 0, range, DataSize, _chunksData).ToList(); Assert.That(jobs.Count, Is.EqualTo(1)); var job = jobs[0]; Assert.That(job.Url, Is.EqualTo("test.com/someData.2")); Assert.That(job.Range.Start, Is.EqualTo(0)); Assert.That(job.Range.End, Is.EqualTo(-1).Or.EqualTo(127)); }
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range, CancellationToken ct = default) { Guard.NotNull(stream, nameof(stream)); var file = GetFile(fileName, nameof(fileName)); try { using (var fileStream = file.OpenRead()) { await fileStream.CopyToAsync(stream, range, ct); } } catch (FileNotFoundException ex) { throw new AssetNotFoundException(fileName, ex); } }
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default) { Guard.NotNullOrEmpty(fileName, nameof(fileName)); Guard.NotNull(stream, nameof(stream)); try { var blob = blobContainer.GetBlockBlobReference(fileName); await using var blobStream = await blob.OpenReadAsync(null, null, null, ct); await blobStream.CopyToAsync(stream, range, ct); } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == 404) { throw new AssetNotFoundException(fileName, ex); } }
/// <summary> /// Makes the Start and End values of a given range correspond to provided chunk sizes. /// </summary> public static BytesRange Chunkify(this BytesRange range, ChunksData chunksData) { long chunkSize = chunksData.ChunkSize; long bottom = (range.Start / chunkSize) * chunkSize; if (range.End == -1) { return(new BytesRange(bottom, -1)); } long top = ((range.End / chunkSize) + 1) * (chunkSize) - 1; if (top > chunksData.Chunks.Length * chunkSize) { top = range.End; } return(new BytesRange(bottom, top)); }
public static async Task CopyToAsync(this Stream source, Stream target, BytesRange range, CancellationToken ct, bool skip = true) { var buffer = Pool.Rent(8192); try { if (skip && range.From > 0) { source.Seek(range.From.Value, SeekOrigin.Begin); } var bytesLeft = range.Length; while (true) { if (bytesLeft <= 0) { return; } ct.ThrowIfCancellationRequested(); var readLength = (int)Math.Min(buffer.Length, bytesLeft); var read = await source.ReadAsync(buffer, 0, readLength, ct); bytesLeft -= read; if (read == 0) { return; } ct.ThrowIfCancellationRequested(); await target.WriteAsync(buffer, 0, read, ct); } } finally { Pool.Return(buffer); } }
public void JobQueuing_WithFullRange() { BytesRange range = BytesRangeUtils.Full(); var jobs = ChunkedHttpDownloader.BuildDownloadJobQueue(_url, 0, range, DataSize, _chunksData).ToList(); int expectedJobCount = (int)(DataSize / PartSize) + 1; Assert.That(jobs.Count, Is.EqualTo(expectedJobCount)); var firstJob = jobs[0]; Assert.That(firstJob.Range.Start, Is.EqualTo(0)); Assert.That(firstJob.Range.End, Is.EqualTo(-1)); var lastJob = jobs[expectedJobCount - 1]; Assert.That(lastJob.Range.Start, Is.EqualTo(0)); Assert.That(lastJob.Range.End, Is.EqualTo(-1)); }
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range, CancellationToken ct = default) { Guard.NotNull(stream); try { var name = GetFileName(fileName, nameof(fileName)); var options = range.IsDefined ? DownloadSeekable : DownloadDefault; using (var readStream = await bucket.OpenDownloadStreamAsync(name, options, ct)) { await readStream.CopyToAsync(stream, range, ct); } } catch (GridFSFileNotFoundException ex) { throw new AssetNotFoundException(fileName, ex); } }
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default) { Guard.NotNullOrEmpty(fileName); try { var downloadOptions = new DownloadObjectOptions(); if (range.IsDefined) { downloadOptions.Range = new RangeHeaderValue(range.From, range.To); } await storageClient.DownloadObjectAsync(bucketName, fileName, stream, downloadOptions, ct); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound) { throw new AssetNotFoundException(fileName, ex); } }
public async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default) { Guard.NotNullOrEmpty(fileName, nameof(fileName)); Guard.NotNull(stream, nameof(stream)); using (var client = GetFtpClient()) { try { using (var ftpStream = await client.OpenReadAsync(fileName, range.From ?? 0, ct)) { await ftpStream.CopyToAsync(stream, range, ct, false); } } catch (FtpException ex) when(IsNotFound(ex)) { throw new AssetNotFoundException(fileName, ex); } } }
public async Task ExecuteAsync(ActionContext context, FileCallbackResult result) { try { var(range, _, serveBody) = SetHeadersAndLog(context, result, result.FileSize, result.FileSize != null); if (!string.IsNullOrWhiteSpace(result.FileDownloadName) && result.SendInline) { var headerValue = new ContentDispositionHeaderValue("inline"); headerValue.SetHttpFileName(result.FileDownloadName); context.HttpContext.Response.Headers[HeaderNames.ContentDisposition] = headerValue.ToString(); } if (serveBody) { var bytesRange = new BytesRange(range?.From, range?.To); await result.Callback(context.HttpContext.Response.Body, bytesRange, context.HttpContext.RequestAborted); } } catch (OperationCanceledException) { return; } catch (Exception e) { if (!context.HttpContext.Response.HasStarted && result.ErrorAs404) { context.HttpContext.Response.Headers.Clear(); context.HttpContext.Response.StatusCode = 404; Logger.LogCritical(new EventId(99), e, "Failed to send result."); } else { throw; } } }
public virtual async Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default) { Guard.NotNullOrEmpty(fileName); Guard.NotNull(stream); if (!streams.TryGetValue(fileName, out var sourceStream)) { throw new AssetNotFoundException(fileName); } using (await readerLock.LockAsync()) { try { await sourceStream.CopyToAsync(stream, range, ct); } finally { sourceStream.Position = 0; } } }
public void SetRange(BytesRange range) { _range = range; }
private static void ValidateOutput(byte[] inputData, MemoryStream outputDataStream, BytesRange bytesRange) { byte[] buffer = new byte[1]; for (long i = bytesRange.Start; i < bytesRange.End; i++) { Assert.AreEqual(1, outputDataStream.Read(buffer, 0, 1), string.Format("Cannot read output data stream at byte {0}.", i)); Assert.AreEqual(inputData[i], buffer[0], string.Format("Output data is different at byte {0}.", i)); } }
public Pack1Unarchiver(string packagePath, Pack1Meta metaData, string destinationDirPath, string key, string suffix, BytesRange range) : this(packagePath, metaData, destinationDirPath, Encoding.ASCII.GetBytes(key), suffix, range) { // do nothing }
public Task DownloadAsync(string fileName, Stream stream, BytesRange range = default, CancellationToken ct = default) { throw new NotSupportedException(); }
public async Task DownloadAsync(DomainId appId, DomainId id, long fileVersion, Stream stream, BytesRange range = default, CancellationToken ct = default) { try { var fileNameNew = GetFileName(appId, id, fileVersion); await assetStore.DownloadAsync(fileNameNew, stream, range, ct); } catch (AssetNotFoundException) { var fileNameOld = GetFileName(id, fileVersion); await assetStore.DownloadAsync(fileNameOld, stream, range, ct); } }
public static IEnumerable <DownloadJob> BuildDownloadJobQueue(ResourceUrl resourceUrl, long currentOffset, BytesRange range, long dataSize, ChunksData chunksData) { // The effective range is the original range contained within multiples of chunk size BytesRange effectiveRange = range.Chunkify(chunksData); var dataBounds = new BytesRange(currentOffset, -1); BytesRange bounds = effectiveRange.ContainIn(dataBounds); // An uncommon edge case might occur, in which bounds.Start is equal to dataSize, // this would cause the download to continue forever, with every request crashing due to invalid range header if (bounds.Start >= dataSize) { yield break; } if (resourceUrl.PartSize == 0) { yield return(new DownloadJob(resourceUrl.Url, bounds.Start, bounds.End)); yield break; } long partSize = resourceUrl.PartSize; int firstPart = (int)(bounds.Start / partSize); int totalPartCount = (int)(dataSize / partSize); if (dataSize % partSize != 0) { totalPartCount += 1; } int lastPart = totalPartCount; if (bounds.End != -1) { lastPart = (int)(bounds.End / partSize); if (bounds.End % partSize != 0) { lastPart += 1; } } long lastByte = dataSize - 1; for (int i = firstPart; i < lastPart; i++) { string url = resourceUrl.Url; if (i > 0) { // second and later indices should have index numebr at the end url += "." + i; } BytesRange partRange = BytesRangeUtils.Make(i * partSize, (i + 1) * partSize - 1, lastByte); BytesRange localBounds = bounds.LocalizeTo(partRange); yield return(new DownloadJob(url, localBounds.Start, localBounds.End)); } }
public DownloadJob(string url, long start = 0, long end = -1) { Url = url; Range = new BytesRange(start, end); }
public Task DownloadAsync(Guid id, long fileVersion, Stream stream, BytesRange range = default, CancellationToken ct = default) { var fileName = GetFileName(id, fileVersion); return(assetStore.DownloadAsync(fileName, stream, range, ct)); }
public override void Execute(CancellationToken cancellationToken) { base.Execute(cancellationToken); foreach (var entry in _entries) { var tempDirName = _packagePath + string.Format("{0}_{1}_{2}", entry.Name, entry.Offset, entry.Size); TemporaryDirectory.ExecuteIn(tempDirName, (tempDir) => { _logger.LogDebug(string.Format("Repairing the file {0}", entry.Name)); string packagePath = Path.Combine(tempDir.Path, ".pack" + Path.GetRandomFileName()); string unarchivePath = Path.Combine(tempDir.Path, Path.GetRandomFileName()); if (!Directory.Exists(unarchivePath)) { DirectoryOperations.CreateDirectory(unarchivePath, cancellationToken); } var downloader = new ChunkedHttpDownloader(packagePath, _resource.ResourceUrls, _resource.ChunksData, _resource.Size); long start = entry.Offset.GetValueOrDefault(); long end = (start + entry.Size.GetValueOrDefault()) - 1; // Offset by 1 to denote a byte index var range = new BytesRange(start, end); downloader.SetRange(range); var effectiveRange = downloader.CalculateContainingChunksRange(range); long totalData = effectiveRange.End == -1 ? _resource.Size - effectiveRange.Start : effectiveRange.End - effectiveRange.Start; var downloadStatus = _entryStatus[entry].DownloadStatus; var repairStatus = _entryStatus[entry].RepairStatus; downloadStatus.IsActive.Value = true; downloadStatus.TotalBytes.Value = totalData; downloadStatus.Description.Value = "Downloading broken file..."; downloadStatus.Bytes.Value = 0; downloader.DownloadProgressChanged += downloadedBytes => { downloadStatus.Bytes.Value = downloadedBytes; }; _logger.LogDebug(string.Format("Downloading the partial package with range {0}-{1}", start, end)); downloader.Download(cancellationToken); downloadStatus.IsActive.Value = false; repairStatus.IsActive.Value = true; repairStatus.Description.Value = "Reparing broken file..."; repairStatus.Progress.Value = 0.0; _logger.LogDebug("Unarchiving the package."); var unarchiver = new Pack1Unarchiver(packagePath, _meta, unarchivePath, _packagePassword, _unpackingSuffix, effectiveRange); unarchiver.UnarchiveProgressChanged += (name, isFile, unarchiveEntry, amount, entryProgress) => { repairStatus.Progress.Value = entryProgress; }; unarchiver.UnarchiveSingleFile(entry, cancellationToken); EmplaceFile(Path.Combine(unarchivePath, entry.Name + _unpackingSuffix), Path.Combine(_localData.Path, entry.Name), cancellationToken); repairStatus.IsActive.Value = false; }); } }
public BytesRange CalculateContainingChunksRange(BytesRange range) { return(range.Chunkify(_chunksData)); }