private static async Task <CompressionDetails> Multiplex(Stream outputStream, CompressionDetails compressionDetailsWithSize) { CompressionDetails v = compressionDetailsWithSize; multiplexState.AddOrUpdate(v.Sequence, v, (i, s) => s); var orderedState = multiplexState.Where(k => !k.Value.IsProcessed).OrderBy(kv => kv.Key); int index = lastProcessedBuffer; foreach (var item in orderedState) { CompressionDetails c = item.Value; if (c.Sequence == (lastProcessedBuffer + 1)) { await outputStream.WriteAsync(c.ChunkSize, 0, c.ChunkSize.Length); await outputStream.WriteAsync(c.CompressedDataSize, 0, c.CompressedDataSize.Length); await outputStream.WriteAsync(c.Bytes, 0, c.Bytes.Length); lastProcessedBuffer = c.Sequence; c.IsProcessed = true; multiplexState[c.Sequence] = c; } else { break; } } return(v); }
private static async Task <CompressionDetails> Compress(CompressionDetails compressionDetails) { byte[] data = compressionDetails.Bytes; using (MemoryStream compressedDataStream = new MemoryStream()) { using (GZipStream streamCompressed = new GZipStream(compressedDataStream, CompressionMode.Compress)) { // write chunk in the compressed stream await streamCompressed.WriteAsync(data, 0, data.Length); } byte[] compressedData = compressedDataStream.GetBuffer(); byte[] compressedDatasize = BitConverter.GetBytes(compressedData.Length); CompressionDetails v = new CompressionDetails { Bytes = compressedData, ChunkSize = compressionDetails.ChunkSize, Sequence = compressionDetails.Sequence, CompressedDataSize = compressedDatasize }; return(v); } }
private static async Task<CompressionDetails> Multiplex(Stream outputStream, CompressionDetails compressionDetailsWithSize) { CompressionDetails v = compressionDetailsWithSize; multiplexState.AddOrUpdate(v.Sequence, v, (i, s) => s); var orderedState = multiplexState.Where(k => !k.Value.IsProcessed).OrderBy(kv => kv.Key); int index = lastProcessedBuffer; foreach (var item in orderedState) { CompressionDetails c = item.Value; if (c.Sequence == (lastProcessedBuffer + 1)) { await outputStream.WriteAsync(c.ChunkSize, 0, c.ChunkSize.Length); await outputStream.WriteAsync(c.CompressedDataSize, 0, c.CompressedDataSize.Length); await outputStream.WriteAsync(c.Bytes, 0, c.Bytes.Length); lastProcessedBuffer = c.Sequence; c.IsProcessed = true; multiplexState[c.Sequence] = c; } else break; } return v; }
public static async Task Compress(Stream inputStream, Stream outputStream) { var buffer = new BufferBlock<CompressionDetails>(new DataflowBlockOptions { BoundedCapacity = BoundedCapacity }); var compressorOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxDegreeOfParallelism, BoundedCapacity = BoundedCapacity }; var writerOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = BoundedCapacity, SingleProducerConstrained = true }; var compressor = new TransformBlock<CompressionDetails, CompressionDetails>(compressionDetails => Compress(compressionDetails), compressorOptions); var writer = new ActionBlock<CompressionDetails>(compressionDetailsWithSize => Multiplex(outputStream, compressionDetailsWithSize), writerOptions); buffer.LinkTo(compressor); compressor.LinkTo(writer); buffer.Completion.ContinueWith(task => compressor.Complete()); compressor.Completion.ContinueWith(task => writer.Complete()); long sourceLength = inputStream.Length; // Write total size to destination byte[] size = BitConverter.GetBytes(sourceLength); await outputStream.WriteAsync(size, 0, size.Length); long chunkSize = 1048576; // 1 MB int index = 0; while (sourceLength > 0) { byte[] data = new byte[chunkSize]; int readCount = await inputStream.ReadAsync(data, 0, data.Length); byte[] bytes = new byte[readCount]; Buffer.BlockCopy(data, 0, bytes, 0, readCount); CompressionDetails compressionDetails = new CompressionDetails { Bytes = bytes, ChunkSize = BitConverter.GetBytes(chunkSize), Sequence = ++index }; while (await buffer.SendAsync(compressionDetails) != true) { } sourceLength -= chunkSize; if (sourceLength < chunkSize) chunkSize = sourceLength; if (sourceLength == 0) buffer.Complete(); } writer.Completion.Wait(); await outputStream.FlushAsync(); inputStream.Dispose(); outputStream.Dispose(); }
private static async Task<CompressionDetails> Compress(CompressionDetails compressionDetails) { byte[] data = compressionDetails.Bytes; using (MemoryStream compressedDataStream = new MemoryStream()) { using (GZipStream streamCompressed = new GZipStream(compressedDataStream, CompressionMode.Compress)) { // write chunk in the compressed stream await streamCompressed.WriteAsync(data, 0, data.Length); } byte[] compressedData = compressedDataStream.GetBuffer(); byte[] compressedDatasize = BitConverter.GetBytes(compressedData.Length); CompressionDetails v = new CompressionDetails { Bytes = compressedData, ChunkSize = compressionDetails.ChunkSize, Sequence = compressionDetails.Sequence, CompressedDataSize = compressedDatasize }; return v; } }
public static async Task Compress(Stream inputStream, Stream outputStream) { var buffer = new BufferBlock <CompressionDetails>(new DataflowBlockOptions { BoundedCapacity = BoundedCapacity }); var compressorOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxDegreeOfParallelism, BoundedCapacity = BoundedCapacity }; var writerOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = BoundedCapacity, SingleProducerConstrained = true }; var compressor = new TransformBlock <CompressionDetails, CompressionDetails>(compressionDetails => Compress(compressionDetails), compressorOptions); var writer = new ActionBlock <CompressionDetails>(compressionDetailsWithSize => Multiplex(outputStream, compressionDetailsWithSize), writerOptions); buffer.LinkTo(compressor); compressor.LinkTo(writer); buffer.Completion.ContinueWith(task => compressor.Complete()); compressor.Completion.ContinueWith(task => writer.Complete()); long sourceLength = inputStream.Length; // Write total size to destination byte[] size = BitConverter.GetBytes(sourceLength); await outputStream.WriteAsync(size, 0, size.Length); long chunkSize = 1048576; // 1 MB int index = 0; while (sourceLength > 0) { byte[] data = new byte[chunkSize]; int readCount = await inputStream.ReadAsync(data, 0, data.Length); byte[] bytes = new byte[readCount]; Buffer.BlockCopy(data, 0, bytes, 0, readCount); CompressionDetails compressionDetails = new CompressionDetails { Bytes = bytes, ChunkSize = BitConverter.GetBytes(chunkSize), Sequence = ++index }; while (await buffer.SendAsync(compressionDetails) != true) { } sourceLength -= chunkSize; if (sourceLength < chunkSize) { chunkSize = sourceLength; } if (sourceLength == 0) { buffer.Complete(); } } writer.Completion.Wait(); await outputStream.FlushAsync(); inputStream.Dispose(); outputStream.Dispose(); }