Ejemplo n.º 1
0
        public static async Task Compress(Stream inputStream, Stream outputStream)
        {
            var buffer = new BufferBlock<CompressionDetails>(new DataflowBlockOptions { BoundedCapacity = BoundedCapacity });
            var compressorOptions = new ExecutionDataflowBlockOptions
            {
                MaxDegreeOfParallelism = MaxDegreeOfParallelism,
                BoundedCapacity = BoundedCapacity
            };

            var writerOptions = new ExecutionDataflowBlockOptions
            {
                BoundedCapacity = BoundedCapacity,
                SingleProducerConstrained = true
            };

            var compressor = new TransformBlock<CompressionDetails, CompressionDetails>(compressionDetails => Compress(compressionDetails), compressorOptions);
            var writer = new ActionBlock<CompressionDetails>(compressionDetailsWithSize => Multiplex(outputStream, compressionDetailsWithSize), writerOptions);

            buffer.LinkTo(compressor);
            compressor.LinkTo(writer);
            
            buffer.Completion.ContinueWith(task => compressor.Complete()); 
            compressor.Completion.ContinueWith(task => writer.Complete());

            long sourceLength = inputStream.Length;
            // Write total size to destination
            byte[] size = BitConverter.GetBytes(sourceLength);
            await outputStream.WriteAsync(size, 0, size.Length);

            long chunkSize = 1048576; // 1 MB
            int index = 0;
            while (sourceLength > 0)
            {
                byte[] data = new byte[chunkSize];
                int readCount = await inputStream.ReadAsync(data, 0, data.Length);

                byte[] bytes = new byte[readCount];
                Buffer.BlockCopy(data, 0, bytes, 0, readCount);

                CompressionDetails compressionDetails = new CompressionDetails
                {
                    Bytes = bytes,
                    ChunkSize = BitConverter.GetBytes(chunkSize),
                    Sequence = ++index
                };

                while (await buffer.SendAsync(compressionDetails) != true) { }

                sourceLength -= chunkSize;
                if (sourceLength < chunkSize)
                    chunkSize = sourceLength;

                if (sourceLength == 0)
                    buffer.Complete();
            }
            writer.Completion.Wait();

            await outputStream.FlushAsync();
            inputStream.Dispose();
            outputStream.Dispose();
        }
Ejemplo n.º 2
0
        private static async Task<CompressionDetails> Multiplex(Stream outputStream, CompressionDetails compressionDetailsWithSize)
        {
            CompressionDetails v = compressionDetailsWithSize;
            multiplexState.AddOrUpdate(v.Sequence, v, (i, s) => s);

            var orderedState = multiplexState.Where(k => !k.Value.IsProcessed).OrderBy(kv => kv.Key);

            int index = lastProcessedBuffer;
            foreach (var item in orderedState)
            {
                CompressionDetails c = item.Value;
                if (c.Sequence == (lastProcessedBuffer + 1))
                {
                    await outputStream.WriteAsync(c.ChunkSize, 0, c.ChunkSize.Length);
                    await outputStream.WriteAsync(c.CompressedDataSize, 0, c.CompressedDataSize.Length);
                    await outputStream.WriteAsync(c.Bytes, 0, c.Bytes.Length);

                    lastProcessedBuffer = c.Sequence;
                    c.IsProcessed = true;
                    multiplexState[c.Sequence] = c;
                }
                else
                    break;
            }
            return v;
        }
Ejemplo n.º 3
0
        private static async Task<CompressionDetails> Compress(CompressionDetails compressionDetails)
        {
            byte[] data = compressionDetails.Bytes;
            using (MemoryStream compressedDataStream = new MemoryStream())
            {
                using (GZipStream streamCompressed = new GZipStream(compressedDataStream, CompressionMode.Compress))
                {
                    // write chunk in the compressed stream
                    await streamCompressed.WriteAsync(data, 0, data.Length);
                }
                byte[] compressedData = compressedDataStream.GetBuffer();
                byte[] compressedDatasize = BitConverter.GetBytes(compressedData.Length);

                CompressionDetails v = new CompressionDetails
                {
                    Bytes = compressedData,
                    ChunkSize = compressionDetails.ChunkSize,
                    Sequence = compressionDetails.Sequence,
                    CompressedDataSize = compressedDatasize
                };
                return v;
            }
        }