public static async Task <ChunkBytes> Compress(ChunkBytes data)
{
using (MemoryStream memStream = new MemoryStream())
{
using (GZipStream gzipStream = new GZipStream(memStream, CompressionLevel.Optimal, leaveOpen: true))
await gzipStream.WriteAsync(data.Bytes, 0, data.Length);
var chunkBytes = new ChunkBytes(memStream.ToArray());
return(chunkBytes);
}
}
public static async Task <ChunkBytes> Decompress(ChunkBytes data, int offset = 0)
{
using (MemoryStream memStream = new MemoryStream())
{
using (MemoryStream input = new MemoryStream(data.Bytes, offset, data.Length - offset))
using (GZipStream gzipStream = new GZipStream(input, CompressionMode.Decompress))
await gzipStream.CopyToAsync(memStream);
var chunkBytes = new ChunkBytes(memStream.ToArray());
return(chunkBytes);
}
}
private static async Task <ChunkBytes> CryptoTransform(ChunkBytes data, ICryptoTransform transform)
{
using (var memStream = new MemoryStream())
{
using (var cryptoStream = new CryptoStream(memStream, transform, CryptoStreamMode.Write))
{
await cryptoStream.WriteAsync(data.Bytes, 0, data.Length);
cryptoStream.FlushFinalBlock();
var chunkBytes = new ChunkBytes(memStream.ToArray());
return(chunkBytes);
}
}
}
public async Task CompressAndEncrypt(
Stream streamSource, Stream streamDestination,
CancellationTokenSource cts = null)
{
cts = cts ?? new CancellationTokenSource();
var compressorOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
BoundedCapacity = _boundedCapacity,
CancellationToken = cts.Token
};
var inputBuffer = new BufferBlock <CompressingDetails>(
new DataflowBlockOptions
{
CancellationToken = cts.Token,
BoundedCapacity = _boundedCapacity
});
var compressor = new TransformBlock <CompressingDetails, CompressedDetails>(
async details =>
{
ChunkBytes compressedData = await IOUtils.Compress(details.Bytes);
return(new CompressedDetails
{
Bytes = compressedData,
ChunkSize = details.ChunkSize,
Sequence = details.Sequence,
CompressedDataSize = new ChunkBytes(BitConverter.GetBytes(compressedData.Length))
});
}, compressorOptions);
var encryptor = new TransformBlock <CompressedDetails, EncryptDetails>(
async details =>
{
var data = IOUtils.CombineByteArrays(details.CompressedDataSize, details.ChunkSize,
details.Bytes);
var encryptedData = await IOUtils.Encrypt(data);
return(new EncryptDetails
{
Bytes = encryptedData,
Sequence = details.Sequence,
EncryptedDataSize = new ChunkBytes(BitConverter.GetBytes(encryptedData.Length))
});
}, compressorOptions);
// Enables Rx integration with TPL Dataflow
encryptor.AsObservable()
.Scan((new Dictionary <int, EncryptDetails>(), 0),
(state, msg) =>
// Runs Rx Scan operation asynchronously
Observable.FromAsync(async() =>
{
(Dictionary <int, EncryptDetails> details, int lastIndexProc) = state;
details.Add(msg.Sequence, msg);
while (details.ContainsKey(lastIndexProc + 1))
{
msg = details[lastIndexProc + 1];
// Persists asynchronously the data; the stream could be replaced
// with a network stream to send the data across the wire.
await streamDestination.WriteAsync(msg.EncryptedDataSize.Bytes, 0,
msg.EncryptedDataSize.Length);
await streamDestination.WriteAsync(msg.Bytes.Bytes, 0, msg.Bytes.Length);
lastIndexProc = msg.Sequence;
// the chunk of data that is processed is removed
// from the local state, keeping track of the items to perform
details.Remove(lastIndexProc);
}
return(details, lastIndexProc);
})
.SingleAsync().Wait())
// Rx subscribes to TaskPoolScheduler
.SubscribeOn(TaskPoolScheduler.Default).Subscribe();
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
inputBuffer.LinkTo(compressor, linkOptions);
compressor.LinkTo(encryptor, linkOptions);
long sourceLength = streamSource.Length;
byte[] size = BitConverter.GetBytes(sourceLength);
await streamDestination.WriteAsync(size, 0, size.Length);
var chunkSize = (_chunkSize < sourceLength) ? _chunkSize : (int)sourceLength;
int indexSequence = 0;
while (sourceLength > 0)
{
var bytes = await IOUtils.ReadFromStream(streamSource, chunkSize);
var compressingDetails = new CompressingDetails
{
Bytes = bytes,
ChunkSize = new ChunkBytes(BitConverter.GetBytes(bytes.Length)),
Sequence = ++indexSequence
};
await inputBuffer.SendAsync(compressingDetails);
sourceLength -= bytes.Length;
if (sourceLength < chunkSize)
{
chunkSize = (int)sourceLength;
}
if (sourceLength == 0)
{
inputBuffer.Complete();
}
}
await inputBuffer.Completion.ContinueWith(task => compressor.Complete());
await compressor.Completion.ContinueWith(task => encryptor.Complete());
await encryptor.Completion;
await streamDestination.FlushAsync();
}
public static Task <ChunkBytes> Decrypt(ChunkBytes data) => CryptoTransform(data, RijndaelAlgorithm.Value.CreateDecryptor());
public static string MD5FromBytes(ChunkBytes data)
{
using (var md5 = MD5.Create())
using (var stream = new MemoryStream(data.Bytes, 0, data.Length))
return(BitConverter.ToString(md5.ComputeHash(stream)).Replace("-", "").ToLower());
}