private CompressionBlock Compress(CompressionBlock block)
{
using (MemoryStream ms = new MemoryStream(_blockLength))
{
using (GZipStream gz = new GZipStream(ms, CompressionMode.Compress)) gz.Write(block.Data, 0, block.Data.Length);
block.Data = ms.ToArray();
}
return(block);
}
private CompressionBlock Decompress(CompressionBlock block)
{
byte[] buffer = new byte[_blockLength];
using (MemoryStream ms = new MemoryStream(block.Data))
{
using (GZipStream gz = new GZipStream(ms, CompressionMode.Decompress)) block.Data = buffer.Take(0, gz.Read(buffer, 0, buffer.Length));
}
return(block);
}
private void DeflateDataMultithreded(Stream inputDataStream, Stream outputStream, IProgress <StorageManagerProgress> progress)
{
CodeProgressImplementation coderProgress = null;
if (!CompressionFileHeader.VerifyFileHeader(inputDataStream))
{
throw new Exception("Invalid file header");
}
if (progress != null)
{
coderProgress = new CodeProgressImplementation(progress, CodeProgressImplementation.CodingOperations.Decoding);
progress.Report(new StorageManagerProgress {
ProgressPercentage = 0, Text = "Starting LZMA multithreaded decoding of file"
});
}
CompressionFileHeader compressionFileHeader = CompressionFileHeader.DecodeHeader(inputDataStream);
inputDataStream.Position = compressionFileHeader.FileHeaderSize;
int currentBlock = 0;
var decoderTasks = new Task[_settings.NumberOfThreads];
var outputMemoryStreams = new MemoryStream[_settings.NumberOfThreads];
while (currentBlock < compressionFileHeader.NumberOfBlocks)
{
int taskCount = 0;
for (int i = 0; i < _settings.NumberOfThreads; i++)
{
CompressionBlock dataBlock = compressionFileHeader.CompressedDataBlocks[currentBlock];
outputMemoryStreams[i] = new MemoryStream();
var buffer = new byte[dataBlock.CompressedBlockSize];
inputDataStream.Read(buffer, 0, buffer.Length);
MemoryStream inputStream = new MemoryStream(buffer)
{
Position = 0
};
decoderTasks[i] = DeflateData(inputStream, outputMemoryStreams[i], dataBlock.CompressedBlockSize, coderProgress);
decoderTasks[i].Start();
currentBlock++;
taskCount++;
progress?.Report(new StorageManagerProgress {
ProgressPercentage = currentBlock / compressionFileHeader.NumberOfBlocks, Text = "Decoding block " + currentBlock
});
if (currentBlock == compressionFileHeader.NumberOfBlocks)
{
break;
}
}
Task.WaitAll(decoderTasks.Take(taskCount).ToArray());
for (int i = 0; i < taskCount; i++)
{
var buffer = outputMemoryStreams[i].ToArray();
outputStream.Write(buffer, 0, buffer.Length);
}
}
}
private void CompressDataMultithreaded(Stream input, Stream output, CodeProgressImplementation coderProgress)
{
//Write file header
CompressionFileHeader compressionFileHeader = new CompressionFileHeader(input.Length, BlockSize);
int sizeOfHeader = compressionFileHeader.FileHeaderSize;
output.Position = sizeOfHeader;
long totalEncodeSize = input.Length;
long bytesLeft = totalEncodeSize;
var tasks = new Task[_settings.NumberOfThreads];
var outMemoryStreams = new MemoryStream[_settings.NumberOfThreads];
var inputBlockSizeArray = new int[_settings.NumberOfThreads];
input.Position = 0;
while (bytesLeft > 0)
{
int taskCount = 0;
for (int i = 0; i < tasks.Length; i++)
{
int encodeSize = Math.Min(BlockSize, (int)bytesLeft);
if (encodeSize <= 0)
{
break;
}
taskCount++;
var buffer = new byte[encodeSize];
int bytesRead = input.Read(buffer, 0, buffer.Length);
bytesLeft -= bytesRead;
if (bytesRead == 0)
{
break;
}
MemoryStream inputStream = new MemoryStream(buffer);
MemoryStream outStream = new MemoryStream();
outMemoryStreams[i] = outStream;
inputBlockSizeArray[i] = bytesRead;
tasks[i] = new Task(() => { CompressData(inputStream, outStream, bytesRead, null); });
tasks[i].Start();
}
var activeTasks = tasks.Take(taskCount).ToArray();
Task.WaitAll(activeTasks);
coderProgress?.SetProgress(totalEncodeSize - bytesLeft, -1);
for (int i = 0; i < taskCount; i++)
{
CompressionBlock compressionBlock = new CompressionBlock();
var outBytes = outMemoryStreams[i].ToArray();
outMemoryStreams[i] = null;
compressionBlock.CompressedBlockSize = outBytes.Length;
compressionBlock.UncompressedBlockSize = inputBlockSizeArray[i];
compressionBlock.StartPosition = output.Position;
compressionBlock.EndPosition = output.Position + outBytes.Length;
output.Write(outBytes, 0, outBytes.Length);
compressionFileHeader.CompressedDataBlocks.Add(compressionBlock);
}
}
// Write file header
output.Position = 0;
var headerBytes = compressionFileHeader.ToBytes();
output.Write(headerBytes, 0, headerBytes.Length);
}