private T DeSerializeAndDecompressObjectFromFile <T>(string path, IProgress <StorageManagerProgress> progress)
{
CodeProgressImplementation coderProgress = null;
FileStream inputFileStream = null;
MemoryStream output = new MemoryStream();
try
{
inputFileStream = File.OpenRead(path);
if (progress != null)
{
coderProgress = new CodeProgressImplementation(progress, CodeProgressImplementation.CodingOperations.Decoding);
progress.Report(new StorageManagerProgress {
ProgressPercentage = 0, Text = "Starting LZMA decoding of file"
});
}
if (_settings.UseMultithreading && CompressionFileHeader.VerifyFileHeader(inputFileStream))
{
DeflateDataMultithreded(inputFileStream, output, progress);
}
else
{
DeflateData(inputFileStream, output, inputFileStream.Length, coderProgress).RunSynchronously();
}
output.Flush();
output.Position = 0;
}
catch (Exception ex)
{
Log.Error(ex, "Error in StorageManager.DeSerializeAndDecompressObjectFromFile()");
}
finally
{
inputFileStream?.Close();
}
T deserializedObj;
var attrs = Attribute.GetCustomAttributes(typeof(T));
if (attrs.OfType <DataContractAttribute>().Any())
{
deserializedObj = Serializer.Deserialize <T>(output);
}
else
{
BinaryFormatter binaryFormatter = new BinaryFormatter();
deserializedObj = (T)binaryFormatter.Deserialize(output);
}
output.Dispose();
GC.Collect();
return(deserializedObj);
}
private T DeSerializeAndDecompressObjectFromEncryptedFile <T>(string path, IProgress <StorageManagerProgress> progress)
{
EncryptionManager encryptionManager = new EncryptionManager();
Stream input = null;
MemoryStream output = new MemoryStream();
try
{
input = encryptionManager.DecryptFileToMemoryStream(path, _settings.GetPassword(), new CryptoProgress(progress));
input.Position = 0;
if (_settings.UseMultithreading && CompressionFileHeader.VerifyFileHeader(input))
{
DeflateDataMultithreded(input, output, progress);
}
else
{
CodeProgressImplementation coderProgress = new CodeProgressImplementation(progress, CodeProgressImplementation.CodingOperations.Decoding);
progress?.Report(new StorageManagerProgress {
ProgressPercentage = 0, Text = "Starting LZMA multithreaded decoding of file"
});
DeflateData(input, output, input.Length, coderProgress).RunSynchronously();
}
output.Flush();
output.Position = 0;
}
catch (Exception ex)
{
Log.Error(ex, "Error in StorageManager.DeSerializeAndDecompressObjectFromEncryptedFile()");
}
finally
{
input?.Close();
}
Attribute[] attrs = Attribute.GetCustomAttributes(typeof(T));
bool protoBufferCompatible = attrs.OfType <DataContractAttribute>().Any();
progress?.Report(protoBufferCompatible ? new StorageManagerProgress {
ProgressPercentage = 0, Text = "Deserializing using Protobuffer"
} : new StorageManagerProgress {
ProgressPercentage = 0, Text = "Deserializing using BinaryFormatter"
});
if (protoBufferCompatible)
{
return(Serializer.Deserialize <T>(output));
}
BinaryFormatter binaryFormatter = new BinaryFormatter();
return((T)binaryFormatter.Deserialize(output));
}
private void DeflateDataMultithreded(Stream inputDataStream, Stream outputStream, IProgress <StorageManagerProgress> progress)
{
CodeProgressImplementation coderProgress = null;
if (!CompressionFileHeader.VerifyFileHeader(inputDataStream))
{
throw new Exception("Invalid file header");
}
if (progress != null)
{
coderProgress = new CodeProgressImplementation(progress, CodeProgressImplementation.CodingOperations.Decoding);
progress.Report(new StorageManagerProgress {
ProgressPercentage = 0, Text = "Starting LZMA multithreaded decoding of file"
});
}
CompressionFileHeader compressionFileHeader = CompressionFileHeader.DecodeHeader(inputDataStream);
inputDataStream.Position = compressionFileHeader.FileHeaderSize;
int currentBlock = 0;
var decoderTasks = new Task[_settings.NumberOfThreads];
var outputMemoryStreams = new MemoryStream[_settings.NumberOfThreads];
while (currentBlock < compressionFileHeader.NumberOfBlocks)
{
int taskCount = 0;
for (int i = 0; i < _settings.NumberOfThreads; i++)
{
CompressionBlock dataBlock = compressionFileHeader.CompressedDataBlocks[currentBlock];
outputMemoryStreams[i] = new MemoryStream();
var buffer = new byte[dataBlock.CompressedBlockSize];
inputDataStream.Read(buffer, 0, buffer.Length);
MemoryStream inputStream = new MemoryStream(buffer)
{
Position = 0
};
decoderTasks[i] = DeflateData(inputStream, outputMemoryStreams[i], dataBlock.CompressedBlockSize, coderProgress);
decoderTasks[i].Start();
currentBlock++;
taskCount++;
progress?.Report(new StorageManagerProgress {
ProgressPercentage = currentBlock / compressionFileHeader.NumberOfBlocks, Text = "Decoding block " + currentBlock
});
if (currentBlock == compressionFileHeader.NumberOfBlocks)
{
break;
}
}
Task.WaitAll(decoderTasks.Take(taskCount).ToArray());
for (int i = 0; i < taskCount; i++)
{
var buffer = outputMemoryStreams[i].ToArray();
outputStream.Write(buffer, 0, buffer.Length);
}
}
}
private void CompressDataMultithreaded(Stream input, Stream output, CodeProgressImplementation coderProgress)
{
//Write file header
CompressionFileHeader compressionFileHeader = new CompressionFileHeader(input.Length, BlockSize);
int sizeOfHeader = compressionFileHeader.FileHeaderSize;
output.Position = sizeOfHeader;
long totalEncodeSize = input.Length;
long bytesLeft = totalEncodeSize;
var tasks = new Task[_settings.NumberOfThreads];
var outMemoryStreams = new MemoryStream[_settings.NumberOfThreads];
var inputBlockSizeArray = new int[_settings.NumberOfThreads];
input.Position = 0;
while (bytesLeft > 0)
{
int taskCount = 0;
for (int i = 0; i < tasks.Length; i++)
{
int encodeSize = Math.Min(BlockSize, (int)bytesLeft);
if (encodeSize <= 0)
{
break;
}
taskCount++;
var buffer = new byte[encodeSize];
int bytesRead = input.Read(buffer, 0, buffer.Length);
bytesLeft -= bytesRead;
if (bytesRead == 0)
{
break;
}
MemoryStream inputStream = new MemoryStream(buffer);
MemoryStream outStream = new MemoryStream();
outMemoryStreams[i] = outStream;
inputBlockSizeArray[i] = bytesRead;
tasks[i] = new Task(() => { CompressData(inputStream, outStream, bytesRead, null); });
tasks[i].Start();
}
var activeTasks = tasks.Take(taskCount).ToArray();
Task.WaitAll(activeTasks);
coderProgress?.SetProgress(totalEncodeSize - bytesLeft, -1);
for (int i = 0; i < taskCount; i++)
{
CompressionBlock compressionBlock = new CompressionBlock();
var outBytes = outMemoryStreams[i].ToArray();
outMemoryStreams[i] = null;
compressionBlock.CompressedBlockSize = outBytes.Length;
compressionBlock.UncompressedBlockSize = inputBlockSizeArray[i];
compressionBlock.StartPosition = output.Position;
compressionBlock.EndPosition = output.Position + outBytes.Length;
output.Write(outBytes, 0, outBytes.Length);
compressionFileHeader.CompressedDataBlocks.Add(compressionBlock);
}
}
// Write file header
output.Position = 0;
var headerBytes = compressionFileHeader.ToBytes();
output.Write(headerBytes, 0, headerBytes.Length);
}
