private T DeSerializeAndDecompressObjectFromFile <T>(string path, IProgress <StorageManagerProgress> progress) { CodeProgressImplementation coderProgress = null; FileStream inputFileStream = null; MemoryStream output = new MemoryStream(); try { inputFileStream = File.OpenRead(path); if (progress != null) { coderProgress = new CodeProgressImplementation(progress, CodeProgressImplementation.CodingOperations.Decoding); progress.Report(new StorageManagerProgress { ProgressPercentage = 0, Text = "Starting LZMA decoding of file" }); } if (_settings.UseMultithreading && CompressionFileHeader.VerifyFileHeader(inputFileStream)) { DeflateDataMultithreded(inputFileStream, output, progress); } else { DeflateData(inputFileStream, output, inputFileStream.Length, coderProgress).RunSynchronously(); } output.Flush(); output.Position = 0; } catch (Exception ex) { Log.Error(ex, "Error in StorageManager.DeSerializeAndDecompressObjectFromFile()"); } finally { inputFileStream?.Close(); } T deserializedObj; var attrs = Attribute.GetCustomAttributes(typeof(T)); if (attrs.OfType <DataContractAttribute>().Any()) { deserializedObj = Serializer.Deserialize <T>(output); } else { BinaryFormatter binaryFormatter = new BinaryFormatter(); deserializedObj = (T)binaryFormatter.Deserialize(output); } output.Dispose(); GC.Collect(); return(deserializedObj); }
private T DeSerializeAndDecompressObjectFromEncryptedFile <T>(string path, IProgress <StorageManagerProgress> progress) { EncryptionManager encryptionManager = new EncryptionManager(); Stream input = null; MemoryStream output = new MemoryStream(); try { input = encryptionManager.DecryptFileToMemoryStream(path, _settings.GetPassword(), new CryptoProgress(progress)); input.Position = 0; if (_settings.UseMultithreading && CompressionFileHeader.VerifyFileHeader(input)) { DeflateDataMultithreded(input, output, progress); } else { CodeProgressImplementation coderProgress = new CodeProgressImplementation(progress, CodeProgressImplementation.CodingOperations.Decoding); progress?.Report(new StorageManagerProgress { ProgressPercentage = 0, Text = "Starting LZMA multithreaded decoding of file" }); DeflateData(input, output, input.Length, coderProgress).RunSynchronously(); } output.Flush(); output.Position = 0; } catch (Exception ex) { Log.Error(ex, "Error in StorageManager.DeSerializeAndDecompressObjectFromEncryptedFile()"); } finally { input?.Close(); } Attribute[] attrs = Attribute.GetCustomAttributes(typeof(T)); bool protoBufferCompatible = attrs.OfType <DataContractAttribute>().Any(); progress?.Report(protoBufferCompatible ? new StorageManagerProgress { ProgressPercentage = 0, Text = "Deserializing using Protobuffer" } : new StorageManagerProgress { ProgressPercentage = 0, Text = "Deserializing using BinaryFormatter" }); if (protoBufferCompatible) { return(Serializer.Deserialize <T>(output)); } BinaryFormatter binaryFormatter = new BinaryFormatter(); return((T)binaryFormatter.Deserialize(output)); }
private MemoryStream SerializeAndCompressObjectToMemoryStream(object obj, IProgress <StorageManagerProgress> progress) { MemoryStream msInput = new MemoryStream(); MemoryStream msOutput = new MemoryStream(); var attrs = Attribute.GetCustomAttributes(obj.GetType()); bool protoBufferCompatible = attrs.OfType <DataContractAttribute>().Any(); progress?.Report(protoBufferCompatible ? new StorageManagerProgress { ProgressPercentage = 0, Text = "Serializing using Protobuffer" } : new StorageManagerProgress { ProgressPercentage = 0, Text = "Serializing using BinaryFormatter" }); if (protoBufferCompatible) { Serializer.NonGeneric.Serialize(msInput, obj); } else { BinaryFormatter binaryFormatter = new BinaryFormatter(); binaryFormatter.Serialize(msInput, obj); } CodeProgressImplementation coderProgress = null; if (progress != null) { coderProgress = new CodeProgressImplementation(progress, CodeProgressImplementation.CodingOperations.Encoding, msInput.Length); progress.Report(new StorageManagerProgress { ProgressPercentage = 0, Text = "Starting LZMA encoding of file" }); } msInput.Position = 0; if (_settings.UseMultithreading) { CompressDataMultithreaded(msInput, msOutput, coderProgress); } else { CompressData(msInput, msOutput, msInput.Length, coderProgress); } GC.Collect(); return(msOutput); }
private void DeflateDataMultithreded(Stream inputDataStream, Stream outputStream, IProgress <StorageManagerProgress> progress) { CodeProgressImplementation coderProgress = null; if (!CompressionFileHeader.VerifyFileHeader(inputDataStream)) { throw new Exception("Invalid file header"); } if (progress != null) { coderProgress = new CodeProgressImplementation(progress, CodeProgressImplementation.CodingOperations.Decoding); progress.Report(new StorageManagerProgress { ProgressPercentage = 0, Text = "Starting LZMA multithreaded decoding of file" }); } CompressionFileHeader compressionFileHeader = CompressionFileHeader.DecodeHeader(inputDataStream); inputDataStream.Position = compressionFileHeader.FileHeaderSize; int currentBlock = 0; var decoderTasks = new Task[_settings.NumberOfThreads]; var outputMemoryStreams = new MemoryStream[_settings.NumberOfThreads]; while (currentBlock < compressionFileHeader.NumberOfBlocks) { int taskCount = 0; for (int i = 0; i < _settings.NumberOfThreads; i++) { CompressionBlock dataBlock = compressionFileHeader.CompressedDataBlocks[currentBlock]; outputMemoryStreams[i] = new MemoryStream(); var buffer = new byte[dataBlock.CompressedBlockSize]; inputDataStream.Read(buffer, 0, buffer.Length); MemoryStream inputStream = new MemoryStream(buffer) { Position = 0 }; decoderTasks[i] = DeflateData(inputStream, outputMemoryStreams[i], dataBlock.CompressedBlockSize, coderProgress); decoderTasks[i].Start(); currentBlock++; taskCount++; progress?.Report(new StorageManagerProgress { ProgressPercentage = currentBlock / compressionFileHeader.NumberOfBlocks, Text = "Decoding block " + currentBlock }); if (currentBlock == compressionFileHeader.NumberOfBlocks) { break; } } Task.WaitAll(decoderTasks.Take(taskCount).ToArray()); for (int i = 0; i < taskCount; i++) { var buffer = outputMemoryStreams[i].ToArray(); outputStream.Write(buffer, 0, buffer.Length); } } }
private void CompressDataMultithreaded(Stream input, Stream output, CodeProgressImplementation coderProgress) { //Write file header CompressionFileHeader compressionFileHeader = new CompressionFileHeader(input.Length, BlockSize); int sizeOfHeader = compressionFileHeader.FileHeaderSize; output.Position = sizeOfHeader; long totalEncodeSize = input.Length; long bytesLeft = totalEncodeSize; var tasks = new Task[_settings.NumberOfThreads]; var outMemoryStreams = new MemoryStream[_settings.NumberOfThreads]; var inputBlockSizeArray = new int[_settings.NumberOfThreads]; input.Position = 0; while (bytesLeft > 0) { int taskCount = 0; for (int i = 0; i < tasks.Length; i++) { int encodeSize = Math.Min(BlockSize, (int)bytesLeft); if (encodeSize <= 0) { break; } taskCount++; var buffer = new byte[encodeSize]; int bytesRead = input.Read(buffer, 0, buffer.Length); bytesLeft -= bytesRead; if (bytesRead == 0) { break; } MemoryStream inputStream = new MemoryStream(buffer); MemoryStream outStream = new MemoryStream(); outMemoryStreams[i] = outStream; inputBlockSizeArray[i] = bytesRead; tasks[i] = new Task(() => { CompressData(inputStream, outStream, bytesRead, null); }); tasks[i].Start(); } var activeTasks = tasks.Take(taskCount).ToArray(); Task.WaitAll(activeTasks); coderProgress?.SetProgress(totalEncodeSize - bytesLeft, -1); for (int i = 0; i < taskCount; i++) { CompressionBlock compressionBlock = new CompressionBlock(); var outBytes = outMemoryStreams[i].ToArray(); outMemoryStreams[i] = null; compressionBlock.CompressedBlockSize = outBytes.Length; compressionBlock.UncompressedBlockSize = inputBlockSizeArray[i]; compressionBlock.StartPosition = output.Position; compressionBlock.EndPosition = output.Position + outBytes.Length; output.Write(outBytes, 0, outBytes.Length); compressionFileHeader.CompressedDataBlocks.Add(compressionBlock); } } // Write file header output.Position = 0; var headerBytes = compressionFileHeader.ToBytes(); output.Write(headerBytes, 0, headerBytes.Length); }
private bool SerializeAndCompressObjectToFile(object obj, string path, IProgress <StorageManagerProgress> progress) { CodeProgressImplementation coderProgress = null; Stream output = null; try { // Use Truncate to avoid https://stackoverflow.com/questions/2152978/using-protobuf-net-i-suddenly-got-an-exception-about-an-unknown-wire-type output = new FileStream(path, FileMode.Truncate); MemoryStream input = new MemoryStream(); var attrs = Attribute.GetCustomAttributes(obj.GetType()); bool protoBufferCompatible = attrs.OfType <DataContractAttribute>().Any(); progress?.Report(protoBufferCompatible ? new StorageManagerProgress { ProgressPercentage = 0, Text = "Serializing using Protobuffer" } : new StorageManagerProgress { ProgressPercentage = 0, Text = "Serializing using BinaryFormatter" }); if (protoBufferCompatible) { Serializer.NonGeneric.Serialize(input, obj); } else { BinaryFormatter binaryFormatter = new BinaryFormatter(); binaryFormatter.Serialize(input, obj); } input.Position = 0; // Encode the file. if (progress != null) { coderProgress = new CodeProgressImplementation(progress, CodeProgressImplementation.CodingOperations.Encoding, input.Length); progress.Report(new StorageManagerProgress { ProgressPercentage = 0, Text = "Starting LZMA encoding of file" }); } if (_settings.UseMultithreading) { CompressDataMultithreaded(input, output, coderProgress); } else { CompressData(input, output, input.Length, coderProgress); } output.Flush(); } catch (Exception ex) { Log.Error(ex, "Error in StorageManager.SerializeAndCompressObjectToFile()"); return(false); } finally { output?.Close(); } return(true); }