public void Decompress(string fileNameIn, string fileNameOut) { Console.WriteLine("Decompress"); var inStream = new FileStream(fileNameIn, FileMode.Open); var outStream = new FileStream(fileNameOut, FileMode.Create); var processingUnits = new List <DataProcessingUnit>(); while (true) { var dataSize = 0; var dataSizeBuffer = new byte[4]; if (inStream.Read(dataSizeBuffer, 0, dataSizeBuffer.Length) != 0) { dataSize = BitConverter.ToInt32(dataSizeBuffer, 0); Console.WriteLine($"DataSize: {dataSize}"); } if (dataSize == 0) { break; } var inBuffer = new byte[dataSize]; if (inStream.Read(inBuffer, 0, inBuffer.Length) == 0) { break; } var processingUnit = new DataProcessingUnit(); var waitHandle = new EventWaitHandle(false, EventResetMode.ManualReset); var thread = new Thread(() => { var decompressedData = DecompressBytes(inBuffer); processingUnit.Data = decompressedData; processingUnit.WaitHandle.Set(); }); thread.Start(); processingUnit.Thread = thread; processingUnit.WaitHandle = waitHandle; processingUnits.Add(processingUnit); if (processingUnits.Count == ThreadNumber) { FlushCompressedData(processingUnits, outStream); } } if (processingUnits.Any()) { FlushCompressedData(processingUnits, outStream); } inStream.Close(); outStream.Close(); }
public static int DataChunkSize = 1024 * 1024 * 32;//Environment.SystemPageSize; public void Compress(string fileNameIn, string fileNameOut) { Console.WriteLine("Compress"); var inStream = new FileStream(fileNameIn, FileMode.Open); var outStream = new FileStream(fileNameOut, FileMode.Create); var processingUnits = new List <DataProcessingUnit>(); while (true) { var inBuffer = new byte[DataChunkSize]; var bytesReadCount = inStream.Read(inBuffer, 0, inBuffer.Length); if (bytesReadCount == 0) { break; } if (bytesReadCount < DataChunkSize) { inBuffer = inBuffer.Skip(0).Take(bytesReadCount).ToArray(); } var processingUnit = new DataProcessingUnit(); var waitHandle = new EventWaitHandle(false, EventResetMode.ManualReset); var thread = new Thread(() => { var compressedBytes = CompressBytes(inBuffer).ToList(); var dataSize = BitConverter.GetBytes(compressedBytes.Count); compressedBytes.InsertRange(0, dataSize); processingUnit.Data = compressedBytes.ToArray(); processingUnit.WaitHandle.Set(); Console.WriteLine($"Thread input:{inBuffer.Length/1024/1024} output:{compressedBytes.Count/1024/1024} size:{compressedBytes.Count}"); }); thread.Start(); processingUnit.Thread = thread; processingUnit.WaitHandle = waitHandle; processingUnits.Add(processingUnit); if (processingUnits.Count == ThreadNumber) { FlushCompressedData(processingUnits, outStream); } } if (processingUnits.Any()) { FlushCompressedData(processingUnits, outStream); } inStream.Close(); outStream.Close(); }