public void Decompress(string fileNameIn, string fileNameOut)
{
Console.WriteLine("Decompress");
var inStream = new FileStream(fileNameIn, FileMode.Open);
var outStream = new FileStream(fileNameOut, FileMode.Create);
var processingUnits = new List <DataProcessingUnit>();
while (true)
{
var dataSize = 0;
var dataSizeBuffer = new byte[4];
if (inStream.Read(dataSizeBuffer, 0, dataSizeBuffer.Length) != 0)
{
dataSize = BitConverter.ToInt32(dataSizeBuffer, 0);
Console.WriteLine($"DataSize: {dataSize}");
}
if (dataSize == 0)
{
break;
}
var inBuffer = new byte[dataSize];
if (inStream.Read(inBuffer, 0, inBuffer.Length) == 0)
{
break;
}
var processingUnit = new DataProcessingUnit();
var waitHandle = new EventWaitHandle(false, EventResetMode.ManualReset);
var thread = new Thread(() => {
var decompressedData = DecompressBytes(inBuffer);
processingUnit.Data = decompressedData;
processingUnit.WaitHandle.Set();
});
thread.Start();
processingUnit.Thread = thread;
processingUnit.WaitHandle = waitHandle;
processingUnits.Add(processingUnit);
if (processingUnits.Count == ThreadNumber)
{
FlushCompressedData(processingUnits, outStream);
}
}
if (processingUnits.Any())
{
FlushCompressedData(processingUnits, outStream);
}
inStream.Close();
outStream.Close();
}
public static int DataChunkSize = 1024 * 1024 * 32;//Environment.SystemPageSize;
public void Compress(string fileNameIn, string fileNameOut)
{
Console.WriteLine("Compress");
var inStream = new FileStream(fileNameIn, FileMode.Open);
var outStream = new FileStream(fileNameOut, FileMode.Create);
var processingUnits = new List <DataProcessingUnit>();
while (true)
{
var inBuffer = new byte[DataChunkSize];
var bytesReadCount = inStream.Read(inBuffer, 0, inBuffer.Length);
if (bytesReadCount == 0)
{
break;
}
if (bytesReadCount < DataChunkSize)
{
inBuffer = inBuffer.Skip(0).Take(bytesReadCount).ToArray();
}
var processingUnit = new DataProcessingUnit();
var waitHandle = new EventWaitHandle(false, EventResetMode.ManualReset);
var thread = new Thread(() => {
var compressedBytes = CompressBytes(inBuffer).ToList();
var dataSize = BitConverter.GetBytes(compressedBytes.Count);
compressedBytes.InsertRange(0, dataSize);
processingUnit.Data = compressedBytes.ToArray();
processingUnit.WaitHandle.Set();
Console.WriteLine($"Thread input:{inBuffer.Length/1024/1024} output:{compressedBytes.Count/1024/1024} size:{compressedBytes.Count}");
});
thread.Start();
processingUnit.Thread = thread;
processingUnit.WaitHandle = waitHandle;
processingUnits.Add(processingUnit);
if (processingUnits.Count == ThreadNumber)
{
FlushCompressedData(processingUnits, outStream);
}
}
if (processingUnits.Any())
{
FlushCompressedData(processingUnits, outStream);
}
inStream.Close();
outStream.Close();
}
