//reading data chunks
public override void Start()
{
StartTime = DateTime.Now;
try
{
using (var inFileStream = new FileStream(InFileName, FileMode.Open))
{
InFileStream = inFileStream;
for (ChunkCounter = 0; inFileStream.Length - inFileStream.Position > 0; ChunkCounter++)
{
if (StopRequested)
{
return;
}
//check the percentage of remaining memory to initial
while ((float)CompInfo.AvailablePhysicalMemory / InitialFreeRam < 0.1)
{
//forcing garbage collector cleanup
GC.Collect(2, GCCollectionMode.Forced);
Thread.Sleep(1000);
}
//optimal data read chunk size based on ram and cpu cores
var dataChunkSize = (long)CompInfo.AvailablePhysicalMemory / CoreCount / 1;
//size of chunk for the last chunck
if (inFileStream.Length - inFileStream.Position <= dataChunkSize)
{
dataChunkSize = inFileStream.Length - inFileStream.Position;
}
//size of chunk for small files
if (inFileStream.Length < dataChunkSize)
{
dataChunkSize = inFileStream.Length / CoreCount + 1;
}
//constant size of chunk for huge files
if (dataChunkSize > 256 * 1024 * 1024)
{
dataChunkSize = 256 * 1024 * 1024;
}
var dataChunk = new byte[dataChunkSize];
//read file
inFileStream.Read(dataChunk, 0, (int)dataChunkSize);
lock (Locker)
IncomingChunks.Add(ChunkCounter, dataChunk);
//starting new thread with chunk number as an argument
var chunkNumber = ChunkCounter;
var compressThread = new Thread(() => ProcessChunk(chunkNumber));
compressThread.Start();
}
IncomingFinished = true;
}
}
catch (Exception ex)
{
StopRequested = true;
Console.Write("There is the problem at the compression process:{0}", ex.Message);
}
}
//read compressed data chunks from disk
public override void Start()
{
StartTime = DateTime.Now;
try
{
using (var inFileStream = new FileStream(InFileName, FileMode.Open))
{
InFileStream = inFileStream;
var header = new byte[8];
for (ChunkCounter = 0; inFileStream.Length - inFileStream.Position > 0; ChunkCounter++)
{
if (StopRequested)
{
return;
}
//ram overload - wait
while ((float)CompInfo.AvailablePhysicalMemory / InitialFreeRam < 0.1)
{
//force gc cleanup
GC.Collect(2, GCCollectionMode.Forced);
Thread.Sleep(1000);
}
//extract compressed data chunk size from gzip header
inFileStream.Read(header, 0, 8);
var dataChunkSize = BitConverter.ToInt32(header, 4);
var compressedChunk = new byte[dataChunkSize];
header.CopyTo(compressedChunk, 0);
//get the rest bytes of data block
inFileStream.Read(compressedChunk, 8, dataChunkSize - 9);
lock (Locker)
IncomingChunks.Add(ChunkCounter, compressedChunk);
var chunkNumber = ChunkCounter;
var decompressThread = new Thread(() => ProcessChunk(chunkNumber));
decompressThread.Start();
}
IncomingFinished = true;
}
}
catch (Exception ex)
{
StopRequested = true;
Console.Write("There is tecompression process:{0}", ex.Message);
}
}
void MeshCompleted(object sender, RendererEventArgs <ReadOnlyChunk> e)
{
IncomingChunks.Add(e.Result);
}
void ChunkConverter_MeshGenerated(object sender, RendererEventArgs <ReadOnlyChunk> e)
{
IncomingChunks.Add(e.Result);
}
