protected override void Write(CustomConcurentQueue outputBlocks, string outputFilePath, object i)
{
int lastBlockIndex = 0;
doneEvents[(int)i] = new ManualResetEvent(false);
using (var writer = new BinaryWriter(File.Open(outputFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None)))
{
while (outputBlocks.Count > 0 || !_isProcessed)
{
if (outputBlocks.Count == 0)
{
continue;
}
writer.BaseStream.Seek(0, SeekOrigin.End);
var block = outputBlocks.GetByIndex(lastBlockIndex);
if (block.Equals(default(KeyValuePair <int, byte[]>)))
{
continue;
}
writer.Write(block.Value);
Interlocked.Increment(ref lastBlockIndex);
}
}
doneEvents[(int)i].Set();
}
protected override void Write(CustomConcurentQueue outputBlocks, string outputFilePath, object i)
{
int lastBlockIndex = 0;
doneEvents[(int)i] = new ManualResetEvent(false);
using (FileStream _fileCompressed = new FileStream(outputFilePath + ".gz", FileMode.Append))
{
while (!_isProcessed || outputBlocks.Count > 0)
{
if (outputBlocks.Count == 0)
{
continue;
}
var block = outputBlocks.GetByIndex(lastBlockIndex);
if (block.Equals(default(KeyValuePair <int, byte[]>)))
{
continue;
}
BitConverter.GetBytes(block.Value.Length).CopyTo(block.Value, 4);
_fileCompressed.Write(block.Value, 0, block.Value.Length);
Interlocked.Increment(ref lastBlockIndex);
}
}
doneEvents[(int)i].Set();
}
protected override void Read(CustomConcurentQueue inputBlocks, string inputFilePath)
{
int blockIndex = 0;
using (FileStream _compressedFile = new FileStream(inputFilePath, FileMode.Open))
{
while (_compressedFile.Position < _compressedFile.Length)
{
if (inputBlocks.Count > _safeElementsCounts)
{
continue;
}
byte[] lengthBuffer = new byte[8];
_compressedFile.Read(lengthBuffer, 0, lengthBuffer.Length);
int blockLength = BitConverter.ToInt32(lengthBuffer, 4);
byte[] compressedData = new byte[blockLength];
lengthBuffer.CopyTo(compressedData, 0);
_compressedFile.Read(compressedData, 8, blockLength - 8);
inputBlocks.Add(blockIndex, compressedData);
blockIndex++;
}
}
_isReadFinished = true;
}
protected GZipArchiver(string inputFilePath, string outputFilePath)
{
if (int.TryParse(ConfigurationManager.AppSettings["bufferSize"], out int blockSize))
{
_blockSize = blockSize;
}
else
{
_blockSize = DEFAULT_BLOCK_SIZE;
}
if (int.TryParse(ConfigurationManager.AppSettings["safeElementsCount"], out int safeElementsCount))
{
_safeElementsCounts = safeElementsCount;
}
else
{
_blockSize = DEFAULT_SAFE_ELEMENTCOUNT_SIZE;
}
_inputFilePath = inputFilePath;
_outputFilePath = outputFilePath;
_threadCount = Environment.ProcessorCount - 1; // one for read action
_inputBlocks = new CustomConcurentQueue();
_outputBlocks = new CustomConcurentQueue();
doneEvents = new ManualResetEvent[_threadCount];
}
protected override void Read(CustomConcurentQueue inputBlocks, string inputFilePath)
{
using (FileStream _fileForCompressing = new FileStream(inputFilePath, FileMode.Open))
{
int bytesRead;
byte[] lastBuffer;
int blockIndex = 0;
while (_fileForCompressing.Position < _fileForCompressing.Length)
{
if (inputBlocks.Count > _safeElementsCounts)
{
continue;
}
if (_fileForCompressing.Length - _fileForCompressing.Position <= _blockSize)
{
bytesRead = (int)(_fileForCompressing.Length - _fileForCompressing.Position);
}
else
{
bytesRead = _blockSize;
}
lastBuffer = new byte[bytesRead];
_fileForCompressing.Read(lastBuffer, 0, bytesRead);
inputBlocks.Add(blockIndex, lastBuffer);
blockIndex++;
}
}
_isReadFinished = true;
}
protected override void ProcessBlock(CustomConcurentQueue outputBlocks, KeyValuePair <int, byte[]> block)
{
using (MemoryStream _memoryStream = new MemoryStream())
{
using (GZipStream cs = new GZipStream(_memoryStream, CompressionMode.Compress))
{
cs.Write(block.Value, 0, block.Value.Length);
}
byte[] compressedData = _memoryStream.ToArray();
outputBlocks.Add(block.Key, compressedData);
}
}
protected override void ProcessBlock(CustomConcurentQueue outputBlocks, KeyValuePair <int, byte[]> block)
{
using (MemoryStream ms = new MemoryStream(block.Value))
{
using (GZipStream _gz = new GZipStream(ms, CompressionMode.Decompress))
{
// The last 4 bytes of the gz file contains the length
byte[] originalLengthByteArray = new byte[4];
Array.Copy(block.Value, block.Value.Length - 4, originalLengthByteArray, 0, 4);
int length = BitConverter.ToInt32(originalLengthByteArray, 0);
byte[] outByte = new byte[length];
_gz.Read(outByte, 0, outByte.Length);
byte[] decompressedData = outByte.ToArray();
outputBlocks.Add(block.Key, decompressedData);
}
}
}
protected abstract void Write(CustomConcurentQueue outputBlocks, string outputFilePath, object i);
protected abstract void Read(CustomConcurentQueue inputBlocks, string inputFilePath);
protected abstract void ProcessBlock(CustomConcurentQueue outputBlocks, KeyValuePair <int, byte[]> block);
