private void CheckMemoryStatus(InputWorkItem inputWorkItem)
{
OutputStream outputStream = inputWorkItem.OutputStream;
DecompressorSettings decompressorSettings = (DecompressorSettings)inputWorkItem.Settings;
if (outputStream.Stream is FileStream)
{
return;
}
if (MemoryStatus.Current.MemoryLoad < 80)
{
return;
}
lock (_lockObject)
{
if (MemoryStatus.Current.MemoryLoad > decompressorSettings.MemoryLoadThreshold)
{
TempFileStream tempFileStream = new TempFileStream();
outputStream.Stream.Position = 0;
outputStream.Stream.CopyTo(tempFileStream);
outputStream.Stream = tempFileStream;
}
}
}
public void Process(InputWorkItem workItem, CancellationToken cancellationToken)
{
using (GZipStream compressionStream = new GZipStream(workItem.OutputStream.Stream, CompressionMode.Compress, true))
{
ProcessInputStream(workItem, compressionStream, cancellationToken);
}
workItem.OutputQueue.Add(new OutputWorkItem(workItem.OutputStream));
}
protected virtual void ProcessInputStream(InputWorkItem workItem, Stream compressionStream, CancellationToken cancellationToken)
{
foreach (IStreamChunk inputStreamChunk in workItem.InputStream.Chunks)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
ProcessInputStreamChunk(workItem, inputStreamChunk, compressionStream);
}
}
public void Process(InputWorkItem workItem, CancellationToken cancellationToken)
{
using (InputStream inputStream = workItem.InputStream)
using (GZipStream decompressionStream = new GZipStream(inputStream.Stream, CompressionMode.Decompress, false))
{
decompressionStream.CopyTo(workItem.OutputStream.Stream);
}
CheckMemoryStatus(workItem);
workItem.OutputQueue.Add(new OutputWorkItem(workItem.OutputStream));
}
protected virtual void ProcessInputStreamChunk(InputWorkItem workItem, IStreamChunk inputStreamChunk, Stream compressionStream)
{
using (Stream chunkStream = new BufferedStream(inputStreamChunk.Stream))
{
byte[] buffer = new byte[inputStreamChunk.Length];
int readedBytes = inputStreamChunk.Stream.Read(buffer, 0, (int)inputStreamChunk.Length);
if (readedBytes < inputStreamChunk.Length)
{
throw new CompressionException("Cannot read input stream");
}
compressionStream.Write(buffer, 0, inputStreamChunk.Length);
}
}
private InputWorkItem GetWorkItem(InputQueue inputQueue)
{
InputWorkItem workItem = null;
lock (inputQueue.SyncRoot)
{
if (inputQueue.Count > 0)
{
workItem = inputQueue[0];
inputQueue.RemoveAt(0);
}
}
return(workItem);
}
public void Work(object parameter)
{
ProcessorWorkerParameter processorWorkerParameter = (ProcessorWorkerParameter)parameter;
InputQueue inputQueue = processorWorkerParameter.InputQueue;
while (true)
{
if (processorWorkerParameter.CancellationToken.IsCancellationRequested)
{
break;
}
InputWorkItem workItem = GetWorkItem(inputQueue);
if (workItem == null)
{
break;
}
_processor.Process(workItem, processorWorkerParameter.CancellationToken);
}
_event.Set();
}
public InputQueue Create(string inputFilePath, string outputFilePath, IEnumerable <InputStream> inputStreams, OutputQueue outputQueue)
{
if (inputStreams == null)
{
throw new ArgumentNullException("Input streams must be non-empty");
}
if (outputQueue == null)
{
throw new ArgumentNullException("Output queue must be non-empty");
}
if (string.IsNullOrEmpty(inputFilePath))
{
throw new ArgumentException("Input file path must be non-empty");
}
if (string.IsNullOrEmpty(outputFilePath))
{
throw new ArgumentException("Output file path must be non-empty");
}
InputQueue inputQueue = new InputQueue();
if (inputStreams.Count() == 1)
{
Stream rawOutputStream = File.Create(outputFilePath);
OutputStream outputStream = new OutputStream(0, rawOutputStream);
InputWorkItem workItem = new InputWorkItem(
inputStreams.First(),
outputStream,
outputQueue,
_compressionSettings
);
inputQueue.Add(workItem);
}
else
{
FileInfo inputFileInfo = new FileInfo(inputFilePath);
if (!inputFileInfo.Exists)
{
throw new ArgumentException("Input file does not exist");
}
long availableMemoryStreamsCount = CalculateAvailableMemoryStreamsCount(inputStreams, inputFileInfo);
int index = 0;
foreach (InputStream inputStream in inputStreams)
{
Stream rawOutputStream =
index < availableMemoryStreamsCount
? (Stream)(new MemoryStream())
: (Stream)(new TempFileStream());
OutputStream outputStream = new OutputStream(index++, rawOutputStream);
InputWorkItem workItem = new InputWorkItem(
inputStream,
outputStream,
outputQueue,
_compressionSettings
);
inputQueue.Add(workItem);
}
}
return(inputQueue);
}