protected override void RunWriter(
IEnumerable <ProcessorWorker> processorWorkers,
IWriter writer,
UserTask writerTask,
string outputFilePath,
OutputQueue outputQueue,
Action <TaskStatus> callback)
{
UserTask newWriterTask = writerTask;
if (_settings.CreateMultiStream && outputQueue.Capacity > 1)
{
newWriterTask =
(parameter) =>
{
List <EventWaitHandle> waitHandles = new List <EventWaitHandle>();
foreach (ProcessorWorker processorWorker in processorWorkers)
{
waitHandles.Add(processorWorker.Event);
}
EventWaitHandle.WaitAll(waitHandles.ToArray());
writerTask.Invoke(parameter);
};
}
base.RunWriter(processorWorkers, writer, newWriterTask, outputFilePath, outputQueue, callback);
}
protected virtual void RunWriter(
IEnumerable <ProcessorWorker> processorWorkers,
IWriter writer,
UserTask writerTask,
string outputFilePath,
OutputQueue outputQueue,
Action <TaskStatus> callback)
{
ThreadPool.QueueUserTask(
writerTask,
null,
callback
);
}
public InputWorkItem(InputStream inputStream, OutputStream outputStream, OutputQueue outputQueue, CompressionSettings settings)
{
if (inputStream == null)
{
throw new ArgumentNullException("Input stream must be non-empty");
}
if (outputStream == null)
{
throw new ArgumentNullException("Output stream must be non-empty");
}
if (outputQueue == null)
{
throw new ArgumentNullException("Output queue must be non-empty");
}
if (settings == null)
{
throw new ArgumentNullException("Settings must be non-empty");
}
_inputStream = inputStream;
_outputStream = outputStream;
_outputQueue = outputQueue;
_settings = settings;
}
public void Write(string outputFilePath, OutputQueue outputQueue, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(outputFilePath))
{
throw new ArgumentException("Output file path must be non-empty");
}
if (outputQueue == null)
{
throw new ArgumentNullException("Output queue must be non-empty");
}
GZipMultiStreamHeader multiStreamHeader = new GZipMultiStreamHeader();
for (int i = 0; i < outputQueue.Count; i++)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
OutputWorkItem workItem = outputQueue[i];
MultiStreamHeaderItem multiStreamHeaderItem = new MultiStreamHeaderItem
{
Length = workItem.OutputStream.Stream.Length
};
multiStreamHeader.Items.Add(multiStreamHeaderItem);
}
OutputWorkItem firstOutputWorkItem = outputQueue[0];
IBlockReader blockReader = new BlockReader();
firstOutputWorkItem.OutputStream.Stream.Position = 0;
Block block = blockReader.Read(firstOutputWorkItem.OutputStream.Stream, BlockFlags.All);
if (cancellationToken.IsCancellationRequested)
{
return;
}
block.ExtraField = multiStreamHeader.Serialize();
multiStreamHeader.Items[0].Length = block.Length;
block.ExtraField = multiStreamHeader.Serialize();
block.Flags |= GZipFlags.FEXTRA;
IGZipBlockWriter blockWriter = new BlockWriter();
using (FileStream outputFileStream = File.Create(outputFilePath))
{
blockWriter.Write(outputFileStream, block, BlockFlags.All);
for (int i = 1; i < outputQueue.Count; i++)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
OutputWorkItem workItem = outputQueue[i];
using (Stream compressedStream = workItem.OutputStream.Stream)
{
compressedStream.Position = 0;
compressedStream.CopyTo(outputFileStream);
}
}
}
}
protected void Process(string inputFilePath, string outputFilePath, Action <CompressionCompletedEventArgs> callback, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(inputFilePath))
{
throw new ArgumentException("Input file must be non-empty string");
}
if (!File.Exists(inputFilePath))
{
throw new ArgumentException("Input file must exist");
}
if (string.IsNullOrEmpty(outputFilePath))
{
throw new ArgumentException("Output file must be non-empty string");
}
if (cancellationToken.IsCancellationRequested)
{
callback(new CompressionCompletedEventArgs(null, cancellationToken.IsCancellationRequested));
return;
}
List <Exception> errors = new List <Exception>();
IEnumerable <InputStream> inputStreams = GetSettings().Reader.Read(inputFilePath, cancellationToken);
OutputQueue outputQueue = new OutputQueue(inputStreams.Count());
InputQueue inputQueue = GetSettings().InputQueueFactory.Create(inputFilePath, outputFilePath, inputStreams, outputQueue);
List <ProcessorWorker> processorWorkers = new List <ProcessorWorker>();
IWriter writer = CreateWriter(inputQueue);
for (int i = 0; i < inputQueue.Count; i++)
{
ProcessorWorker processorWorker = new ProcessorWorker(GetSettings().ProcessorFactory.Create());
processorWorkers.Add(processorWorker);
}
RunProcessors(
processorWorkers,
inputQueue,
(taskStatus) =>
{
if (taskStatus.Error != null)
{
errors.Add(taskStatus.Error);
}
},
cancellationToken
);
RunWriter(
processorWorkers,
writer,
(parameter) =>
{
writer.Write(outputFilePath, outputQueue, cancellationToken);
},
outputFilePath,
outputQueue,
(taskStatus) =>
{
if (taskStatus.Error != null)
{
errors.Add(taskStatus.Error);
}
Exception error = null;
if (errors.Count > 0)
{
error = new AggregateException(errors);
}
callback(new CompressionCompletedEventArgs(error, cancellationToken.IsCancellationRequested));
}
);
}
public InputQueue Create(string inputFilePath, string outputFilePath, IEnumerable <InputStream> inputStreams, OutputQueue outputQueue)
{
if (inputStreams == null)
{
throw new ArgumentNullException("Input streams must be non-empty");
}
if (outputQueue == null)
{
throw new ArgumentNullException("Output queue must be non-empty");
}
if (string.IsNullOrEmpty(inputFilePath))
{
throw new ArgumentException("Input file path must be non-empty");
}
if (string.IsNullOrEmpty(outputFilePath))
{
throw new ArgumentException("Output file path must be non-empty");
}
InputQueue inputQueue = new InputQueue();
if (inputStreams.Count() == 1)
{
Stream rawOutputStream = File.Create(outputFilePath);
OutputStream outputStream = new OutputStream(0, rawOutputStream);
InputWorkItem workItem = new InputWorkItem(
inputStreams.First(),
outputStream,
outputQueue,
_compressionSettings
);
inputQueue.Add(workItem);
}
else
{
FileInfo inputFileInfo = new FileInfo(inputFilePath);
if (!inputFileInfo.Exists)
{
throw new ArgumentException("Input file does not exist");
}
long availableMemoryStreamsCount = CalculateAvailableMemoryStreamsCount(inputStreams, inputFileInfo);
int index = 0;
foreach (InputStream inputStream in inputStreams)
{
Stream rawOutputStream =
index < availableMemoryStreamsCount
? (Stream)(new MemoryStream())
: (Stream)(new TempFileStream());
OutputStream outputStream = new OutputStream(index++, rawOutputStream);
InputWorkItem workItem = new InputWorkItem(
inputStream,
outputStream,
outputQueue,
_compressionSettings
);
inputQueue.Add(workItem);
}
}
return(inputQueue);
}
public void Write(string outputFilePath, OutputQueue outputQueue, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(outputFilePath))
{
throw new ArgumentException("Output file path must be non-empty");
}
if (outputQueue == null)
{
throw new ArgumentNullException("Output queue must be non-empty");
}
Stream outputFileStream = null;
try
{
int index = 0;
while (true)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (!outputQueue.Contains(index))
{
outputQueue.Event.WaitOne();
}
else
{
OutputWorkItem workItem = outputQueue[index];
if (outputQueue.Capacity == 1)
{
if (!(workItem.OutputStream.Stream is FileStream))
{
throw new CompressionException("Output stream must be file stream");
}
workItem.OutputStream.Stream.Close();
break;
}
if (outputFileStream == null)
{
outputFileStream = File.Create(outputFilePath);
}
using (Stream compressedStream = workItem.OutputStream.Stream)
{
compressedStream.Position = 0;
compressedStream.CopyTo(outputFileStream);
}
outputQueue.Remove(index);
index++;
if (index == outputQueue.Capacity)
{
break;
}
}
}
}
finally
{
if (outputFileStream != null)
{
outputFileStream.Dispose();
}
}
}