protected override void RunWriter( IEnumerable <ProcessorWorker> processorWorkers, IWriter writer, UserTask writerTask, string outputFilePath, OutputQueue outputQueue, Action <TaskStatus> callback) { UserTask newWriterTask = writerTask; if (_settings.CreateMultiStream && outputQueue.Capacity > 1) { newWriterTask = (parameter) => { List <EventWaitHandle> waitHandles = new List <EventWaitHandle>(); foreach (ProcessorWorker processorWorker in processorWorkers) { waitHandles.Add(processorWorker.Event); } EventWaitHandle.WaitAll(waitHandles.ToArray()); writerTask.Invoke(parameter); }; } base.RunWriter(processorWorkers, writer, newWriterTask, outputFilePath, outputQueue, callback); }
protected virtual void RunWriter( IEnumerable <ProcessorWorker> processorWorkers, IWriter writer, UserTask writerTask, string outputFilePath, OutputQueue outputQueue, Action <TaskStatus> callback) { ThreadPool.QueueUserTask( writerTask, null, callback ); }
public InputWorkItem(InputStream inputStream, OutputStream outputStream, OutputQueue outputQueue, CompressionSettings settings) { if (inputStream == null) { throw new ArgumentNullException("Input stream must be non-empty"); } if (outputStream == null) { throw new ArgumentNullException("Output stream must be non-empty"); } if (outputQueue == null) { throw new ArgumentNullException("Output queue must be non-empty"); } if (settings == null) { throw new ArgumentNullException("Settings must be non-empty"); } _inputStream = inputStream; _outputStream = outputStream; _outputQueue = outputQueue; _settings = settings; }
public void Write(string outputFilePath, OutputQueue outputQueue, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(outputFilePath)) { throw new ArgumentException("Output file path must be non-empty"); } if (outputQueue == null) { throw new ArgumentNullException("Output queue must be non-empty"); } GZipMultiStreamHeader multiStreamHeader = new GZipMultiStreamHeader(); for (int i = 0; i < outputQueue.Count; i++) { if (cancellationToken.IsCancellationRequested) { break; } OutputWorkItem workItem = outputQueue[i]; MultiStreamHeaderItem multiStreamHeaderItem = new MultiStreamHeaderItem { Length = workItem.OutputStream.Stream.Length }; multiStreamHeader.Items.Add(multiStreamHeaderItem); } OutputWorkItem firstOutputWorkItem = outputQueue[0]; IBlockReader blockReader = new BlockReader(); firstOutputWorkItem.OutputStream.Stream.Position = 0; Block block = blockReader.Read(firstOutputWorkItem.OutputStream.Stream, BlockFlags.All); if (cancellationToken.IsCancellationRequested) { return; } block.ExtraField = multiStreamHeader.Serialize(); multiStreamHeader.Items[0].Length = block.Length; block.ExtraField = multiStreamHeader.Serialize(); block.Flags |= GZipFlags.FEXTRA; IGZipBlockWriter blockWriter = new BlockWriter(); using (FileStream outputFileStream = File.Create(outputFilePath)) { blockWriter.Write(outputFileStream, block, BlockFlags.All); for (int i = 1; i < outputQueue.Count; i++) { if (cancellationToken.IsCancellationRequested) { break; } OutputWorkItem workItem = outputQueue[i]; using (Stream compressedStream = workItem.OutputStream.Stream) { compressedStream.Position = 0; compressedStream.CopyTo(outputFileStream); } } } }
protected void Process(string inputFilePath, string outputFilePath, Action <CompressionCompletedEventArgs> callback, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(inputFilePath)) { throw new ArgumentException("Input file must be non-empty string"); } if (!File.Exists(inputFilePath)) { throw new ArgumentException("Input file must exist"); } if (string.IsNullOrEmpty(outputFilePath)) { throw new ArgumentException("Output file must be non-empty string"); } if (cancellationToken.IsCancellationRequested) { callback(new CompressionCompletedEventArgs(null, cancellationToken.IsCancellationRequested)); return; } List <Exception> errors = new List <Exception>(); IEnumerable <InputStream> inputStreams = GetSettings().Reader.Read(inputFilePath, cancellationToken); OutputQueue outputQueue = new OutputQueue(inputStreams.Count()); InputQueue inputQueue = GetSettings().InputQueueFactory.Create(inputFilePath, outputFilePath, inputStreams, outputQueue); List <ProcessorWorker> processorWorkers = new List <ProcessorWorker>(); IWriter writer = CreateWriter(inputQueue); for (int i = 0; i < inputQueue.Count; i++) { ProcessorWorker processorWorker = new ProcessorWorker(GetSettings().ProcessorFactory.Create()); processorWorkers.Add(processorWorker); } RunProcessors( processorWorkers, inputQueue, (taskStatus) => { if (taskStatus.Error != null) { errors.Add(taskStatus.Error); } }, cancellationToken ); RunWriter( processorWorkers, writer, (parameter) => { writer.Write(outputFilePath, outputQueue, cancellationToken); }, outputFilePath, outputQueue, (taskStatus) => { if (taskStatus.Error != null) { errors.Add(taskStatus.Error); } Exception error = null; if (errors.Count > 0) { error = new AggregateException(errors); } callback(new CompressionCompletedEventArgs(error, cancellationToken.IsCancellationRequested)); } ); }
public InputQueue Create(string inputFilePath, string outputFilePath, IEnumerable <InputStream> inputStreams, OutputQueue outputQueue) { if (inputStreams == null) { throw new ArgumentNullException("Input streams must be non-empty"); } if (outputQueue == null) { throw new ArgumentNullException("Output queue must be non-empty"); } if (string.IsNullOrEmpty(inputFilePath)) { throw new ArgumentException("Input file path must be non-empty"); } if (string.IsNullOrEmpty(outputFilePath)) { throw new ArgumentException("Output file path must be non-empty"); } InputQueue inputQueue = new InputQueue(); if (inputStreams.Count() == 1) { Stream rawOutputStream = File.Create(outputFilePath); OutputStream outputStream = new OutputStream(0, rawOutputStream); InputWorkItem workItem = new InputWorkItem( inputStreams.First(), outputStream, outputQueue, _compressionSettings ); inputQueue.Add(workItem); } else { FileInfo inputFileInfo = new FileInfo(inputFilePath); if (!inputFileInfo.Exists) { throw new ArgumentException("Input file does not exist"); } long availableMemoryStreamsCount = CalculateAvailableMemoryStreamsCount(inputStreams, inputFileInfo); int index = 0; foreach (InputStream inputStream in inputStreams) { Stream rawOutputStream = index < availableMemoryStreamsCount ? (Stream)(new MemoryStream()) : (Stream)(new TempFileStream()); OutputStream outputStream = new OutputStream(index++, rawOutputStream); InputWorkItem workItem = new InputWorkItem( inputStream, outputStream, outputQueue, _compressionSettings ); inputQueue.Add(workItem); } } return(inputQueue); }
public void Write(string outputFilePath, OutputQueue outputQueue, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(outputFilePath)) { throw new ArgumentException("Output file path must be non-empty"); } if (outputQueue == null) { throw new ArgumentNullException("Output queue must be non-empty"); } Stream outputFileStream = null; try { int index = 0; while (true) { if (cancellationToken.IsCancellationRequested) { break; } if (!outputQueue.Contains(index)) { outputQueue.Event.WaitOne(); } else { OutputWorkItem workItem = outputQueue[index]; if (outputQueue.Capacity == 1) { if (!(workItem.OutputStream.Stream is FileStream)) { throw new CompressionException("Output stream must be file stream"); } workItem.OutputStream.Stream.Close(); break; } if (outputFileStream == null) { outputFileStream = File.Create(outputFilePath); } using (Stream compressedStream = workItem.OutputStream.Stream) { compressedStream.Position = 0; compressedStream.CopyTo(outputFileStream); } outputQueue.Remove(index); index++; if (index == outputQueue.Capacity) { break; } } } } finally { if (outputFileStream != null) { outputFileStream.Dispose(); } } }