private void Write(IBlockingCollection <Chunk> chunks) { while (chunks.TryTake(out var chunk)) { Interlocked.Increment(ref writtenChunks); var viewStream = memoryMappedFile.CreateViewStream( chunk.InitialOffset, chunk.Content.Length, MemoryMappedFileAccess.ReadWrite ); chunk.Content.CopyToAsync(viewStream, buffersPool, OnComplete); } isDone = true; void OnComplete(Stream source, Stream target) { source.Dispose(); target.Dispose(); Interlocked.Decrement(ref writtenChunks); if (isDone && writtenChunks == 0) { manualResetEvent.Set(); } } }
public GZipProcessManager(string pathFromName, string pathToName, GZipOperation operation) { this.pathFrom = pathFromName; this.pathTo = pathToName; this.allWorkers = new List <IWorker>(); this.producerQueue = new BlockingQueue(); if (operation == GZipOperation.Compress) { this.consumerCollection = new BlockingQueue(); this.blockGZipper = new BlockGZipCompressor(); var fileReader = new SimpleFileFactory(pathFrom, BLOCK_SIZE).GetFileReader(); this.blocksProducer = new BlocksProducer(fileReader, this.producerQueue); this.blocksConsumer = new CompressBlocksConsumer( new CompressedFileFactory(pathTo, fileReader.NumberOfBlocks).GetFileWriter(), this.consumerCollection); } else { this.blockGZipper = new BlockGZipDecompressor(); this.consumerCollection = new BlockingDictionary(); this.blocksProducer = new BlocksProducer( new CompressedFileFactory(pathFrom).GetFileReader(), this.producerQueue); this.blocksConsumer = new DecompressBlocksConsumer( new SimpleFileFactory(pathTo).GetFileWriter(), this.consumerCollection); } this.End = new AutoResetEvent(false); }
public DecompressFileReader(string fileName, long fileHeaderSize, IThreadPool threadPool, int concurrency) { this.fileName = fileName; this.fileHeaderSize = fileHeaderSize; bag = new DisposableBlockingBag <Stream>(concurrency); worker = new Worker(threadPool); }
public CompressFileReader(string fileName, int batchSize, IThreadPool threadPool, int concurrency) { worker = new Worker(threadPool); this.fileName = fileName; this.batchSize = batchSize; producingBag = new DisposableBlockingBag <Chunk>(concurrency); memoryMappedFile = MemoryMappedFile.CreateFromFile(fileName, FileMode.Open, null); }
public GZipper(BlockingQueue inputQueue, IBlockingCollection outputQueue, IBlockGZipper blockGZipper, int numOfThreads) { this.inputQueue = inputQueue; this.outputQueue = outputQueue; this.blockGZipper = blockGZipper; this.numOfThreads = numOfThreads; }
private void Write(IBlockingCollection <Stream> producingBag) { using var fileStream = File.Open(fileName, FileMode.Open, FileAccess.Write); fileStream.Position += fileHeaderSize; while (producingBag.TryTake(out var stream)) { stream.CopyTo(fileStream); } }
public Processor( IProducer <TIn> producer, IConsumer <TOut> consumer, IThreadPool threadPool, Func <TIn, TOut> mapper, int transformConcurrency) { this.producer = producer; this.consumer = consumer; this.transformConcurrency = transformConcurrency; this.threadPool = threadPool; this.mapper = mapper; consumingBag = new DisposableBlockingBag <TOut>(transformConcurrency); transformTasks = new List <ITask>(); }
private void Start() { producingBag = producer.StartProducing(); consumer.StartConsuming(consumingBag); for (var i = 0; i < transformConcurrency; i++) { var task = new Task(() => { while (producingBag.TryTake(out var chunk)) { consumingBag.Add(mapper(chunk)); } }); transformTasks.Add(task); threadPool.RunTask(task); } }
public void StartConsuming(IBlockingCollection <Chunk> consumingBag) => worker.Run(() => Write(consumingBag));
public BlocksConsumer(IFileWriter fileWriter, IBlockingCollection dataQueue) { this.fileWriter = fileWriter; this.dataCollection = dataQueue; }
public DecompressBlocksConsumer(IFileWriter fileWriter, IBlockingCollection dataQueue) : base(fileWriter, dataQueue) { }