private void Write(IBlockingCollection <Chunk> chunks)
{
while (chunks.TryTake(out var chunk))
{
Interlocked.Increment(ref writtenChunks);
var viewStream = memoryMappedFile.CreateViewStream(
chunk.InitialOffset,
chunk.Content.Length,
MemoryMappedFileAccess.ReadWrite
);
chunk.Content.CopyToAsync(viewStream, buffersPool, OnComplete);
}
isDone = true;
void OnComplete(Stream source, Stream target)
{
source.Dispose();
target.Dispose();
Interlocked.Decrement(ref writtenChunks);
if (isDone && writtenChunks == 0)
{
manualResetEvent.Set();
}
}
}
public GZipProcessManager(string pathFromName, string pathToName, GZipOperation operation)
{
this.pathFrom = pathFromName;
this.pathTo = pathToName;
this.allWorkers = new List <IWorker>();
this.producerQueue = new BlockingQueue();
if (operation == GZipOperation.Compress)
{
this.consumerCollection = new BlockingQueue();
this.blockGZipper = new BlockGZipCompressor();
var fileReader = new SimpleFileFactory(pathFrom, BLOCK_SIZE).GetFileReader();
this.blocksProducer = new BlocksProducer(fileReader,
this.producerQueue);
this.blocksConsumer = new CompressBlocksConsumer(
new CompressedFileFactory(pathTo, fileReader.NumberOfBlocks).GetFileWriter(),
this.consumerCollection);
}
else
{
this.blockGZipper = new BlockGZipDecompressor();
this.consumerCollection = new BlockingDictionary();
this.blocksProducer = new BlocksProducer(
new CompressedFileFactory(pathFrom).GetFileReader(),
this.producerQueue);
this.blocksConsumer = new DecompressBlocksConsumer(
new SimpleFileFactory(pathTo).GetFileWriter(),
this.consumerCollection);
}
this.End = new AutoResetEvent(false);
}
public DecompressFileReader(string fileName, long fileHeaderSize, IThreadPool threadPool, int concurrency)
{
this.fileName = fileName;
this.fileHeaderSize = fileHeaderSize;
bag = new DisposableBlockingBag <Stream>(concurrency);
worker = new Worker(threadPool);
}
public CompressFileReader(string fileName, int batchSize, IThreadPool threadPool, int concurrency)
{
worker = new Worker(threadPool);
this.fileName = fileName;
this.batchSize = batchSize;
producingBag = new DisposableBlockingBag <Chunk>(concurrency);
memoryMappedFile = MemoryMappedFile.CreateFromFile(fileName, FileMode.Open, null);
}
public GZipper(BlockingQueue inputQueue, IBlockingCollection outputQueue,
IBlockGZipper blockGZipper, int numOfThreads)
{
this.inputQueue = inputQueue;
this.outputQueue = outputQueue;
this.blockGZipper = blockGZipper;
this.numOfThreads = numOfThreads;
}
private void Write(IBlockingCollection <Stream> producingBag)
{
using var fileStream = File.Open(fileName, FileMode.Open, FileAccess.Write);
fileStream.Position += fileHeaderSize;
while (producingBag.TryTake(out var stream))
{
stream.CopyTo(fileStream);
}
}
public Processor(
IProducer <TIn> producer,
IConsumer <TOut> consumer,
IThreadPool threadPool,
Func <TIn, TOut> mapper,
int transformConcurrency)
{
this.producer = producer;
this.consumer = consumer;
this.transformConcurrency = transformConcurrency;
this.threadPool = threadPool;
this.mapper = mapper;
consumingBag = new DisposableBlockingBag <TOut>(transformConcurrency);
transformTasks = new List <ITask>();
}
private void Start()
{
producingBag = producer.StartProducing();
consumer.StartConsuming(consumingBag);
for (var i = 0; i < transformConcurrency; i++)
{
var task = new Task(() =>
{
while (producingBag.TryTake(out var chunk))
{
consumingBag.Add(mapper(chunk));
}
});
transformTasks.Add(task);
threadPool.RunTask(task);
}
}
public void StartConsuming(IBlockingCollection <Chunk> consumingBag) => worker.Run(() => Write(consumingBag));
public BlocksConsumer(IFileWriter fileWriter, IBlockingCollection dataQueue)
{
this.fileWriter = fileWriter;
this.dataCollection = dataQueue;
}
public DecompressBlocksConsumer(IFileWriter fileWriter, IBlockingCollection dataQueue) :
base(fileWriter, dataQueue) { }