private Result <int, ErrorCodes?> MultyThreadedDecompression(Stream inputStream, Stream outputStream)
{
using (var writerFinished = new ManualResetEvent(false))
using (var workersPool = new WorkersPool.WorkersPool(settings.ThreadsCount, log))
using (var tasksQueue = new BlockingTasksQueue(settings.CompressingQueueSize, workersPool, log))
{
var outputFile = new OutputFileWithoutOffsetStore(outputStream);
var blockId = 0;
var writeDecompressedTask =
RunDequeueingOutputWriteDecompressedBlocks(writerFinished, outputFile, tasksQueue);
workersPool.PushTask(writeDecompressedTask);
var needWaitTasksFinished = false;
var needEndTasks = true;
StreamingGzipBlock streamingGzipBlock = null;
try
{
foreach (var block in gzipBlockSplitter.SplitBlocks(inputStream))
{
if (block is StreamingGzipBlock gzipBlock)
{
tasksQueue.EndTasks();
needEndTasks = false;
if (needWaitTasksFinished)
{
writerFinished.WaitOne();
needWaitTasksFinished = false;
}
streamingGzipBlock = gzipBlock;
break;
}
tasksQueue.EnqueueTask(
new DelegateTask(
blockId++.ToString(),
() => ((IndependentGzipBlock)block).Decompress())
);
needWaitTasksFinished = true;
}
}
finally
{
if (needEndTasks)
{
tasksQueue.EndTasks();
}
}
if (needWaitTasksFinished)
{
writerFinished.WaitOne();
}
streamingGzipBlock?.WriteDecompressedDataTo(outputStream);
return(outputFile.CompressionRatio(inputStream.Length));
}
}
public Result <int, ErrorCodes?> Compress([NotNull] Stream inputStream, [NotNull] Stream outputStream)
{
if (inputStream == null)
{
throw new ArgumentNullException(nameof(inputStream));
}
if (outputStream == null)
{
throw new ArgumentNullException(nameof(outputStream));
}
if (inputStream.Position == inputStream.Length)
{
return(ErrorCodes.NothingToCompress);
}
using (var workersPool = new WorkersPool.WorkersPool(settings.ThreadsCount, log))
using (var tasksQueue = new BlockingTasksQueue(settings.CompressingQueueSize, workersPool, log))
{
var outputFile = new OutputFile(outputStream, settings.OffsetLabel);
var(lastReadOffset, offsetError) = outputFile.GetLastOffset();
if (offsetError.HasValue)
{
return(offsetError);
}
var compressInputFileTask = RunEnqueueingInputFileBlockCompression(inputStream, lastReadOffset, tasksQueue);
workersPool.PushTask(compressInputFileTask);
foreach (var result in tasksQueue.ConsumeTaskResults())
{
if (result.IsFailed)
{
if (result.Exception is CompressorException fileSystemException)
{
return(fileSystemException.Error);
}
throw result.Exception;
}
var resultBlock = (Block)result.Result;
outputFile.Append(resultBlock.Bytes, resultBlock.Offset);
}
var commitError = outputFile.Commit();
if (commitError.HasValue)
{
return(commitError);
}
return(outputFile.CompressionRatio(inputStream.Length));
}
}
public void Should_execute_all_tasks_after_many_iterations(int tasksCount, int workersCount)
{
using (var newWorkersPool = new WorkersPool.WorkersPool(workersCount, new TestLog(output)))
{
var expectedResults = Enumerable.Range(0, tasksCount).ToList();
var actualResults = new int[tasksCount];
var startSignals = Enumerable.Range(0, tasksCount).Select(_ => new CountdownEvent(1)).ToArray();
var stopwatch = new Stopwatch();
using (new Disposables(startSignals))
using (var finishSignal = new CountdownEvent(tasksCount))
{
var workerTasks = Enumerable.Range(0, tasksCount)
.Select(
id => WorkerTasks.Task(
"task " + id,
startSignal: startSignals[id],
finishSignal: finishSignal,
action: () => actualResults[id] = id
)
)
.ToList();
stopwatch.Start();
for (var i = 0; i < workerTasks.Count; i++)
{
newWorkersPool.PushTask(workerTasks[i]);
startSignals[i].Wait();
}
finishSignal.Wait(5.Seconds());
stopwatch.Stop();
}
actualResults.Should().BeEquivalentTo(expectedResults);
output.WriteLine("Task finished for " + stopwatch.Elapsed);
}
}
public PushTask(ITestOutputHelper output)
{
workersPool = new WorkersPool.WorkersPool(3, new TestLog(output));
this.output = output;
}
public TaskInProgress(ITestOutputHelper output)
{
this.output = output;
workersPool = new WorkersPool.WorkersPool(3, new TestLog(output));
}
