/// <summary>
/// Kick start of a zip/unzip process
/// </summary>
/// <param name="reader"></param>
/// <param name="readerThread"></param>
/// <param name="workers"></param>
/// <param name="buffers"></param>
static void StartConveyor(ArchReader reader, ThreadWorker readerThread, ThreadWorker[] workers, InOutBuffer[] buffers)
{
Debug.Assert(workers.Length == buffers.Length);
for (int idx = 0; idx < workers.Length; idx++)
{
ThreadWorker worker = workers[idx];
General.Log($"{idx} start reading");
reader.Buffer = buffers[idx];
readerThread.haveWork.Set();
readerThread.notifyDone.WaitOne();
General.Log($"{idx} reading chunk complete, start zippin");
worker.haveWork.Set();
worker.Start();
}
}
/// <summary>
/// Kick start of a zip/unzip process
/// </summary>
/// <param name="reader"></param>
/// <param name="workers"></param>
/// <param name="workersTasks"></param>
/// <param name="buffers"></param>
async static Task StartConveyor(ArchReader reader, ArchWorker[] workers, Task[] workerTasks, InOutBuffer[] buffers)
{
Debug.Assert(workers.Length == buffers.Length);
for (int idx = 0; idx < workers.Length; idx++)
{
ArchWorker worker = workers[idx];
General.Log($"{idx} start reading");
reader.Buffer = buffers[idx];
//readerThread.haveWork.Set();
//readerThread.notifyDone.WaitOne();
await reader.ReadAsync();
General.Log($"{idx} reading chunk complete, start zippin");
workerTasks[idx] = worker.DoWorkAsync();
}
}
async private static Task ManageThreadsAsync(string inputName, string outputName, ArchAction act, int bufferSize, uint workersCount = 0)
{
InOutBuffer[] buffers = null;
Task[] workerTasks = null;
try
{
General.Log($"Started '{inputName}' {act} into '{outputName}'...");
if (bufferSize < defaultChunkSize)
{
bufferSize = defaultChunkSize;
}
if (workersCount < 1)
{
workersCount = ChooseWorkersCount();
}
General.Log($"buffer size {bufferSize}, workers count {workersCount}");
//to feel good let's use 'using'
// objects for input reading
using FileStream inFileStream = File.OpenRead(inputName);
ArchReader reader = new ArchReader(inFileStream, act);
// objects for output writing
using FileStream outFileStream = File.Create(outputName);
ArchWriter writer = new ArchWriter(outFileStream, act);
// create objects to do zipping/unzipping
CreateBuffersAndWorkers(act, bufferSize, workersCount, out buffers, out ArchWorker[] workers, out workerTasks);
await StartConveyor(reader, workers, workerTasks, buffers);
int idx = 0;
// as long as there are bytes read from input, keep conveyor
// when no more input data, set this flag & wait till all therads are finihed.
bool finishing = false;
do
{
ArchWorker worker = workers[idx];
if (worker == null)
{
General.Log($"all workers are finished");
break;
}
General.Log($"Wait for worker {idx} done, to start writting");
await workerTasks[idx];//wait till zipper idx has done
General.Log($"{idx} start writting");
writer.Buffer = buffers[idx];
Task writerTask = writer.WriteAsync();
if (!finishing)
{
General.Log($"{idx} start reading next portion");
reader.Buffer = buffers[idx];
Task readerTask = reader.ReadAsync();
General.Log($"{idx} need both In & Out buffers to be Ready to start next part zip/unzip");
await readerTask;
// check if have read anyting
if (reader.Buffer.BytesRead <= 0)
{
//have read all, and all is (being) processed
finishing = true;
}
}
await writerTask;
if (finishing)
{
workers[idx] = null;
workerTasks[idx] = null; //according to info from ms , no need to dispose a Task
buffers[idx] = null;
}
else
{
workerTasks[idx] = worker.DoWorkAsync();
}
//cause rotating, and output shall be in-order
if (++idx >= workers.Length)
{
idx = 0;
}
} while (true);
General.Log($"{act} finished successfuly.");
}
catch (Exception exc)
{
General.Log($"ManageThreads encountered an error {exc.Message} returning to caller.");
throw;
}
finally
{
CleanUp(buffers, workerTasks);
}
}
private static void ManageThreads(string inputName, string outputName, ArchAction act, int bufferSize, uint workersCount = 0)
{
InOutBuffer[] buffers = null;
ThreadWorker[] workers = null;
try
{
General.Log($"Started '{inputName}' {act} into '{outputName}'...");
if (bufferSize < defaultChunkSize)
{
bufferSize = defaultChunkSize;
}
if (workersCount < 1)
{
workersCount = ChooseWorkersCount();
}
General.Log($"buffer size {bufferSize}, workers count {workersCount}");
//to feel good let's use 'using'
// objects for input reading
using FileStream inFileStream = File.OpenRead(inputName);
ArchReader reader = new ArchReader(inFileStream, act);
using ThreadWorker readerThread = new ThreadWorker(reader, true);
// objects for output writing
using FileStream outFileStream = File.Create(outputName);
ArchWriter writer = new ArchWriter(outFileStream, act);
using ThreadWorker writerThread = new ThreadWorker(writer, true);
// create objects to do zipping/unzipping
CreateBuffersAndWorkers(act, bufferSize, workersCount, out buffers, out workers);
StartConveyor(reader, readerThread, workers, buffers);
int idx = 0;
// as long as there are bytes read from input, keep conveyor
// when no more input data, set this flag & wait till all therads are finihed.
bool finishing = false;
do
{
ThreadWorker worker = workers[idx];
if (worker == null)
{
General.Log($"all threads are finished");
// good place to let the writer go
writerThread.Finish();
//writerThread.Dispose();
break;
}
General.Log($"main thread about to Wait for worker {idx} done, to start writting");
worker.notifyDone.WaitOne(); //wait till zipper idx has done
General.Log($"{idx} start writting");
writer.Buffer = buffers[idx];
writerThread.haveWork.Set();
if (!finishing)
{
General.Log($"{idx} start reading next portion");
reader.Buffer = buffers[idx];
readerThread.haveWork.Set();
General.Log($"{idx} need both In & Out buffers to be Ready to start next part zippin");
readerThread.notifyDone.WaitOne();
// check if have read anyting
if (reader.Buffer.BytesRead <= 0)
{
//have read all, and all is (being) processed
finishing = true;
// right place to set the reader free
readerThread.Finish();
//readerThread.Dispose();
}
}
writerThread.notifyDone.WaitOne();
if (finishing)
{
worker.Finish();
workers[idx].Dispose();
workers[idx] = null;
//can clean buffers here as well
buffers[idx].Dispose();
buffers[idx] = null;
}
else
{
worker.haveWork.Set();
}
//cause rotating, and output shall be in-order
if (++idx >= workers.Length)
{
idx = 0;
}
} while (true);
General.Log($"{act} finished successfuly.");
}
catch (Exception exc)
{
General.Log($"ManageThreads encountered an error {exc.Message} returning to caller.");
throw;
}
finally
{
CleanUp(buffers, workers);
}
}
