/// <summary>
/// Creates all the objects required for the requested operation
/// </summary>
/// <param name="act"></param>
/// <param name="bufferSize"></param>
/// <param name="workersCount"></param>
/// <param name="buffers"></param>
/// <param name="workers"></param>
/// <param name="workersTasks"></param>
static void CreateBuffersAndWorkers(ArchAction act, int bufferSize, uint workersCount, out InOutBuffer[] buffers, out ArchWorker[] workers, out Task[] workerTasks)
{
CreateBuffers(workersCount, bufferSize, out buffers);
CreateArchivers(buffers, act, out workers);
workerTasks = new Task[workersCount];
}
/// <summary>
/// Creates all the objects required for the requested operation
/// </summary>
/// <param name="act"></param>
/// <param name="bufferSize"></param>
/// <param name="workersCount"></param>
/// <param name="buffers"></param>
/// <param name="workers"></param>
static void CreateBuffersAndWorkers(ArchAction act, int bufferSize, uint workersCount, out InOutBuffer[] buffers, out ThreadWorker[] workers)
{
CreateBuffers(workersCount, bufferSize, out buffers);
CreateArchivers(buffers, act, out ArchWorker[] archivers);
CreateThreadWorkers(archivers, out workers);
}
/// <summary>
/// c-tor
/// </summary>
/// <param name="inFileStream"></param>
/// <param name="act">What action is requested - zipping or unzipping</param>
public ArchReader(FileStream inFileStream, ArchAction act)
{
this.inFileStream = inFileStream;
if (act == ArchAction.Zip)
{
Read = ReadNextChunk;
}
else
{
Read = ReadNextZippedChunk;
}
}
/// <summary>
/// c-tor
/// </summary>
/// <param name="outFileStream"></param>
/// <param name="act">What action was performed - zipping or unzipping</param>
public ArchWriter(FileStream outFileStream, ArchAction act)
{
this.outFileStream = outFileStream;
if (act == ArchAction.Zip)
{
Write = WriteZippedChunk;
}
else
{
Write = WriteChunk;
}
}
/// <summary>
/// Create those who do zip/unzip
/// </summary>
/// <param name="buffers"></param>
/// <param name="act">zip or unzip</param>
/// <param name="archivers">out param</param>
/// <returns></returns>
static void CreateArchivers(InOutBuffer[] buffers, ArchAction act, out ArchWorker[] archivers)
{
int count = buffers.Length;
Debug.Assert(count > 0);
General.Log($"Creating {count} (un)zippers");
archivers = new ArchWorker[count];
for (int i = 0; i < archivers.Length; i++)
{
archivers[i] = new ArchWorker(inBuffer: buffers[i], outBuffer: buffers[i], act);
}
}
async private static Task ManageThreadsAsync(string inputName, string outputName, ArchAction act, int bufferSize, uint workersCount = 0)
{
InOutBuffer[] buffers = null;
Task[] workerTasks = null;
try
{
General.Log($"Started '{inputName}' {act} into '{outputName}'...");
if (bufferSize < defaultChunkSize)
{
bufferSize = defaultChunkSize;
}
if (workersCount < 1)
{
workersCount = ChooseWorkersCount();
}
General.Log($"buffer size {bufferSize}, workers count {workersCount}");
//to feel good let's use 'using'
// objects for input reading
using FileStream inFileStream = File.OpenRead(inputName);
ArchReader reader = new ArchReader(inFileStream, act);
// objects for output writing
using FileStream outFileStream = File.Create(outputName);
ArchWriter writer = new ArchWriter(outFileStream, act);
// create objects to do zipping/unzipping
CreateBuffersAndWorkers(act, bufferSize, workersCount, out buffers, out ArchWorker[] workers, out workerTasks);
await StartConveyor(reader, workers, workerTasks, buffers);
int idx = 0;
// as long as there are bytes read from input, keep conveyor
// when no more input data, set this flag & wait till all therads are finihed.
bool finishing = false;
do
{
ArchWorker worker = workers[idx];
if (worker == null)
{
General.Log($"all workers are finished");
break;
}
General.Log($"Wait for worker {idx} done, to start writting");
await workerTasks[idx];//wait till zipper idx has done
General.Log($"{idx} start writting");
writer.Buffer = buffers[idx];
Task writerTask = writer.WriteAsync();
if (!finishing)
{
General.Log($"{idx} start reading next portion");
reader.Buffer = buffers[idx];
Task readerTask = reader.ReadAsync();
General.Log($"{idx} need both In & Out buffers to be Ready to start next part zip/unzip");
await readerTask;
// check if have read anyting
if (reader.Buffer.BytesRead <= 0)
{
//have read all, and all is (being) processed
finishing = true;
}
}
await writerTask;
if (finishing)
{
workers[idx] = null;
workerTasks[idx] = null; //according to info from ms , no need to dispose a Task
buffers[idx] = null;
}
else
{
workerTasks[idx] = worker.DoWorkAsync();
}
//cause rotating, and output shall be in-order
if (++idx >= workers.Length)
{
idx = 0;
}
} while (true);
General.Log($"{act} finished successfuly.");
}
catch (Exception exc)
{
General.Log($"ManageThreads encountered an error {exc.Message} returning to caller.");
throw;
}
finally
{
CleanUp(buffers, workerTasks);
}
}
private static void ManageThreads(string inputName, string outputName, ArchAction act, int bufferSize, uint workersCount = 0)
{
InOutBuffer[] buffers = null;
ThreadWorker[] workers = null;
try
{
General.Log($"Started '{inputName}' {act} into '{outputName}'...");
if (bufferSize < defaultChunkSize)
{
bufferSize = defaultChunkSize;
}
if (workersCount < 1)
{
workersCount = ChooseWorkersCount();
}
General.Log($"buffer size {bufferSize}, workers count {workersCount}");
//to feel good let's use 'using'
// objects for input reading
using FileStream inFileStream = File.OpenRead(inputName);
ArchReader reader = new ArchReader(inFileStream, act);
using ThreadWorker readerThread = new ThreadWorker(reader, true);
// objects for output writing
using FileStream outFileStream = File.Create(outputName);
ArchWriter writer = new ArchWriter(outFileStream, act);
using ThreadWorker writerThread = new ThreadWorker(writer, true);
// create objects to do zipping/unzipping
CreateBuffersAndWorkers(act, bufferSize, workersCount, out buffers, out workers);
StartConveyor(reader, readerThread, workers, buffers);
int idx = 0;
// as long as there are bytes read from input, keep conveyor
// when no more input data, set this flag & wait till all therads are finihed.
bool finishing = false;
do
{
ThreadWorker worker = workers[idx];
if (worker == null)
{
General.Log($"all threads are finished");
// good place to let the writer go
writerThread.Finish();
//writerThread.Dispose();
break;
}
General.Log($"main thread about to Wait for worker {idx} done, to start writting");
worker.notifyDone.WaitOne(); //wait till zipper idx has done
General.Log($"{idx} start writting");
writer.Buffer = buffers[idx];
writerThread.haveWork.Set();
if (!finishing)
{
General.Log($"{idx} start reading next portion");
reader.Buffer = buffers[idx];
readerThread.haveWork.Set();
General.Log($"{idx} need both In & Out buffers to be Ready to start next part zippin");
readerThread.notifyDone.WaitOne();
// check if have read anyting
if (reader.Buffer.BytesRead <= 0)
{
//have read all, and all is (being) processed
finishing = true;
// right place to set the reader free
readerThread.Finish();
//readerThread.Dispose();
}
}
writerThread.notifyDone.WaitOne();
if (finishing)
{
worker.Finish();
workers[idx].Dispose();
workers[idx] = null;
//can clean buffers here as well
buffers[idx].Dispose();
buffers[idx] = null;
}
else
{
worker.haveWork.Set();
}
//cause rotating, and output shall be in-order
if (++idx >= workers.Length)
{
idx = 0;
}
} while (true);
General.Log($"{act} finished successfuly.");
}
catch (Exception exc)
{
General.Log($"ManageThreads encountered an error {exc.Message} returning to caller.");
throw;
}
finally
{
CleanUp(buffers, workers);
}
}
private static void ManageThreads(string inputName, string outputName, ArchAction act, int bufferSize, uint workersCount = 0)
{
General.Log($"{act} started.");
Archiver.act = act;
if (bufferSize > defaultChunkSize)
{
ArchWorker.ChunkSize = bufferSize;
}
using (FileStream outFileStream = File.Create(outputName))
{
ArchWorker.OutFileStream = outFileStream;
using FileStream inFileStream = File.OpenRead(inputName);
if (workersCount < 1)
{
workersCount = ChooseWorkersCount();
}
ArchWorker[] archivers = CreateWorkers(workersCount);
int idx = 0;
do
{
ArchWorker arch = archivers[idx];
General.Log($"main thread about to Wait for arch {idx} done, to start it in new thread again");
arch.Done.WaitOne();
//Load the next portion to be processed and written
int read = act == ArchAction.Zip ? arch.ReadChunk(inFileStream) : arch.ReadZippedChunk(inFileStream);
if (read > 0)
{
//TODO
//Thread th = new Thread(new ParameterizedThreadStart(act == ArchAction.Zip ? ZipWork : UnzipWork));
Thread th;
if (act == ArchAction.Zip)
{
//th = new Thread(new ParameterizedThreadStart(ZipWork));
th = new Thread(ZipWork);
}
else
{
//th = new Thread(new ParameterizedThreadStart(UnzipWork));
th = new Thread(UnzipWork);
}
th.Start(arch);
if (++idx >= archivers.Length)
{
idx = 0;
}
}
else
{
//have read all
if (archivers.Length == 1)
{
break;
}
if (--idx < 0)
{
idx = archivers.Length - 1;
}
General.Log($"main thread about to Wait for the last arch {idx} done...");
archivers[idx].Done.WaitOne(); //prev is finished, meaning all before prev are finished
//now safe to go out and dispose outputStream
break;
}
} while (true);
//archivers = null;//no need
}
General.Log($"{act} finished.");
}
