/// <summary> /// Creates all the objects required for the requested operation /// </summary> /// <param name="act"></param> /// <param name="bufferSize"></param> /// <param name="workersCount"></param> /// <param name="buffers"></param> /// <param name="workers"></param> /// <param name="workersTasks"></param> static void CreateBuffersAndWorkers(ArchAction act, int bufferSize, uint workersCount, out InOutBuffer[] buffers, out ArchWorker[] workers, out Task[] workerTasks) { CreateBuffers(workersCount, bufferSize, out buffers); CreateArchivers(buffers, act, out workers); workerTasks = new Task[workersCount]; }
/// <summary> /// Creates all the objects required for the requested operation /// </summary> /// <param name="act"></param> /// <param name="bufferSize"></param> /// <param name="workersCount"></param> /// <param name="buffers"></param> /// <param name="workers"></param> static void CreateBuffersAndWorkers(ArchAction act, int bufferSize, uint workersCount, out InOutBuffer[] buffers, out ThreadWorker[] workers) { CreateBuffers(workersCount, bufferSize, out buffers); CreateArchivers(buffers, act, out ArchWorker[] archivers); CreateThreadWorkers(archivers, out workers); }
/// <summary> /// c-tor /// </summary> /// <param name="inFileStream"></param> /// <param name="act">What action is requested - zipping or unzipping</param> public ArchReader(FileStream inFileStream, ArchAction act) { this.inFileStream = inFileStream; if (act == ArchAction.Zip) { Read = ReadNextChunk; } else { Read = ReadNextZippedChunk; } }
/// <summary> /// c-tor /// </summary> /// <param name="outFileStream"></param> /// <param name="act">What action was performed - zipping or unzipping</param> public ArchWriter(FileStream outFileStream, ArchAction act) { this.outFileStream = outFileStream; if (act == ArchAction.Zip) { Write = WriteZippedChunk; } else { Write = WriteChunk; } }
/// <summary> /// Create those who do zip/unzip /// </summary> /// <param name="buffers"></param> /// <param name="act">zip or unzip</param> /// <param name="archivers">out param</param> /// <returns></returns> static void CreateArchivers(InOutBuffer[] buffers, ArchAction act, out ArchWorker[] archivers) { int count = buffers.Length; Debug.Assert(count > 0); General.Log($"Creating {count} (un)zippers"); archivers = new ArchWorker[count]; for (int i = 0; i < archivers.Length; i++) { archivers[i] = new ArchWorker(inBuffer: buffers[i], outBuffer: buffers[i], act); } }
async private static Task ManageThreadsAsync(string inputName, string outputName, ArchAction act, int bufferSize, uint workersCount = 0) { InOutBuffer[] buffers = null; Task[] workerTasks = null; try { General.Log($"Started '{inputName}' {act} into '{outputName}'..."); if (bufferSize < defaultChunkSize) { bufferSize = defaultChunkSize; } if (workersCount < 1) { workersCount = ChooseWorkersCount(); } General.Log($"buffer size {bufferSize}, workers count {workersCount}"); //to feel good let's use 'using' // objects for input reading using FileStream inFileStream = File.OpenRead(inputName); ArchReader reader = new ArchReader(inFileStream, act); // objects for output writing using FileStream outFileStream = File.Create(outputName); ArchWriter writer = new ArchWriter(outFileStream, act); // create objects to do zipping/unzipping CreateBuffersAndWorkers(act, bufferSize, workersCount, out buffers, out ArchWorker[] workers, out workerTasks); await StartConveyor(reader, workers, workerTasks, buffers); int idx = 0; // as long as there are bytes read from input, keep conveyor // when no more input data, set this flag & wait till all therads are finihed. bool finishing = false; do { ArchWorker worker = workers[idx]; if (worker == null) { General.Log($"all workers are finished"); break; } General.Log($"Wait for worker {idx} done, to start writting"); await workerTasks[idx];//wait till zipper idx has done General.Log($"{idx} start writting"); writer.Buffer = buffers[idx]; Task writerTask = writer.WriteAsync(); if (!finishing) { General.Log($"{idx} start reading next portion"); reader.Buffer = buffers[idx]; Task readerTask = reader.ReadAsync(); General.Log($"{idx} need both In & Out buffers to be Ready to start next part zip/unzip"); await readerTask; // check if have read anyting if (reader.Buffer.BytesRead <= 0) { //have read all, and all is (being) processed finishing = true; } } await writerTask; if (finishing) { workers[idx] = null; workerTasks[idx] = null; //according to info from ms , no need to dispose a Task buffers[idx] = null; } else { workerTasks[idx] = worker.DoWorkAsync(); } //cause rotating, and output shall be in-order if (++idx >= workers.Length) { idx = 0; } } while (true); General.Log($"{act} finished successfuly."); } catch (Exception exc) { General.Log($"ManageThreads encountered an error {exc.Message} returning to caller."); throw; } finally { CleanUp(buffers, workerTasks); } }
private static void ManageThreads(string inputName, string outputName, ArchAction act, int bufferSize, uint workersCount = 0) { InOutBuffer[] buffers = null; ThreadWorker[] workers = null; try { General.Log($"Started '{inputName}' {act} into '{outputName}'..."); if (bufferSize < defaultChunkSize) { bufferSize = defaultChunkSize; } if (workersCount < 1) { workersCount = ChooseWorkersCount(); } General.Log($"buffer size {bufferSize}, workers count {workersCount}"); //to feel good let's use 'using' // objects for input reading using FileStream inFileStream = File.OpenRead(inputName); ArchReader reader = new ArchReader(inFileStream, act); using ThreadWorker readerThread = new ThreadWorker(reader, true); // objects for output writing using FileStream outFileStream = File.Create(outputName); ArchWriter writer = new ArchWriter(outFileStream, act); using ThreadWorker writerThread = new ThreadWorker(writer, true); // create objects to do zipping/unzipping CreateBuffersAndWorkers(act, bufferSize, workersCount, out buffers, out workers); StartConveyor(reader, readerThread, workers, buffers); int idx = 0; // as long as there are bytes read from input, keep conveyor // when no more input data, set this flag & wait till all therads are finihed. bool finishing = false; do { ThreadWorker worker = workers[idx]; if (worker == null) { General.Log($"all threads are finished"); // good place to let the writer go writerThread.Finish(); //writerThread.Dispose(); break; } General.Log($"main thread about to Wait for worker {idx} done, to start writting"); worker.notifyDone.WaitOne(); //wait till zipper idx has done General.Log($"{idx} start writting"); writer.Buffer = buffers[idx]; writerThread.haveWork.Set(); if (!finishing) { General.Log($"{idx} start reading next portion"); reader.Buffer = buffers[idx]; readerThread.haveWork.Set(); General.Log($"{idx} need both In & Out buffers to be Ready to start next part zippin"); readerThread.notifyDone.WaitOne(); // check if have read anyting if (reader.Buffer.BytesRead <= 0) { //have read all, and all is (being) processed finishing = true; // right place to set the reader free readerThread.Finish(); //readerThread.Dispose(); } } writerThread.notifyDone.WaitOne(); if (finishing) { worker.Finish(); workers[idx].Dispose(); workers[idx] = null; //can clean buffers here as well buffers[idx].Dispose(); buffers[idx] = null; } else { worker.haveWork.Set(); } //cause rotating, and output shall be in-order if (++idx >= workers.Length) { idx = 0; } } while (true); General.Log($"{act} finished successfuly."); } catch (Exception exc) { General.Log($"ManageThreads encountered an error {exc.Message} returning to caller."); throw; } finally { CleanUp(buffers, workers); } }
private static void ManageThreads(string inputName, string outputName, ArchAction act, int bufferSize, uint workersCount = 0) { General.Log($"{act} started."); Archiver.act = act; if (bufferSize > defaultChunkSize) { ArchWorker.ChunkSize = bufferSize; } using (FileStream outFileStream = File.Create(outputName)) { ArchWorker.OutFileStream = outFileStream; using FileStream inFileStream = File.OpenRead(inputName); if (workersCount < 1) { workersCount = ChooseWorkersCount(); } ArchWorker[] archivers = CreateWorkers(workersCount); int idx = 0; do { ArchWorker arch = archivers[idx]; General.Log($"main thread about to Wait for arch {idx} done, to start it in new thread again"); arch.Done.WaitOne(); //Load the next portion to be processed and written int read = act == ArchAction.Zip ? arch.ReadChunk(inFileStream) : arch.ReadZippedChunk(inFileStream); if (read > 0) { //TODO //Thread th = new Thread(new ParameterizedThreadStart(act == ArchAction.Zip ? ZipWork : UnzipWork)); Thread th; if (act == ArchAction.Zip) { //th = new Thread(new ParameterizedThreadStart(ZipWork)); th = new Thread(ZipWork); } else { //th = new Thread(new ParameterizedThreadStart(UnzipWork)); th = new Thread(UnzipWork); } th.Start(arch); if (++idx >= archivers.Length) { idx = 0; } } else { //have read all if (archivers.Length == 1) { break; } if (--idx < 0) { idx = archivers.Length - 1; } General.Log($"main thread about to Wait for the last arch {idx} done..."); archivers[idx].Done.WaitOne(); //prev is finished, meaning all before prev are finished //now safe to go out and dispose outputStream break; } } while (true); //archivers = null;//no need } General.Log($"{act} finished."); }