private void LaunchWorkerThreads([NotNull] ParallelJsonLauncherOptions options, DateTime startTime) { _threads.Clear(); Thread archiveThread = null; //archiving Thread if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory)) { archiveThread = new Thread(() => ArchiveEverythingThread(options)); archiveThread.Start(); } if (!options.ArchiveOnly) { var totalCount = _calculationsToProcess.Count; int maxThreads = (int)Math.Floor(Environment.ProcessorCount / 2.0 * 0.95); if (options.NumberOfCores > 0) { maxThreads = options.NumberOfCores; } for (var i = 0; i < maxThreads && i < totalCount; i++) { var i1 = i; var t = new Thread(() => SingleThreadExecutor(i1)); t.Start(); t.Name = "Executor " + i; _threads.Add(t); Thread.Sleep(500); } } foreach (var thread in _threads) { thread.Join(); } if (archiveThread != null) { Logger.Info("Waiting for archiving to finish, " + _foldersToArchive.Count + " folders left"); while (_foldersToArchive.Count > 0 && archiveThread.IsAlive) { Logger.Info("Waiting for archiving to finish, " + _foldersToArchive.Count + " folders left"); Thread.Sleep(1000); } _continueArchiving = false; archiveThread.Join(); } Logger.Info("All threads finished"); var duration = DateTime.Now - startTime; Logger.Info("Calculation Duration: " + duration + " (" + duration.TotalMinutes.ToString("F1", CultureInfo.InvariantCulture) + " minutes)"); }
public static void LaunchParallel([NotNull] ParallelJsonLauncherOptions options) { Logger.LogFileIndex = "JsonParallelCalculationStarter"; Logger.LogToFile = true; CalculationProfiler calculationProfiler = new CalculationProfiler(); calculationProfiler.StartPart(Utili.GetCurrentMethodAndClass()); var pl = new ParallelJsonLauncher(); pl.LaunchParallelInternal(options); calculationProfiler.StopPart(Utili.GetCurrentMethodAndClass()); }
private void LaunchParallelInternal([NotNull] ParallelJsonLauncherOptions options) { Logger.LogToFile = true; Logger.Info("Reading options"); Logger.Info("Json Directory: " + options.JsonDirectory); Logger.Info("Maximum number of calculations to perform before quitting: " + options.MaximumNumberOfCalculations); Logger.Info("Number of cores to use: " + options.NumberOfCores); Logger.Info("Search in the subdirectories for files: " + options.SearchSecondLevel); Logger.Info("Archiving folder is " + options.ArchiveDirectory); BaseDirectoryInfo = new DirectoryInfo(Environment.CurrentDirectory); var startTime = DateTime.Now; if (string.IsNullOrEmpty(options.JsonDirectory)) { Logger.Error("No JSON directory file was set!"); return; } if (options.NumberOfCores == 0) { Logger.Error("Number of cores to use was set to 0. Not very useful."); return; } if (!File.Exists("simulationengine.exe")) { throw new LPGException("simulationengine.exe was not found in the directory " + Environment.CurrentDirectory); } FillCalculationQueue(options); Logger.Info("Read " + _calculationsToProcess.Count + " entries to calculate."); LaunchWorkerThreads(options, startTime); Logger.Info("Cleanup finished"); foreach (var calc in _completedCalcs) { Logger.Info("\"" + calc.FileName + "\": " + calc.Duration.ToString() + " Status: " + calc.Status); } int successCount = _completedCalcs.Count(x => x.IsSuccess); Logger.Info("Finished " + _completedCalcs.Count + " calculations, " + successCount + " successfully"); }
private string GetArchiveDirectory([NotNull] DirectoryInfo outputDirectory, [NotNull] ParallelJsonLauncherOptions pjl) { if (BaseDirectoryInfo == null) { throw new LPGException("BaseDirectory was not set."); } var relativePath = outputDirectory.FullName.Replace(BaseDirectoryInfo.FullName, ""); if (pjl.ArchiveDirectory == null || string.IsNullOrWhiteSpace(pjl.ArchiveDirectory)) { throw new Exception("Output directory was null"); } if (relativePath.StartsWith("\\", StringComparison.OrdinalIgnoreCase)) { relativePath = relativePath.Substring(1); } var newPath = Path.Combine(pjl.ArchiveDirectory, relativePath); Logger.Info("Archiving new path is " + newPath + ", archive base: " + pjl.ArchiveDirectory); return(newPath); }
private void ArchiveEverythingThread([NotNull] ParallelJsonLauncherOptions options) { if (options.ArchiveDirectory == null || string.IsNullOrWhiteSpace(options.ArchiveDirectory)) { Logger.Info("archive directory was not set, quitting archiving thread"); return; } Logger.Info("Archive directory is " + options.ArchiveDirectory); try { if (!Directory.Exists(options.ArchiveDirectory)) { Directory.CreateDirectory(options.ArchiveDirectory); Thread.Sleep(500); } while (_continueArchiving) { if (_foldersToArchive.Count > 0) { var success = _foldersToArchive.TryDequeue(out var calculationDirectory); if (success) { try { var oldPath = new DirectoryInfo(calculationDirectory); var files = oldPath.GetFiles("*.*", SearchOption.AllDirectories); long totalSize = 0; foreach (var fileInfo in files) { totalSize += fileInfo.Length; } if (options.ArchiveDirectory?.Contains(":") == true) { var di = new DriveInfo(options.ArchiveDirectory.Substring(0, 1)); while (di.AvailableFreeSpace * 1.1 < totalSize) { Logger.Info("Free Space on the Archive Drive: " + di.AvailableFreeSpace / (1024.0 * 1024 * 1024) + " GB"); Logger.Warning("Needed: " + totalSize * 1.1 / 1024 / 1024 / 1024 + " GB"); Thread.Sleep(5000); } } var newPath = GetArchiveDirectory(oldPath, options); if (!options.TestArchiving) { CopyAll(oldPath, new DirectoryInfo(newPath)); oldPath.Delete(true); } Logger.Info("finished archiving " + calculationDirectory + " to " + newPath + ", calcs left:" + _calculationsToProcess.Count + ", directories to archive left right now: " + _foldersToArchive.Count); } catch (Exception ex) { Logger.Error("Archiving failed: for " + calculationDirectory); Logger.Exception(ex); } } else { Thread.Sleep(1000); } } } Logger.Error("Exiting Archive Thread"); } catch (Exception ex) { Logger.Exception(ex); } }
private void FillCalculationQueue([NotNull] ParallelJsonLauncherOptions options) { Logger.Info("Starting to look for all the json files to calculate..."); if (options.JsonDirectory == null) { throw new LPGCommandlineException("Json directory was not set."); } DirectoryInfo di = new DirectoryInfo(options.JsonDirectory); if (!di.Exists) { throw new LPGCommandlineException("Directory " + di.FullName + " was not found."); } var inputFiles = di.GetFiles("*.json").ToList(); if (options.SearchSecondLevel) { foreach (DirectoryInfo subdir in di.GetDirectories()) { inputFiles.AddRange(subdir.GetFiles("*.json")); } } if (inputFiles.Count == 0) { throw new LPGCommandlineException("No input files were found."); } if (options.MaximumNumberOfCalculations > 0 && options.MaximumNumberOfCalculations < inputFiles.Count) { inputFiles = inputFiles.Take(options.MaximumNumberOfCalculations).ToList(); } for (var calcidx = 0; calcidx < inputFiles.Count; calcidx++) { HouseCreationAndCalculationJob jcs = LoadFromFile(inputFiles[calcidx].FullName); if (jcs == null) { continue; } //check if there even is an output directory if (string.IsNullOrWhiteSpace(jcs.CalcSpec?.OutputDirectory)) { Logger.Error("Skipping file: No output directory set in the file " + inputFiles[calcidx].Name); if (ThrowOnInvalidFile) { throw new LPGException("No output file was set: " + jcs.CalcSpec?.OutputDirectory); } continue; } // ReSharper disable once AssignNullToNotNullAttribute DirectoryInfo outputDir = new DirectoryInfo(jcs.CalcSpec.OutputDirectory); //check the archive directory if the results already exist if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory)) { //make archive path string archivePath = GetArchiveDirectory(outputDir, options); string finishedFlagPath = Path.Combine(archivePath, "finished.flag"); if (File.Exists(finishedFlagPath)) { Logger.Info("Found finished folder in archive: " + archivePath); continue; } } //check current directory if (outputDir.Exists) { var calculatedFiles = outputDir.GetFiles("finished.flag"); if (calculatedFiles.Length == 1) { Logger.Warning(outputDir.Name + " seems finished. Enqueueing for archiving"); _foldersToArchive.Enqueue(outputDir.FullName); } else { Logger.Warning(outputDir.Name + " had files left over. Deleting and trying again."); outputDir.Delete(true); } } Logger.Info("Enqueuing file " + inputFiles[calcidx].Name); _calculationsToProcess.Enqueue(new CalcJobQueueEntry(inputFiles[calcidx], calcidx)); _totalCalculations++; } }