private void LaunchWorkerThreads([NotNull] ParallelJsonLauncherOptions options, DateTime startTime)
{
_threads.Clear();
Thread archiveThread = null;
//archiving Thread
if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory))
{
archiveThread = new Thread(() => ArchiveEverythingThread(options));
archiveThread.Start();
}
if (!options.ArchiveOnly)
{
var totalCount = _calculationsToProcess.Count;
int maxThreads = (int)Math.Floor(Environment.ProcessorCount / 2.0 * 0.95);
if (options.NumberOfCores > 0)
{
maxThreads = options.NumberOfCores;
}
for (var i = 0; i < maxThreads && i < totalCount; i++)
{
var i1 = i;
var t = new Thread(() => SingleThreadExecutor(i1));
t.Start();
t.Name = "Executor " + i;
_threads.Add(t);
Thread.Sleep(500);
}
}
foreach (var thread in _threads)
{
thread.Join();
}
if (archiveThread != null)
{
Logger.Info("Waiting for archiving to finish, " + _foldersToArchive.Count + " folders left");
while (_foldersToArchive.Count > 0 && archiveThread.IsAlive)
{
Logger.Info("Waiting for archiving to finish, " + _foldersToArchive.Count + " folders left");
Thread.Sleep(1000);
}
_continueArchiving = false;
archiveThread.Join();
}
Logger.Info("All threads finished");
var duration = DateTime.Now - startTime;
Logger.Info("Calculation Duration: " + duration + " (" + duration.TotalMinutes.ToString("F1", CultureInfo.InvariantCulture) + " minutes)");
}
public static void LaunchParallel([NotNull] ParallelJsonLauncherOptions options)
{
Logger.LogFileIndex = "JsonParallelCalculationStarter";
Logger.LogToFile = true;
CalculationProfiler calculationProfiler = new CalculationProfiler();
calculationProfiler.StartPart(Utili.GetCurrentMethodAndClass());
var pl = new ParallelJsonLauncher();
pl.LaunchParallelInternal(options);
calculationProfiler.StopPart(Utili.GetCurrentMethodAndClass());
}
private void LaunchParallelInternal([NotNull] ParallelJsonLauncherOptions options)
{
Logger.LogToFile = true;
Logger.Info("Reading options");
Logger.Info("Json Directory: " + options.JsonDirectory);
Logger.Info("Maximum number of calculations to perform before quitting: " + options.MaximumNumberOfCalculations);
Logger.Info("Number of cores to use: " + options.NumberOfCores);
Logger.Info("Search in the subdirectories for files: " + options.SearchSecondLevel);
Logger.Info("Archiving folder is " + options.ArchiveDirectory);
BaseDirectoryInfo = new DirectoryInfo(Environment.CurrentDirectory);
var startTime = DateTime.Now;
if (string.IsNullOrEmpty(options.JsonDirectory))
{
Logger.Error("No JSON directory file was set!");
return;
}
if (options.NumberOfCores == 0)
{
Logger.Error("Number of cores to use was set to 0. Not very useful.");
return;
}
if (!File.Exists("simulationengine.exe"))
{
throw new LPGException("simulationengine.exe was not found in the directory " + Environment.CurrentDirectory);
}
FillCalculationQueue(options);
Logger.Info("Read " + _calculationsToProcess.Count + " entries to calculate.");
LaunchWorkerThreads(options, startTime);
Logger.Info("Cleanup finished");
foreach (var calc in _completedCalcs)
{
Logger.Info("\"" + calc.FileName + "\": " + calc.Duration.ToString() + " Status: " + calc.Status);
}
int successCount = _completedCalcs.Count(x => x.IsSuccess);
Logger.Info("Finished " + _completedCalcs.Count + " calculations, " + successCount + " successfully");
}
private string GetArchiveDirectory([NotNull] DirectoryInfo outputDirectory, [NotNull] ParallelJsonLauncherOptions pjl)
{
if (BaseDirectoryInfo == null)
{
throw new LPGException("BaseDirectory was not set.");
}
var relativePath = outputDirectory.FullName.Replace(BaseDirectoryInfo.FullName, "");
if (pjl.ArchiveDirectory == null || string.IsNullOrWhiteSpace(pjl.ArchiveDirectory))
{
throw new Exception("Output directory was null");
}
if (relativePath.StartsWith("\\", StringComparison.OrdinalIgnoreCase))
{
relativePath = relativePath.Substring(1);
}
var newPath = Path.Combine(pjl.ArchiveDirectory, relativePath);
Logger.Info("Archiving new path is " + newPath + ", archive base: " + pjl.ArchiveDirectory);
return(newPath);
}
private void ArchiveEverythingThread([NotNull] ParallelJsonLauncherOptions options)
{
if (options.ArchiveDirectory == null || string.IsNullOrWhiteSpace(options.ArchiveDirectory))
{
Logger.Info("archive directory was not set, quitting archiving thread");
return;
}
Logger.Info("Archive directory is " + options.ArchiveDirectory);
try {
if (!Directory.Exists(options.ArchiveDirectory))
{
Directory.CreateDirectory(options.ArchiveDirectory);
Thread.Sleep(500);
}
while (_continueArchiving)
{
if (_foldersToArchive.Count > 0)
{
var success = _foldersToArchive.TryDequeue(out var calculationDirectory);
if (success)
{
try {
var oldPath = new DirectoryInfo(calculationDirectory);
var files = oldPath.GetFiles("*.*", SearchOption.AllDirectories);
long totalSize = 0;
foreach (var fileInfo in files)
{
totalSize += fileInfo.Length;
}
if (options.ArchiveDirectory?.Contains(":") == true)
{
var di = new DriveInfo(options.ArchiveDirectory.Substring(0, 1));
while (di.AvailableFreeSpace * 1.1 < totalSize)
{
Logger.Info("Free Space on the Archive Drive: " + di.AvailableFreeSpace / (1024.0 * 1024 * 1024) + " GB");
Logger.Warning("Needed: " + totalSize * 1.1 / 1024 / 1024 / 1024 + " GB");
Thread.Sleep(5000);
}
}
var newPath = GetArchiveDirectory(oldPath, options);
if (!options.TestArchiving)
{
CopyAll(oldPath, new DirectoryInfo(newPath));
oldPath.Delete(true);
}
Logger.Info("finished archiving " + calculationDirectory + " to " + newPath + ", calcs left:" + _calculationsToProcess.Count + ", directories to archive left right now: " + _foldersToArchive.Count);
}
catch (Exception ex) {
Logger.Error("Archiving failed: for " + calculationDirectory);
Logger.Exception(ex);
}
}
else
{
Thread.Sleep(1000);
}
}
}
Logger.Error("Exiting Archive Thread");
}
catch (Exception ex) {
Logger.Exception(ex);
}
}
private void FillCalculationQueue([NotNull] ParallelJsonLauncherOptions options)
{
Logger.Info("Starting to look for all the json files to calculate...");
if (options.JsonDirectory == null)
{
throw new LPGCommandlineException("Json directory was not set.");
}
DirectoryInfo di = new DirectoryInfo(options.JsonDirectory);
if (!di.Exists)
{
throw new LPGCommandlineException("Directory " + di.FullName + " was not found.");
}
var inputFiles = di.GetFiles("*.json").ToList();
if (options.SearchSecondLevel)
{
foreach (DirectoryInfo subdir in di.GetDirectories())
{
inputFiles.AddRange(subdir.GetFiles("*.json"));
}
}
if (inputFiles.Count == 0)
{
throw new LPGCommandlineException("No input files were found.");
}
if (options.MaximumNumberOfCalculations > 0 && options.MaximumNumberOfCalculations < inputFiles.Count)
{
inputFiles = inputFiles.Take(options.MaximumNumberOfCalculations).ToList();
}
for (var calcidx = 0; calcidx < inputFiles.Count; calcidx++)
{
HouseCreationAndCalculationJob jcs = LoadFromFile(inputFiles[calcidx].FullName);
if (jcs == null)
{
continue;
}
//check if there even is an output directory
if (string.IsNullOrWhiteSpace(jcs.CalcSpec?.OutputDirectory))
{
Logger.Error("Skipping file: No output directory set in the file " + inputFiles[calcidx].Name);
if (ThrowOnInvalidFile)
{
throw new LPGException("No output file was set: " + jcs.CalcSpec?.OutputDirectory);
}
continue;
}
// ReSharper disable once AssignNullToNotNullAttribute
DirectoryInfo outputDir = new DirectoryInfo(jcs.CalcSpec.OutputDirectory);
//check the archive directory if the results already exist
if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory))
{
//make archive path
string archivePath = GetArchiveDirectory(outputDir, options);
string finishedFlagPath = Path.Combine(archivePath, "finished.flag");
if (File.Exists(finishedFlagPath))
{
Logger.Info("Found finished folder in archive: " + archivePath);
continue;
}
}
//check current directory
if (outputDir.Exists)
{
var calculatedFiles = outputDir.GetFiles("finished.flag");
if (calculatedFiles.Length == 1)
{
Logger.Warning(outputDir.Name + " seems finished. Enqueueing for archiving");
_foldersToArchive.Enqueue(outputDir.FullName);
}
else
{
Logger.Warning(outputDir.Name + " had files left over. Deleting and trying again.");
outputDir.Delete(true);
}
}
Logger.Info("Enqueuing file " + inputFiles[calcidx].Name);
_calculationsToProcess.Enqueue(new CalcJobQueueEntry(inputFiles[calcidx], calcidx));
_totalCalculations++;
}
}
