private void ArchiveEverything([NotNull] ParallelLauncherOptions options)
{
try {
if (!string.IsNullOrEmpty(options.ArchiveDirectory))
{
if (!Directory.Exists(options.ArchiveDirectory))
{
Directory.CreateDirectory(options.ArchiveDirectory);
Thread.Sleep(500);
}
while (_continueArchiving)
{
Thread.Sleep(1000);
if (_foldersToArchive.Count > 0)
{
var success = _foldersToArchive.TryDequeue(out var archivePath);
if (success)
{
try {
var oldPath = new DirectoryInfo(archivePath);
var files = oldPath.GetFiles("*.*", SearchOption.AllDirectories);
long totalSize = 0;
foreach (var fileInfo in files)
{
totalSize += fileInfo.Length;
}
if (options.ArchiveDirectory?.Contains(":") == true)
{
var di = new DriveInfo(options.ArchiveDirectory.Substring(0, 1));
while (di.AvailableFreeSpace * 1.1 < totalSize)
{
Logger.Info("Free Space on the Archive Drive: " +
di.AvailableFreeSpace / (1024.0 * 1024 * 1024) + " GB");
Logger.Warning("Needed: " + totalSize * 1.1 / 1024 / 1024 / 1024 + " GB");
Thread.Sleep(5000);
}
}
if (options.ArchiveDirectory != null)
{
var newPath =
new DirectoryInfo(Path.Combine(options.ArchiveDirectory, oldPath.Name));
CopyAll(oldPath, newPath);
oldPath.Delete(true);
Logger.Info("finished archiving " + archivePath + ", calcs left:" +
_calculationsToProcess.Count);
}
}
catch (Exception ex) {
Logger.Error("Archiving failed: for " + archivePath);
Logger.Exception(ex);
}
}
}
}
}
Logger.Error("Exiting Archive Thread");
}
catch (Exception ex) {
Logger.Exception(ex);
}
}
private void LaunchParallelInternal([NotNull] ParallelLauncherOptions options)
{
Logger.LogToFile = true;
Logger.Info("Reading options");
var startTime = DateTime.Now;
if (string.IsNullOrEmpty(options.Batchfile))
{
Logger.Error("No batch file was set!");
return;
}
if (options.NumberOfCores == 0)
{
Logger.Error("No number of cores to use was set");
return;
}
Logger.Info("Archiving folder is " + options.ArchiveDirectory);
var outputDirectories = new List <string>();
var batchFileLines = new List <string>();
using (var sr = new StreamReader(options.Batchfile)) {
while (!sr.EndOfStream)
{
var s = sr.ReadLine();
if (!string.IsNullOrWhiteSpace(s))
{
batchFileLines.Add(s);
}
}
}
if (options.MaximumCount > 0 && options.MaximumCount < batchFileLines.Count)
{
batchFileLines = batchFileLines.Take(options.MaximumCount).ToList();
}
for (var calcidx = 0; calcidx < batchFileLines.Count; calcidx++)
{
var addthis = true;
if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory))
{
// find existing directories
var arrs = batchFileLines[calcidx].Split(' ');
var dstdirName = string.Empty;
for (var i = 0; i < arrs.Length; i++)
{
if (arrs[i] == "-OutputDirectory")
{
dstdirName = arrs[i + 1];
outputDirectories.Add(dstdirName);
}
}
if (string.IsNullOrWhiteSpace(dstdirName))
{
Logger.Error(
"Could not identify the output directory parameter in the command line. Can't archive.");
}
else
{
var dstFullName = Path.Combine(options.ArchiveDirectory, dstdirName);
if (Directory.Exists(dstFullName))
{
addthis = false;
}
}
}
if (addthis)
{
_calculationsToProcess.Enqueue(batchFileLines[calcidx]);
_totalCalculations++;
}
else
{
Logger.Warning("Skipping " + batchFileLines[calcidx]);
}
}
if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory) && Directory.Exists(options.ArchiveDirectory))
{
var archiveDirectoryInfo = new DirectoryInfo(options.ArchiveDirectory);
var subfos = archiveDirectoryInfo.GetDirectories();
foreach (var subfo in subfos)
{
if (!outputDirectories.Contains(subfo.Name))
{
if (archiveDirectoryInfo.Parent == null)
{
throw new LPGException("Directory archive was null");
}
var oldarchivePath = Path.Combine(archiveDirectoryInfo.Parent.FullName,
"Old_" + archiveDirectoryInfo.Name);
if (!Directory.Exists(oldarchivePath))
{
Directory.CreateDirectory(oldarchivePath);
Thread.Sleep(500);
}
Logger.Info("Moving " + subfo.FullName + " to " + oldarchivePath);
var newpath = Path.Combine(oldarchivePath, subfo.Name);
subfo.MoveTo(newpath);
}
}
}
Logger.Info("Read " + _calculationsToProcess.Count + " entries to calculate.");
var threads = new List <Thread>();
var totalCount = _calculationsToProcess.Count;
int maxThreads = (int)Math.Floor(Environment.ProcessorCount / 2.0 * 0.95);
if (options.NumberOfCores > 0)
{
maxThreads = options.NumberOfCores;
}
for (var i = 0; i < maxThreads && i < totalCount; i++)
{
var i1 = i;
var t = new Thread(() => Executor(i1, options));
t.Start();
threads.Add(t);
Thread.Sleep(500);
}
if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory))
{
var archive = new Thread(() => ArchiveEverything(options));
archive.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
Logger.Info("All threads finished");
if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory))
{
Thread.Sleep(10000);
while (_foldersToArchive.Count > 0)
{
Thread.Sleep(1000);
}
}
_continueArchiving = false;
var duration = DateTime.Now - startTime;
Logger.Info("Calculation Duration: " + duration + " (" + duration.TotalMinutes + " minutes)");
Logger.Info("Cleanup finished");
}
public static void LaunchParallel([NotNull] ParallelLauncherOptions options)
{
var pl = new ParallelLauncher();
pl.LaunchParallelInternal(options);
}
private void Executor(int index, [NotNull] ParallelLauncherOptions options)
{
try {
Logger.Info("Starting thread " + index);
var newfilename = "db" + index + ".db3";
Logger.Info("Copying database to " + newfilename);
if (File.Exists(newfilename))
{
File.Delete(newfilename);
Thread.Sleep(500 + _r.Next(1000));
}
File.Copy("profilegenerator.db3", newfilename);
Thread.Sleep(500 + _r.Next(1000));
var di = new DirectoryInfo(Directory.GetCurrentDirectory());
DriveInfo driveInfo = null;
if (di.FullName.Contains(":"))
{
var drive = di.FullName.Substring(0, 1);
driveInfo = new DriveInfo(drive);
}
while (!_calculationsToProcess.IsEmpty && _continueProcessing)
{
try {
if (Console.KeyAvailable)
{
if (Console.ReadKey(true).Key == ConsoleKey.Q)
{
_continueProcessing = false;
while (!_calculationsToProcess.IsEmpty)
{
_calculationsToProcess.TryDequeue(out var _);
}
Logger.Info("Thread: " + index + "Q was pressed. Quitting soon.");
}
}
}
catch (Exception ex) {
Logger.Info(ex.Message);
Logger.Exception(ex);
}
var success = _calculationsToProcess.TryDequeue(out var path);
if (success && _continueProcessing)
{
if (driveInfo != null)
{
Logger.Info("Thread " + index + ": Free Space on drive " + driveInfo.RootDirectory + ": " +
driveInfo.AvailableFreeSpace / 1024.0 / 1024 / 1024.0 + " GB");
var count = 0;
while (driveInfo.AvailableFreeSpace / 1024.0 / 1024 / 1024 < 5)
{
count++;
Logger.Warning("Thread " + index + ": Please make at least 5GB space on " +
driveInfo.RootDirectory);
Thread.Sleep(5000);
if (count > 5 && !Config.CatchErrors)
{
throw new LPGException("Not Enough space!");
}
}
}
while (_foldersToArchive.Count > 0 && !string.IsNullOrWhiteSpace(options.ArchiveDirectory))
{
Logger.Warning("Thread " + index + ": Waiting for archiving to finish. " +
_foldersToArchive.Count + " left.");
Thread.Sleep(5000);
}
if (!_continueProcessing)
{
return;
}
var processesleft = _calculationsToProcess.Count;
var processesdone = _totalCalculations - processesleft;
var progress = " (" + processesdone + "/" + _totalCalculations + ")";
Logger.Info("Thread " + index + ": Processing " + path + progress);
var arguments = path.Substring("SimulationEngine.exe".Length);
var splitarguments = arguments.Split(' ');
if (splitarguments.Length < 2)
{
throw new LPGException("could not split the arguments!");
}
string outputfolder = null;
for (var i = 0; i < splitarguments.Length; i++)
{
if (splitarguments[i] == "-OutputDirectory")
{
outputfolder = splitarguments[i + 1];
}
}
var startinfo = new ProcessStartInfo();
using (var process = new Process()) {
startinfo.Arguments = arguments + " -Database " + newfilename;
startinfo.UseShellExecute = true;
startinfo.WindowStyle = ProcessWindowStyle.Normal;
startinfo.FileName = "SimulationEngine.exe";
process.StartInfo = startinfo;
process.Start();
SetWindowText(process.MainWindowHandle, arguments);
process.WaitForExit();
}
Logger.Info("Thread " + index + ": Finished " + path);
Thread.Sleep(1000);
if (outputfolder != null)
{
var fullOutputFolder = Path.Combine(Directory.GetCurrentDirectory(), outputfolder);
string finishedFile = Path.Combine(fullOutputFolder, "finished.flag");
string errorpath = Path.Combine(fullOutputFolder, "Logfile.Error.txt");
if (File.Exists(finishedFile) && !File.Exists(errorpath))
{
_foldersToArchive.Enqueue(fullOutputFolder);
Logger.Info("Thread " + index + ": Enqueued for moving: " + fullOutputFolder +
progress);
}
else
{
Logger.Info("Thread " + index + ": Not enqueued for moving due to missing finished flag: " + fullOutputFolder +
progress);
}
}
}
}
}
catch (Exception ex) {
Logger.Info("Thread " + index + ": Exception:" + Environment.NewLine + ex.Message);
Logger.Exception(ex);
}
}