예제 #1
0
 private void ArchiveEverything([NotNull] ParallelLauncherOptions options)
 {
     try {
         if (!string.IsNullOrEmpty(options.ArchiveDirectory))
         {
             if (!Directory.Exists(options.ArchiveDirectory))
             {
                 Directory.CreateDirectory(options.ArchiveDirectory);
                 Thread.Sleep(500);
             }
             while (_continueArchiving)
             {
                 Thread.Sleep(1000);
                 if (_foldersToArchive.Count > 0)
                 {
                     var success = _foldersToArchive.TryDequeue(out var archivePath);
                     if (success)
                     {
                         try {
                             var  oldPath   = new DirectoryInfo(archivePath);
                             var  files     = oldPath.GetFiles("*.*", SearchOption.AllDirectories);
                             long totalSize = 0;
                             foreach (var fileInfo in files)
                             {
                                 totalSize += fileInfo.Length;
                             }
                             if (options.ArchiveDirectory?.Contains(":") == true)
                             {
                                 var di = new DriveInfo(options.ArchiveDirectory.Substring(0, 1));
                                 while (di.AvailableFreeSpace * 1.1 < totalSize)
                                 {
                                     Logger.Info("Free Space on the Archive Drive: " +
                                                 di.AvailableFreeSpace / (1024.0 * 1024 * 1024) + " GB");
                                     Logger.Warning("Needed: " + totalSize * 1.1 / 1024 / 1024 / 1024 + " GB");
                                     Thread.Sleep(5000);
                                 }
                             }
                             if (options.ArchiveDirectory != null)
                             {
                                 var newPath =
                                     new DirectoryInfo(Path.Combine(options.ArchiveDirectory, oldPath.Name));
                                 CopyAll(oldPath, newPath);
                                 oldPath.Delete(true);
                                 Logger.Info("finished archiving " + archivePath + ", calcs left:" +
                                             _calculationsToProcess.Count);
                             }
                         }
                         catch (Exception ex) {
                             Logger.Error("Archiving failed: for " + archivePath);
                             Logger.Exception(ex);
                         }
                     }
                 }
             }
         }
         Logger.Error("Exiting Archive Thread");
     }
     catch (Exception ex) {
         Logger.Exception(ex);
     }
 }
예제 #2
0
        private void LaunchParallelInternal([NotNull] ParallelLauncherOptions options)
        {
            Logger.LogToFile = true;
            Logger.Info("Reading options");
            var startTime = DateTime.Now;

            if (string.IsNullOrEmpty(options.Batchfile))
            {
                Logger.Error("No batch file was set!");
                return;
            }
            if (options.NumberOfCores == 0)
            {
                Logger.Error("No number of cores to use was set");
                return;
            }
            Logger.Info("Archiving folder is " + options.ArchiveDirectory);
            var outputDirectories = new List <string>();
            var batchFileLines    = new List <string>();

            using (var sr = new StreamReader(options.Batchfile)) {
                while (!sr.EndOfStream)
                {
                    var s = sr.ReadLine();

                    if (!string.IsNullOrWhiteSpace(s))
                    {
                        batchFileLines.Add(s);
                    }
                }
            }
            if (options.MaximumCount > 0 && options.MaximumCount < batchFileLines.Count)
            {
                batchFileLines = batchFileLines.Take(options.MaximumCount).ToList();
            }
            for (var calcidx = 0; calcidx < batchFileLines.Count; calcidx++)
            {
                var addthis = true;
                if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory))
                {
                    // find existing directories
                    var arrs       = batchFileLines[calcidx].Split(' ');
                    var dstdirName = string.Empty;
                    for (var i = 0; i < arrs.Length; i++)
                    {
                        if (arrs[i] == "-OutputDirectory")
                        {
                            dstdirName = arrs[i + 1];
                            outputDirectories.Add(dstdirName);
                        }
                    }
                    if (string.IsNullOrWhiteSpace(dstdirName))
                    {
                        Logger.Error(
                            "Could not identify the output directory parameter in the command line. Can't archive.");
                    }
                    else
                    {
                        var dstFullName = Path.Combine(options.ArchiveDirectory, dstdirName);
                        if (Directory.Exists(dstFullName))
                        {
                            addthis = false;
                        }
                    }
                }
                if (addthis)
                {
                    _calculationsToProcess.Enqueue(batchFileLines[calcidx]);
                    _totalCalculations++;
                }
                else
                {
                    Logger.Warning("Skipping " + batchFileLines[calcidx]);
                }
            }

            if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory) && Directory.Exists(options.ArchiveDirectory))
            {
                var archiveDirectoryInfo = new DirectoryInfo(options.ArchiveDirectory);
                var subfos = archiveDirectoryInfo.GetDirectories();
                foreach (var subfo in subfos)
                {
                    if (!outputDirectories.Contains(subfo.Name))
                    {
                        if (archiveDirectoryInfo.Parent == null)
                        {
                            throw new LPGException("Directory archive was null");
                        }
                        var oldarchivePath = Path.Combine(archiveDirectoryInfo.Parent.FullName,
                                                          "Old_" + archiveDirectoryInfo.Name);
                        if (!Directory.Exists(oldarchivePath))
                        {
                            Directory.CreateDirectory(oldarchivePath);
                            Thread.Sleep(500);
                        }
                        Logger.Info("Moving " + subfo.FullName + " to " + oldarchivePath);
                        var newpath = Path.Combine(oldarchivePath, subfo.Name);
                        subfo.MoveTo(newpath);
                    }
                }
            }

            Logger.Info("Read " + _calculationsToProcess.Count + " entries to calculate.");
            var threads    = new List <Thread>();
            var totalCount = _calculationsToProcess.Count;
            int maxThreads = (int)Math.Floor(Environment.ProcessorCount / 2.0 * 0.95);

            if (options.NumberOfCores > 0)
            {
                maxThreads = options.NumberOfCores;
            }
            for (var i = 0; i < maxThreads && i < totalCount; i++)
            {
                var i1 = i;
                var t  = new Thread(() => Executor(i1, options));
                t.Start();
                threads.Add(t);
                Thread.Sleep(500);
            }
            if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory))
            {
                var archive = new Thread(() => ArchiveEverything(options));
                archive.Start();
            }
            foreach (var thread in threads)
            {
                thread.Join();
            }
            Logger.Info("All threads finished");
            if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory))
            {
                Thread.Sleep(10000);
                while (_foldersToArchive.Count > 0)
                {
                    Thread.Sleep(1000);
                }
            }
            _continueArchiving = false;
            var duration = DateTime.Now - startTime;

            Logger.Info("Calculation Duration: " + duration + " (" + duration.TotalMinutes + " minutes)");
            Logger.Info("Cleanup finished");
        }
예제 #3
0
        public static void LaunchParallel([NotNull] ParallelLauncherOptions options)
        {
            var pl = new ParallelLauncher();

            pl.LaunchParallelInternal(options);
        }
예제 #4
0
 private void Executor(int index, [NotNull] ParallelLauncherOptions options)
 {
     try {
         Logger.Info("Starting thread " + index);
         var newfilename = "db" + index + ".db3";
         Logger.Info("Copying database to " + newfilename);
         if (File.Exists(newfilename))
         {
             File.Delete(newfilename);
             Thread.Sleep(500 + _r.Next(1000));
         }
         File.Copy("profilegenerator.db3", newfilename);
         Thread.Sleep(500 + _r.Next(1000));
         var       di        = new DirectoryInfo(Directory.GetCurrentDirectory());
         DriveInfo driveInfo = null;
         if (di.FullName.Contains(":"))
         {
             var drive = di.FullName.Substring(0, 1);
             driveInfo = new DriveInfo(drive);
         }
         while (!_calculationsToProcess.IsEmpty && _continueProcessing)
         {
             try {
                 if (Console.KeyAvailable)
                 {
                     if (Console.ReadKey(true).Key == ConsoleKey.Q)
                     {
                         _continueProcessing = false;
                         while (!_calculationsToProcess.IsEmpty)
                         {
                             _calculationsToProcess.TryDequeue(out var _);
                         }
                         Logger.Info("Thread: " + index + "Q was pressed. Quitting soon.");
                     }
                 }
             }
             catch (Exception ex) {
                 Logger.Info(ex.Message);
                 Logger.Exception(ex);
             }
             var success = _calculationsToProcess.TryDequeue(out var path);
             if (success && _continueProcessing)
             {
                 if (driveInfo != null)
                 {
                     Logger.Info("Thread " + index + ": Free Space on drive " + driveInfo.RootDirectory + ": " +
                                 driveInfo.AvailableFreeSpace / 1024.0 / 1024 / 1024.0 + " GB");
                     var count = 0;
                     while (driveInfo.AvailableFreeSpace / 1024.0 / 1024 / 1024 < 5)
                     {
                         count++;
                         Logger.Warning("Thread " + index + ": Please make at least 5GB space on " +
                                        driveInfo.RootDirectory);
                         Thread.Sleep(5000);
                         if (count > 5 && !Config.CatchErrors)
                         {
                             throw new LPGException("Not Enough space!");
                         }
                     }
                 }
                 while (_foldersToArchive.Count > 0 && !string.IsNullOrWhiteSpace(options.ArchiveDirectory))
                 {
                     Logger.Warning("Thread " + index + ": Waiting for archiving to finish. " +
                                    _foldersToArchive.Count + " left.");
                     Thread.Sleep(5000);
                 }
                 if (!_continueProcessing)
                 {
                     return;
                 }
                 var processesleft = _calculationsToProcess.Count;
                 var processesdone = _totalCalculations - processesleft;
                 var progress      = " (" + processesdone + "/" + _totalCalculations + ")";
                 Logger.Info("Thread " + index + ": Processing " + path + progress);
                 var arguments      = path.Substring("SimulationEngine.exe".Length);
                 var splitarguments = arguments.Split(' ');
                 if (splitarguments.Length < 2)
                 {
                     throw new LPGException("could not split the arguments!");
                 }
                 string outputfolder = null;
                 for (var i = 0; i < splitarguments.Length; i++)
                 {
                     if (splitarguments[i] == "-OutputDirectory")
                     {
                         outputfolder = splitarguments[i + 1];
                     }
                 }
                 var startinfo = new ProcessStartInfo();
                 using (var process = new Process()) {
                     startinfo.Arguments       = arguments + " -Database " + newfilename;
                     startinfo.UseShellExecute = true;
                     startinfo.WindowStyle     = ProcessWindowStyle.Normal;
                     startinfo.FileName        = "SimulationEngine.exe";
                     process.StartInfo         = startinfo;
                     process.Start();
                     SetWindowText(process.MainWindowHandle, arguments);
                     process.WaitForExit();
                 }
                 Logger.Info("Thread " + index + ": Finished " + path);
                 Thread.Sleep(1000);
                 if (outputfolder != null)
                 {
                     var    fullOutputFolder = Path.Combine(Directory.GetCurrentDirectory(), outputfolder);
                     string finishedFile     = Path.Combine(fullOutputFolder, "finished.flag");
                     string errorpath        = Path.Combine(fullOutputFolder, "Logfile.Error.txt");
                     if (File.Exists(finishedFile) && !File.Exists(errorpath))
                     {
                         _foldersToArchive.Enqueue(fullOutputFolder);
                         Logger.Info("Thread " + index + ": Enqueued for moving: " + fullOutputFolder +
                                     progress);
                     }
                     else
                     {
                         Logger.Info("Thread " + index + ": Not enqueued for moving due to missing finished flag: " + fullOutputFolder +
                                     progress);
                     }
                 }
             }
         }
     }
     catch (Exception ex) {
         Logger.Info("Thread " + index + ": Exception:" + Environment.NewLine + ex.Message);
         Logger.Exception(ex);
     }
 }