/// <summary>
/// Copy file from source to target
/// </summary>
/// <param name="sourceFile"></param>
/// <param name="targetFile"></param>
public static void CopyFile(string sourceFile, string targetFile)
{
FileInfo Source = new FileInfo(sourceFile);
if (Source == null)
{
StaticClass.Logger.LogError("FileHandling Source failed to instantiate");
}
FileInfo Target = new FileInfo(targetFile);
if (Target == null)
{
StaticClass.Logger.LogError("FileHandling Target failed to instantiate");
}
if (Target.Exists)
{
// Delete the Target file first
lock (FileLock)
{
File.SetAttributes(Target.FullName, FileAttributes.Normal);
Target.Delete();
}
}
// Copy to target file
lock (FileLock)
{
Source.CopyTo(targetFile);
}
StaticClass.Log(string.Format("Copied File {0} -> {1}", sourceFile, targetFile));
}
/// <summary>
/// Log a Status and write to csv file
/// </summary>
/// <param name="job"></param>
/// <param name="status"></param>
/// <param name="timeSlot"></param>
public void ListStatus(string job, JobStatus status, JobType timeSlot)
{
StatusData entry = new StatusData();
if (entry == null)
{
StaticClass.Logger.LogError("StatusEntry entry failed to instantiate");
}
entry.Job = job;
entry.JobStatus = status;
switch (timeSlot)
{
case JobType.TIME_RECEIVED:
entry.TimeReceived = DateTime.Now;
break;
case JobType.TIME_START:
entry.TimeStarted = DateTime.Now;
break;
case JobType.TIME_COMPLETE:
entry.TimeCompleted = DateTime.Now;
break;
}
// Add entry to status data list
StaticClass.StatusDataList.Add(entry);
// Output to Console also
StaticClass.Log(string.Format("Status: Job: {0} Job Status: {1} Time Type: {2}",
job, status, timeSlot.ToString()));
}
/// <summary>
/// Method to start new jobs from the Input Buffer
/// </summary>
/// <param name="directory"></param>
public void StartInputJob(string directory)
{
// Get data found in Job xml file
JobXmlData jobXmlData = StaticClass.GetJobXmlFileInfo(directory, DirectoryScanType.INPUT_BUFFER);
if (jobXmlData == null)
{
StaticClass.Logger.LogError("InputJobsScanThread GetJobXmlData failed");
}
// Display job xml data found
StaticClass.Log("Input Job : " + jobXmlData.Job);
StaticClass.Log("Input Job Directory : " + jobXmlData.JobDirectory);
StaticClass.Log("Input Job Serial Number : " + jobXmlData.JobSerialNumber);
StaticClass.Log("Input Job Time Stamp : " + jobXmlData.TimeStamp);
StaticClass.Log("Input Job Xml File : " + jobXmlData.XmlFileName);
StaticClass.Log(string.Format("Started Input Job {0} executing Slot {1} at {2:HH:mm:ss.fff}",
jobXmlData.Job, StaticClass.NumberOfJobsExecuting + 1, DateTime.Now));
// Create a thread to run the job, and then start the thread
JobRunThread jobRunThread = new JobRunThread(jobXmlData, DirectoryScanType.INPUT_BUFFER);
if (jobRunThread == null)
{
StaticClass.Logger.LogError("InputJobsScanThread jobRunThread failed to instantiate");
}
jobRunThread.ThreadProc();
}
/// <summary>
/// Display the Modeler Process information
/// </summary>
/// <param name="modelerProcess"></param>
public void DisplayProcessInfo(string job, Process modelerProcess)
{
// Display Modeler Executable information
StaticClass.Log($"\nJob {job} Modeler execution process data:");
StaticClass.Log($"ProcessName : {modelerProcess.ProcessName}");
StaticClass.Log($"StartTime : {modelerProcess.StartTime}");
StaticClass.Log($"MainWindowTitle : {modelerProcess.MainWindowTitle}");
StaticClass.Log($"MainModule : {modelerProcess.MainModule}");
StaticClass.Log($"StartInfo : {modelerProcess.StartInfo}");
StaticClass.Log($"GetType : {modelerProcess.GetType()}");
StaticClass.Log($"MainWindowHandle : {modelerProcess.MainWindowHandle}");
StaticClass.Log($"Handle : {modelerProcess.Handle}");
StaticClass.Log($"Id : {modelerProcess.Id}");
StaticClass.Log($"PriorityClass : {modelerProcess.PriorityClass}");
StaticClass.Log($"Basepriority : {modelerProcess.BasePriority}");
StaticClass.Log($"PriorityBoostEnabled : {modelerProcess.PriorityBoostEnabled}");
StaticClass.Log($"Responding : {modelerProcess.Responding}");
StaticClass.Log($"ProcessorAffinity : {modelerProcess.ProcessorAffinity}");
StaticClass.Log($"HandleCount : {modelerProcess.HandleCount}");
StaticClass.Log($"MaxWorkingSet : {modelerProcess.MaxWorkingSet}");
StaticClass.Log($"MinWorkingSet : {modelerProcess.MinWorkingSet}");
StaticClass.Log($"NonpagedSystemMemorySize64 : {modelerProcess.NonpagedSystemMemorySize64}");
StaticClass.Log($"PeakVirtualMemorySize64 : {modelerProcess.PeakVirtualMemorySize64}");
StaticClass.Log($"PagedSystemMemorySize64 : {modelerProcess.PagedSystemMemorySize64}");
StaticClass.Log($"PrivateMemorySize64 : {modelerProcess.PrivateMemorySize64}");
StaticClass.Log($"VirtualMemorySize64 : {modelerProcess.VirtualMemorySize64}");
StaticClass.Log($"NonpagedSystemMemorySize64 : {modelerProcess.PagedMemorySize64}");
StaticClass.Log($"WorkingSet64 : {modelerProcess.WorkingSet64}");
StaticClass.Log($"PeakWorkingSet64 : {modelerProcess.PeakWorkingSet64}");
StaticClass.Log($"PrivilegedProcessorTime : {modelerProcess.PrivilegedProcessorTime}");
StaticClass.Log($"TotalProcessorTime : {modelerProcess.TotalProcessorTime}");
StaticClass.Log($"UserProcessorTime : {modelerProcess.UserProcessorTime}");
}
/// <summary>
/// Check if the Modeler has deposited the OverallResult entry in the job data.xml file
/// </summary>
/// <param name="dataXmlFileName"></param>
/// <returns></returns>
public bool OverallResultEntryCheck(string dataXmlFileName)
{
int numOfRetries = 0;
do
{
// Check for data.xml file to be ready
if (StaticClass.CheckFileReady(dataXmlFileName))
{
// Check if the OverallResult node exists
XmlDocument dataXmlDoc = new XmlDocument();
dataXmlDoc.Load(dataXmlFileName);
XmlNode OverallResult = dataXmlDoc.DocumentElement.SelectSingleNode("/Data/OverallResult/result");
if (OverallResult != null)
{
return(true);
}
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("Overall Result Entry Check") == true)
{
return(false);
}
Thread.Sleep(StaticClass.FILE_WAIT_DELAY);
}
}while (numOfRetries++ < StaticClass.NUM_RESULTS_ENTRY_RETRIES);
StaticClass.Log(string.Format("File {0} did not have Overall Result entry even after {1} retries at {2:HH:mm:ss.fff}",
dataXmlFileName, StaticClass.NUM_RESULTS_ENTRY_RETRIES, DateTime.Now));
return(false);
}
/// <summary>
/// The Add or Change of files callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string fullDirectory = e.FullPath;
string jobDirectory = fullDirectory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
string jobFile = jobDirectory.Substring(jobDirectory.LastIndexOf('\\') + 1);
string job = jobDirectory.Substring(0, jobDirectory.LastIndexOf('\\'));
// If Number of files is not complete
if (StaticClass.NumberOfInputFilesFound[job] < StaticClass.NumberOfInputFilesNeeded[job])
{
// Wait some for the file creation to complete
Thread.Sleep(StaticClass.FILE_RECEIVE_WAIT);
if (StaticClass.CheckFileReady(fullDirectory) == true)
{
// Increment the number of Input Buffer Job files found
StaticClass.NumberOfInputFilesFound[job]++;
StaticClass.Log(string.Format("Input File Watcher detected {0} for Job {1} file {2} of {3} at {4:HH:mm:ss.fff}",
jobFile, job, StaticClass.NumberOfInputFilesFound[job], StaticClass.NumberOfInputFilesNeeded[job], DateTime.Now));
// If Number of Input files is complete
if (StaticClass.NumberOfInputFilesFound[job] == StaticClass.NumberOfInputFilesNeeded[job])
{
StaticClass.Log(string.Format("\nInput File Watcher detected Job {0} complete set of {1} files at {2:HH:mm:ss.fff}",
job, StaticClass.NumberOfInputFilesNeeded[job], DateTime.Now));
// Signal the Run thread that the Input Buffer files were found
StaticClass.InputFileScanComplete[job] = true;
}
}
}
}
/// <summary>
/// The Add of files callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string fullDirectory = e.FullPath;
string jobDirectory = fullDirectory.Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
string jobFile = jobDirectory.Substring(jobDirectory.LastIndexOf('\\') + 1);
string job = jobDirectory.Substring(0, jobDirectory.LastIndexOf('\\'));
// If Number of files is not complete
if (StaticClass.NumberOfProcessingFilesFound[job] < StaticClass.NumberOfProcessingFilesNeeded[job])
{
// Increment the number of Processing Buffer Job files found
StaticClass.NumberOfProcessingFilesFound[job]++;
StaticClass.Log(string.Format("Processing File Watcher detected {0} for Job {1} file {2} of {3} at {4:HH:mm:ss.fff}",
jobFile, job, StaticClass.NumberOfProcessingFilesFound[job], StaticClass.NumberOfProcessingFilesNeeded[job], DateTime.Now));
// If Number of Processing files is complete
if (StaticClass.NumberOfProcessingFilesFound[job] == StaticClass.NumberOfProcessingFilesNeeded[job])
{
StaticClass.Log(string.Format("Processing File Watcher detected Job {0} complete set of {1} files at {2:HH:mm:ss.fff}",
job, StaticClass.NumberOfProcessingFilesNeeded[job], DateTime.Now));
// Short pause so that files from the network that are slow
Thread.Sleep(StaticClass.WAIT_FOR_FILES_TO_COMPLETE);
// Signal the Run thread that the Processing Buffer files were found
StaticClass.ProcessingFileScanComplete[job] = true;
}
}
}
// Define the event handlers.
/// <summary>
/// The directory created callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string jobDirectory = e.FullPath;
string job = jobDirectory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
StaticClass.Log(string.Format("\nInput Directory Watcher checking new Job {0} for Input Job list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
if (StaticClass.ShutDownPauseCheck("Directory Watcher OnCreated") == false)
{
Thread.Sleep(StaticClass.WAIT_FOR_FILES_TO_COMPLETE);
// Check directory contents complete
if (StaticClass.CheckDirectoryReady(jobDirectory) == true)
{
StaticClass.AddJobToList(job);
if ((StaticClass.IniData.DebugMode & (byte)DebugModeState.JOB_LIST) != 0)
{
StaticClass.DisplayJobList();
}
}
else
{
StaticClass.Logger.LogError("DirectoryWatcherThread Job {0} directory check failed at {1:HH:mm:ss.fff}",
job, DateTime.Now);
}
}
}
/// <summary>
/// Write to Log File
/// </summary>
/// <param name="text"></param>
public void WriteToLogFile(string text)
{
bool tooBig = false;
int MaxFileSize = StaticClass.LogFileSizeLimit * 1024 * 1024;
lock (FileLock)
{
using (FileStream stream = new FileStream(LogFileName, FileMode.Append))
{
using (StreamWriter writer = new StreamWriter(stream))
{
// Check file size before writing
if (stream.Position < MaxFileSize)
{
writer.WriteLine(text);
}
else
{
tooBig = true;
}
}
}
}
if (tooBig)
{
StaticClass.Log("\nProcess log.txt file too big. Reducing 10%...\n");
lock (FileLock)
{
// Remove old data from log file
using (MemoryStream memoryStream = new MemoryStream(StaticClass.LogFileSizeLimit))
{
using (FileStream stream = new FileStream(LogFileName, FileMode.Open, FileAccess.ReadWrite))
{
// Reduce size of log file by 10%
int sizeLimit = (int)(MaxFileSize * 0.9);
stream.Seek(-sizeLimit, SeekOrigin.End);
byte[] bytes = new byte[sizeLimit];
stream.Read(bytes, 0, sizeLimit);
memoryStream.Write(bytes, 0, sizeLimit);
memoryStream.Position = 0;
stream.SetLength(sizeLimit);
stream.Position = 0;
memoryStream.CopyTo(stream);
using (StreamWriter writer = new StreamWriter(stream))
{
writer.WriteLine(text);
}
}
}
}
}
}
public void RunInputBufferScan(string directory, string job, StatusMonitorData monitorData)
{
// Add initial entry to status list
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_INPUT, JobType.TIME_START);
// Monitor the Input Buffer job directory until it has the total number of consumed files
string inputBufferJobDir = StaticClass.IniData.InputDir;
int numberOfFilesNeeded = monitorData.NumFilesConsumed;
if (Directory.Exists(inputBufferJobDir))
{
string inputJobFileDir = inputBufferJobDir + @"\" + job;
// Register with the File Watcher class event and start its thread
InputFileWatcherThread inputFileWatch = new InputFileWatcherThread(inputJobFileDir, numberOfFilesNeeded);
if (inputFileWatch == null)
{
StaticClass.Logger.LogError("Job Run Thread inputFileWatch failed to instantiate");
}
inputFileWatch.ThreadProc();
// Wait for Input file scan to complete
do
{
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
Thread.Yield();
}while (StaticClass.InputFileScanComplete[job] == false);
StaticClass.Log(string.Format("Finished Input file scan for Job {0} at {1:HH:mm:ss.fff}",
inputJobFileDir, DateTime.Now));
// Add copying entry to status list
StaticClass.StatusDataEntry(job, JobStatus.COPYING_TO_PROCESSING, JobType.TIME_START);
// Move files from Input directory to the Processing directory, creating it first if needed
FileHandling.CopyFolderContents(inputJobFileDir, directory, true, true);
}
else
{
StaticClass.Logger.LogError("Could not find Input Buffer Directory");
throw new InvalidOperationException("Could not find Input Buffer Directory");
}
}
/// <summary>
/// Check for unfinished Input Buffer jobs
/// </summary>
public void RunInputJobsFound()
{
// Check if there are unfinished Input jobs waiting to run
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
int index = 0;
string job = string.Empty;
Task AddTask = Task.Run(() =>
{
if (StaticClass.InputJobsToRun.Count > 0)
{
index = StaticClass.InputJobsToRun.Count;
job = StaticClass.InputJobsToRun.Read(index);
StaticClass.InputJobsToRun.Delete(index);
StaticClass.Log(string.Format("\nInput Directory Watcher removed Job {0} from Input Job list index {1} at {2:HH:mm:ss.fff}",
job, index, DateTime.Now));
}
});
TimeSpan timeSpan = TimeSpan.FromMilliseconds(StaticClass.DELETE_TASK_DELAY);
if (!AddTask.Wait(timeSpan))
{
StaticClass.Logger.LogError("InputJobScanThread Delete Job {0} timed out at {1:HH:mm:ss.fff}", job, DateTime.Now);
}
if (job != string.Empty)
{
string directory = StaticClass.IniData.InputDir + @"\" + job;
StaticClass.Log(string.Format("Starting Input Job {0} at {1:HH:mm:ss.fff}", directory, DateTime.Now));
// Reset Input job file scan flag
StaticClass.InputFileScanComplete[job] = false;
// Start an Input Buffer Job
StartInputJob(directory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
}
/// <summary>
/// Display the Modeler Process information
/// </summary>
/// <param name=job"></param>
/// <param name="modelerProcess"></param>
public void DisplayProcessInfo(string job, Process modelerProcess)
{
if ((StaticClass.IniData.DebugMode & (byte)DebugModeState.MODELER) != 0)
{
// Display Modeler Executable information
StaticClass.Log($"\nJob {job} Modeler execution process data:");
StaticClass.Log($"ProcessName : {modelerProcess.ProcessName}");
StaticClass.Log($"StartTime : {modelerProcess.StartTime}");
StaticClass.Log($"MainWindowTitle : {modelerProcess.MainWindowTitle}");
StaticClass.Log($"MainModule : {modelerProcess.MainModule}");
StaticClass.Log($"StartInfo : {modelerProcess.StartInfo}");
StaticClass.Log($"GetType : {modelerProcess.GetType()}");
StaticClass.Log($"MainWindowHandle : {modelerProcess.MainWindowHandle}");
StaticClass.Log($"Handle : {modelerProcess.Handle}");
StaticClass.Log($"Id : {modelerProcess.Id}");
StaticClass.Log($"PriorityClass : {modelerProcess.PriorityClass}");
StaticClass.Log($"Basepriority : {modelerProcess.BasePriority}");
StaticClass.Log($"PriorityBoostEnabled : {modelerProcess.PriorityBoostEnabled}");
StaticClass.Log($"Responding : {modelerProcess.Responding}");
StaticClass.Log($"ProcessorAffinity : {modelerProcess.ProcessorAffinity}");
StaticClass.Log($"HandleCount : {modelerProcess.HandleCount}");
StaticClass.Log($"MaxWorkingSet : {modelerProcess.MaxWorkingSet}");
StaticClass.Log($"MinWorkingSet : {modelerProcess.MinWorkingSet}");
StaticClass.Log($"NonpagedSystemMemorySize64 : {modelerProcess.NonpagedSystemMemorySize64}");
StaticClass.Log($"PeakVirtualMemorySize64 : {modelerProcess.PeakVirtualMemorySize64}");
StaticClass.Log($"PagedSystemMemorySize64 : {modelerProcess.PagedSystemMemorySize64}");
StaticClass.Log($"PrivateMemorySize64 : {modelerProcess.PrivateMemorySize64}");
StaticClass.Log($"VirtualMemorySize64 : {modelerProcess.VirtualMemorySize64}");
StaticClass.Log($"NonpagedSystemMemorySize64 : {modelerProcess.PagedMemorySize64}");
StaticClass.Log($"WorkingSet64 : {modelerProcess.WorkingSet64}");
StaticClass.Log($"PeakWorkingSet64 : {modelerProcess.PeakWorkingSet64}");
StaticClass.Log($"PrivilegedProcessorTime : {modelerProcess.PrivilegedProcessorTime}");
StaticClass.Log($"TotalProcessorTime : {modelerProcess.TotalProcessorTime}");
StaticClass.Log($"UserProcessorTime : {modelerProcess.UserProcessorTime}");
}
else
{
StaticClass.Log(string.Format("\nJob {0} Modeler execution started with base priority {1} at {2:HH:mm:ss.fff}\n",
job, modelerProcess.BasePriority, modelerProcess.StartTime));
}
}
/// <summary>
/// Start the Monitor process by starting the new jobs scan thread
/// </summary>
public void StartMonitorProcess()
{
StaticClass.ShutdownFlag = false;
// Check for pause state and reset it if the Start button is pressed when in Pause mode
if (StaticClass.PauseFlag == true)
{
StaticClass.Log(string.Format("Taking the system out of Pause Mode at {0:HH:mm:ss.fff}", DateTime.Now));
StaticClass.PauseFlag = false;
}
else
{
// Start Modeler job Processing thread
InputJobsScanThread newJobsScanThread = new InputJobsScanThread();
if (newJobsScanThread == null)
{
StaticClass.Logger.LogError("StartMonitorProcess newJobsScanThread failed to instantiate");
}
newJobsScanThread.ThreadProc();
}
}
/// <summary>
/// Deletes a directory after deleting files inside
/// </summary>
/// <param name="targetDirectory"></param>
public static void DeleteDirectory(string targetDirectory)
{
// First delete all files in target directory
string[] files = Directory.GetFiles(targetDirectory);
foreach (string file in files)
{
lock (FileLock)
{
File.SetAttributes(file, FileAttributes.Normal);
File.Delete(file);
}
}
lock (FileLock)
{
// Then delete the directory
File.SetAttributes(targetDirectory, FileAttributes.Normal);
Directory.Delete(targetDirectory, false);
}
StaticClass.Log(string.Format("Deleted Directory {0}", targetDirectory));
}
// Define the event handlers.
/// <summary>
/// The directory created callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string jobDirectory = e.FullPath;
string job = jobDirectory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
int index = 0;
// Check Input Buffer Job directory to see if it is ready if adding the first jobs
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
StaticClass.Log(string.Format("\nInput Directory Watcher checking new Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Wait some for the directory creation and file copies to complete
Thread.Sleep(StaticClass.DIRECTORY_RECEIVE_WAIT);
StaticClass.CheckJobDirectoryComplete(jobDirectory);
}
// Do Shutdown Pause check
if (StaticClass.ShutDownPauseCheck("Directory Watcher OnCreated") == false)
{
// Loop shutdown/Pause check
Task AddTask = Task.Run(() =>
{
index = StaticClass.InputJobsToRun.Count + 1;
StaticClass.InputJobsToRun.Add(index, job);
StaticClass.Log(string.Format("Input Directory Watcher added new Job {0} to Input Job list index {1} at {2:HH:mm:ss.fff}",
job, index, DateTime.Now));
});
TimeSpan timeSpan = TimeSpan.FromMilliseconds(StaticClass.ADD_JOB_DELAY);
if (!AddTask.Wait(timeSpan))
{
StaticClass.Logger.LogError("DirectoryWatcherThread Add Job {0} timed out at {1:HH:mm:ss.fff}", job, DateTime.Now);
}
}
}
/// <summary>
/// Start a Processing directory job
/// </summary>
/// <param name="directory"></param>
public void StartProcessingJob(string directory)
{
// Delete the data.xml file in the Processing directory if found
string dataXmlFile = directory + @"\" + "data.xml";
if (File.Exists(dataXmlFile))
{
File.Delete(dataXmlFile);
}
// Get data found in job Xml file
JobXmlData jobXmlData = StaticClass.GetJobXmlFileInfo(directory, DirectoryScanType.PROCESSING_BUFFER);
if (jobXmlData == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread GetJobXmlData failed");
}
// Display job xml Data found
StaticClass.Log("Processing Job : " + jobXmlData.Job);
StaticClass.Log("Processing Job Directory : " + jobXmlData.JobDirectory);
StaticClass.Log("Processing Job Serial Number : " + jobXmlData.JobSerialNumber);
StaticClass.Log("Processing Job Time Stamp : " + jobXmlData.TimeStamp);
StaticClass.Log("Processing Job Xml File : " + jobXmlData.XmlFileName);
StaticClass.Log(string.Format("Starting Processing directory Job {0} Executing Slot {1} at {2:HH:mm:ss.fff}",
jobXmlData.Job, StaticClass.NumberOfJobsExecuting + 1, DateTime.Now));
// Create a thread to run the job, and then start the thread
JobRunThread jobRunThread = new JobRunThread(jobXmlData, DirectoryScanType.PROCESSING_BUFFER);
if (jobRunThread == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread jobRunThread failed to instantiate");
}
jobRunThread.ThreadProc();
}
/// <summary>
/// Execute the Modeler command line
/// </summary>
/// <param name="job"></param>
/// <returns></returns>
public Process ExecuteCommand(string job)
{
ProcessStartInfo startInfo = new ProcessStartInfo(Executable)
{
Arguments = string.Format("{0} {1} {2}", ProcessingDirParam, StartPortParam, CpuCoresParam),
UseShellExecute = true,
WorkingDirectory = ProcessingDirectory,
WindowStyle = ProcessWindowStyle.Minimized
};
// Check object
if (startInfo == null)
{
StaticClass.Logger.LogError("CommandLineGenerator startInfo failed to instantiate");
}
// Start the Modeler process window
Process ModelerProcess = Process.Start(startInfo);
if (ModelerProcess == null)
{
StaticClass.Logger.LogError("CommandLineGeneratorThread ModelerProcess failes to instantiate");
}
// Set Process Priority
ModelerProcess.PriorityClass = ProcessPriorityClass.Normal;
// Store process handle to use for stopping
StaticClass.ProcessHandles[job] = ModelerProcess;
// Give the Modeler time so you can read the Main Window Title parameter for display confirmation
Thread.Sleep(StaticClass.DISPLAY_PROCESS_TITLE_WAIT);
StaticClass.Log(string.Format("{0} {1}", ModelerProcess.MainWindowTitle, ModelerProcess.StartInfo.Arguments));
return(ModelerProcess);
}
/// <summary>
/// Check for unfinished processing jobs after one completes
/// </summary>
public void RunUnfinishedProcessingJobs()
{
// Start Processing jobs currently waiting
foreach (string job in ProcessingJobsToRun.Reverse <string>())
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
string directory = StaticClass.IniData.ProcessingDir + @"\" + job;
StaticClass.Log(string.Format("\nStarting Processing Job {0} at {1:HH:mm:ss.fff}", directory, DateTime.Now));
// Remove job run from Processing Job list
lock (RemoveLock)
{
if (ProcessingJobsToRun.Remove(job) == false)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread failed to remove Job {0} from Processing Job list", job);
}
}
StaticClass.Log(string.Format("Processing Directory Watcher removed Job {0} from Job list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Reset Processing job and file scan flags
StaticClass.ProcessingFileScanComplete[job] = false;
StaticClass.ProcessingJobScanComplete[job] = false;
StaticClass.JobShutdownFlag[job] = false;
// Start a Processing Buffer Job
StartProcessingJob(directory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
}
public void WatchFiles(string directory)
{
// Get job name from directory name
string job = directory.Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
// Check to see if the directory is already full
if (StaticClass.NumberOfProcessingFilesFound[job] == StaticClass.NumberOfProcessingFilesNeeded[job])
{
StaticClass.Log(string.Format("Processing File Watcher detected Job {0} has complete file set at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Signal the Run thread that the Processing files were found
StaticClass.ProcessingFileScanComplete[job] = true;
return;
}
// Create a new FileSystemWatcher and set its properties
using (FileSystemWatcher watcher = new FileSystemWatcher())
{
if (watcher == null)
{
StaticClass.Logger.LogError("ProcessingFileWatcherThread watcher failed to instantiate");
}
// Watch for file changes in the watched directory
watcher.NotifyFilter =
NotifyFilters.FileName |
NotifyFilters.CreationTime |
NotifyFilters.LastAccess;
// Set the Path to scan for files
watcher.Path = directory;
// Watch for any file to get directory changes
watcher.Filter = "*.*";
// Add event handlers
watcher.Created += OnCreated;
watcher.Changed += OnChanged;
// Begin watching for file changes to Processing job directory
watcher.EnableRaisingEvents = true;
StaticClass.Log(string.Format("Processing File Watcher watching {0} at {1:HH:mm:ss.fff}",
directory, DateTime.Now));
// Wait for Processing file scan to Complete with a full set of job output files
do
{
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("ProcessingWatchFiles") == true)
{
return;
}
Thread.Yield();
}while ((StaticClass.ProcessingFileScanComplete[job] == false) || (StaticClass.TcpIpScanComplete[job] == false));
// set the Processing of a Job scan complete flag
StaticClass.ProcessingJobScanComplete[job] = true;
}
}
/// <summary>
/// Get the Config.ini file data from the working directory
/// </summary>
public void GetIniFileData()
{
// Check that Config.ini file exists
string IniFileName = "Config.ini";
if (File.Exists(IniFileName) == false)
{
StaticClass.Logger.LogCritical("Missing Config.ini file");
throw new InvalidOperationException("Config.ini file does not exist in the Handler directory");
}
IniFileHandler IniParser = new IniFileHandler(IniFileName);
StaticClass.IniData.IniFileName = IniFileName;
StaticClass.IniData.InputDir = IniParser.Read("Paths", "Input");
StaticClass.IniData.ProcessingDir = IniParser.Read("Paths", "Processing");
StaticClass.IniData.RepositoryDir = IniParser.Read("Paths", "Repository");
StaticClass.IniData.FinishedDir = IniParser.Read("Paths", "Finished");
StaticClass.IniData.ErrorDir = IniParser.Read("Paths", "Error");
StaticClass.IniData.ModelerRootDir = IniParser.Read("Paths", "ModelerRootDir");
StaticClass.IniData.CPUCores = int.Parse(IniParser.Read("Process", "CPUCores"));
StaticClass.IniData.ExecutionLimit = int.Parse(IniParser.Read("Process", "ExecutionLimit"));
StaticClass.IniData.StartPort = int.Parse(IniParser.Read("Process", "StartPort"));
StaticClass.IniData.StatusLogFile = IniParser.Read("Process", "StatusLogFile");
StaticClass.IniData.ProcessLogFile = IniParser.Read("Process", "ProcessLogFile");
string scanWaitTime = IniParser.Read("Process", "ScanWaitTime");
StaticClass.IniData.ScanWaitTime = int.Parse(scanWaitTime.Substring(0, scanWaitTime.IndexOf("#")));
string timeLimitString = IniParser.Read("Process", "MaxJobTimeLimit");
StaticClass.IniData.MaxJobTimeLimit = double.Parse(timeLimitString.Substring(0, timeLimitString.IndexOf("#")));
string logFileHistoryLimit = IniParser.Read("Process", "LogFileHistoryLimit");
StaticClass.IniData.LogFileHistoryLimit = int.Parse(logFileHistoryLimit.Substring(0, logFileHistoryLimit.IndexOf("#")));
string inputBufferTimeLimit = IniParser.Read("Process", "InputbufferTimeLimit");
StaticClass.IniData.InputBufferTimeLimit = int.Parse(inputBufferTimeLimit.Substring(0, inputBufferTimeLimit.IndexOf("#")));
string logFileMaxSize = IniParser.Read("Process", "logFileMaxSize");
StaticClass.IniData.LogFileMaxSize = int.Parse(logFileMaxSize.Substring(0, logFileMaxSize.IndexOf("#")));
// Set the static class data needed for global use
StaticClass.ScanWaitTime = StaticClass.IniData.ScanWaitTime;
StaticClass.LogFileSizeLimit = StaticClass.IniData.LogFileMaxSize;
StaticClass.MaxJobTimeLimitSeconds = StaticClass.IniData.MaxJobTimeLimit * 60.0 * 60.0;
// Set the file logging object handle only once here
LoggingToFile loggingToFile = new LoggingToFile(StaticClass.IniData.ProcessLogFile);
StaticClass.FileLoggerObject = loggingToFile;
// Output the Data.ini informatino found
StaticClass.Log($"\nConfig.ini data found:\n");
StaticClass.Log($"Input Dir : {StaticClass.IniData.InputDir}");
StaticClass.Log($"Processing Dir : {StaticClass.IniData.ProcessingDir}");
StaticClass.Log($"Repository Dir : {StaticClass.IniData.RepositoryDir}");
StaticClass.Log($"Finished Dir : {StaticClass.IniData.FinishedDir}");
StaticClass.Log($"Error Dir : {StaticClass.IniData.ErrorDir}");
StaticClass.Log($"Modeler Root Dir : {StaticClass.IniData.ModelerRootDir}");
StaticClass.Log($"Status Log File : {StaticClass.IniData.StatusLogFile}");
StaticClass.Log($"Process Log File : {StaticClass.IniData.ProcessLogFile}");
StaticClass.Log($"CPU Cores : {StaticClass.IniData.CPUCores} Cores");
StaticClass.Log($"Execution Limit : {StaticClass.IniData.ExecutionLimit} Jobs");
StaticClass.Log($"Start Port : {StaticClass.IniData.StartPort}");
StaticClass.Log($"Scan Wait Time : {StaticClass.IniData.ScanWaitTime} Miliseconds");
StaticClass.Log($"Max Job Time Limit : {StaticClass.IniData.MaxJobTimeLimit} Hours");
StaticClass.Log($"Log File History Limit : {StaticClass.IniData.LogFileHistoryLimit} Days");
StaticClass.Log($"Log File Max Size : {StaticClass.IniData.LogFileMaxSize} Megabytes");
}
/// <summary>
/// Pause the Monitor System
/// </summary>
public void PauseMonitor()
{
StaticClass.Log(string.Format("Putting the system into Pause Mode at {0:HH:mm:ss.fff}", DateTime.Now));
StaticClass.PauseFlag = true;
}
/// <summary>
/// Method to scan for unfinished jobs in the Processing Buffer
/// </summary>
public void CheckForUnfinishedProcessingJobs()
{
// Register with the Processing Buffer Jobs completion event and start its thread
DirectoryInfo processingDirectoryInfo = new DirectoryInfo(StaticClass.IniData.ProcessingDir);
if (processingDirectoryInfo == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread processingDirectoryInfo failed to instantiate");
}
// Get the current list of directories from the Job Processing Buffer
List <DirectoryInfo> processingDirectoryInfoList = processingDirectoryInfo.EnumerateDirectories().ToList();
if (processingDirectoryInfoList == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread processingDirectoryInfoList failed to instantiate");
}
StaticClass.Log("\nChecking for unfinished Processing Jobs...");
if (processingDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nUnfinished Processing Jobs waiting...");
}
else
{
StaticClass.Log("\nNo unfinished Processing Jobs waiting...");
}
// Start the jobs in the directory list found for the Processing Buffer
foreach (DirectoryInfo dirInfo in processingDirectoryInfoList.Reverse <DirectoryInfo>())
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
string directory = dirInfo.ToString();
string job = directory.ToString().Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
StaticClass.Log(string.Format("Starting Processing Job {0} at {1:HH:mm:ss.fff}", directory, DateTime.Now));
// Remove job run from Processing Job list
lock (RemoveLock)
{
if (processingDirectoryInfoList.Remove(dirInfo) == false)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread failed to remove Job {0} from Processing Job list", job);
}
}
StaticClass.Log(string.Format("Processing Directory Watcher removed Job {0} from directory list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Reset Processing file scan flag
StaticClass.ProcessingFileScanComplete[job] = false;
// Start a Processing Buffer Job
StartProcessingJob(directory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
if (processingDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nMore unfinished Processing Jobs then Execution Slots available...");
}
// Sort the Processing Buffer directory list by older dates first
processingDirectoryInfoList.Sort((x, y) => - x.LastAccessTime.CompareTo(y.LastAccessTime));
// Add the jobs in the directory list to the Processing Buffer Jobs to run list
foreach (DirectoryInfo dirInfo in processingDirectoryInfoList)
{
string directory = dirInfo.ToString();
string job = directory.Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
ProcessingJobsToRun.Add(job);
int index = ProcessingJobsToRun.IndexOf(job);
StaticClass.Log(string.Format("Unfinished Processing jobs check added new Job {0} to Processing Job List index {1} at {2:HH:mm:ss.fff}",
job, index, DateTime.Now));
}
// Clear the Directory Info List after done with it
processingDirectoryInfoList.Clear();
// Run check loop until all unfinished Processing jobs are complete
do
{
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("Processing Jobs Scan Thread") == true)
{
return;
}
// Run any unfinished Processing jobs
RunUnfinishedProcessingJobs();
Thread.Yield();
}while (ProcessingJobsToRun.Count > 0);
// Scan of the Processing Buffer jobs is complete
StaticClass.UnfinishedProcessingJobsScanComplete = true;
}
/// <summary>
/// Get the data from the Job xml file
/// </summary>
/// <param name="jobXmlData"></param>
/// <param name="monitorData"></param>
/// <returns></returns>
public StatusMonitorData GetJobXmlData(JobXmlData jobXmlData, StatusMonitorData monitorData)
{
// Wait for Job xml file to be ready
string jobXmlFileName = jobXmlData.JobDirectory + @"\" + jobXmlData.XmlFileName;
if (StaticClass.CheckFileReady(jobXmlFileName))
{
// Read Job xml file and get the top node
XmlDocument jobXmlDoc = new XmlDocument();
jobXmlDoc.Load(jobXmlFileName);
XmlElement root = jobXmlDoc.DocumentElement;
string TopNode = root.LocalName;
// Get nodes for the number of files and names of files to transfer from Job .xml file
XmlNode UnitNumberdNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/listitem/value");
XmlNode ConsumedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Consumed");
XmlNode ProducedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Produced");
XmlNode TransferedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Transfered");
XmlNode ModelerNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Modeler");
// Assign then increment port number for this Job
monitorData.JobPortNumber = StaticClass.IniData.StartPort + StaticClass.JobPortIndex++;
// Get the modeler and number of files to transfer
int NumFilesToTransfer = 0;
monitorData.UnitNumber = UnitNumberdNode.InnerText;
monitorData.Modeler = ModelerNode.InnerText;
monitorData.NumFilesConsumed = Convert.ToInt32(ConsumedNode.InnerText);
monitorData.NumFilesProduced = Convert.ToInt32(ProducedNode.InnerText);
if (TransferedNode != null)
{
NumFilesToTransfer = Convert.ToInt32(TransferedNode.InnerText);
}
monitorData.NumFilesToTransfer = NumFilesToTransfer;
// Get the modeler and number of files to transfer
StaticClass.Log($"Unit Number : {monitorData.UnitNumber}");
StaticClass.Log($"Modeler : {monitorData.Modeler}");
StaticClass.Log($"Num Files Consumed : {monitorData.NumFilesConsumed}");
StaticClass.Log($"Num Files Produced : {monitorData.NumFilesProduced}");
StaticClass.Log($"Num Files To Transfer : {monitorData.NumFilesToTransfer}");
StaticClass.Log($"Job Port Number : {monitorData.JobPortNumber}");
// Create the Transfered file list from the Xml file entries
monitorData.TransferedFileList = new List <string>(NumFilesToTransfer);
for (int i = 1; i < NumFilesToTransfer + 1; i++)
{
string transferFileNodeName = ("/" + TopNode + "/FileConfiguration/Transfered" + i.ToString());
XmlNode TransferedFileXml = jobXmlDoc.DocumentElement.SelectSingleNode(transferFileNodeName);
monitorData.TransferedFileList.Add(TransferedFileXml.InnerText);
StaticClass.Log(string.Format("Transfer File{0} : {1}", i, TransferedFileXml.InnerText));
}
}
else
{
StaticClass.Logger.LogError("File {0} is not available at {1:HH:mm:ss.fff}\n", jobXmlFileName, DateTime.Now);
}
return(monitorData);
}
/// <summary>
/// Run a job from Input or Processing Buffers
/// </summary>
/// <param name="dirScanType"></param>
/// <param name="jobXmlData"></param>
public void RunJob(JobXmlData jobXmlData, DirectoryScanType dirScanType)
{
// Increment number of Jobs executing in only one place!
StaticClass.NumberOfJobsExecuting++;
// Create the Job Run common strings
string job = jobXmlData.Job;
string xmlJobDirectory = jobXmlData.JobDirectory;
string processingBufferDirectory = StaticClass.IniData.ProcessingDir;
string processingBufferJobDir = processingBufferDirectory + @"\" + job;
// Set the job start time
StaticClass.JobStartTime[job] = DateTime.Now;
// Create new status monitor data and fill it in with the job xml data
StatusMonitorData monitorData = new StatusMonitorData
{
Job = job,
JobDirectory = xmlJobDirectory,
StartTime = DateTime.Now,
JobSerialNumber = jobXmlData.JobSerialNumber,
TimeStamp = jobXmlData.TimeStamp,
XmlFileName = jobXmlData.XmlFileName
};
// Add initial entry to status list
StaticClass.StatusDataEntry(job, JobStatus.JOB_STARTED, JobType.TIME_RECEIVED);
// Get the Job xml data
monitorData = GetJobXmlData(jobXmlData, monitorData);
// If this job comes from the Input directory, run the Input job check and start job
if (dirScanType == DirectoryScanType.INPUT_BUFFER)
{
RunInputBufferScan(processingBufferJobDir, job, monitorData);
}
// If the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
// Add entry to status list
StaticClass.StatusDataEntry(job, JobStatus.EXECUTING, JobType.TIME_START);
StaticClass.Log(string.Format("Starting Job {0} with Modeler {1} on Port {2} with {3} CPU's at {4:HH:mm:ss.fff}",
job, monitorData.Modeler, monitorData.JobPortNumber, StaticClass.IniData.CPUCores, DateTime.Now));
// Execute Modeler using the command line generator
string executable = StaticClass.IniData.ModelerRootDir + @"\" + monitorData.Modeler + @"\" + monitorData.Modeler + ".exe";
string processingBuffer = processingBufferJobDir;
int port = monitorData.JobPortNumber;
int cpuCores = StaticClass.IniData.CPUCores;
CommandLineGenerator cmdLineGenerator = new CommandLineGenerator(
executable, processingBuffer, port, cpuCores);
if (cmdLineGenerator == null)
{
StaticClass.Logger.LogError("JobRunThread cmdLineGenerator failed to instantiate");
}
Process modelerProcess = cmdLineGenerator.ExecuteCommand(job);
// Register with the Processing File Watcher class and start its thread
ProcessingFileWatcherThread processingFileWatcher = new ProcessingFileWatcherThread(processingBufferJobDir, monitorData);
if (processingFileWatcher == null)
{
StaticClass.Logger.LogError("JobRunThread ProcessingFileWatch failed to instantiate");
}
processingFileWatcher.ThreadProc();
// Start the TCP/IP Communications thread before checking for Processing job files
TcpIpListenThread tcpIpThread = new TcpIpListenThread(monitorData);
if (tcpIpThread == null)
{
StaticClass.Logger.LogError("ProcessingFileWatcherThread tcpIpThread failed to instantiate");
}
tcpIpThread.ThreadProc();
// Add entry to status list
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_PROCESSING, JobType.TIME_START);
// Wait 45 seconds for Modeler to get started before reading it's information
Thread.Sleep(StaticClass.DISPLAY_PROCESS_DATA_WAIT);
// Display the Modeler Process information
DisplayProcessInfo(job, modelerProcess);
// Wait for the Processing job scan complete or shut down
do
{
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
Thread.Yield();
}while ((StaticClass.ProcessingJobScanComplete[job] == false) && (StaticClass.JobShutdownFlag[job] == false));
// Wait to make sure the data.xml is done being handled
Thread.Sleep(StaticClass.POST_PROCESS_WAIT);
if (StaticClass.JobShutdownFlag[job] == false)
{
// Wait for the data.xml file to contain a result
string dataXmlFileName = processingBufferJobDir + @"\" + "data.xml";
if (OverallResultEntryCheck(dataXmlFileName) == true)
{
StaticClass.Log(string.Format("Overall Results check confirmed for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
}
else
{
StaticClass.Log(string.Format("Overall Results check failed for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
}
}
// Make sure Modeler Process is stopped
if (StaticClass.ProcessHandles[job].HasExited == false)
{
StaticClass.Log(string.Format("Shutting down Modeler Executable for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
StaticClass.ProcessHandles[job].Kill();
StaticClass.ProcessHandles.Remove(job);
// Wait for Process to end
Thread.Sleep(StaticClass.SHUTDOWN_PROCESS_WAIT);
}
// Run the Job Complete handler
RunJobFileProcessing(job, monitorData);
// Add entry to status list
StaticClass.StatusDataEntry(job, JobStatus.COMPLETE, JobType.TIME_COMPLETE);
// Show Job Complete message
TimeSpan timeSpan = DateTime.Now - StaticClass.JobStartTime[job];
StaticClass.Log(string.Format("Job {0} Complete taking {1:hh\\:mm\\:ss}. Decrementing Job count to {2} at {3:HH:mm:ss.fff}",
job, timeSpan, StaticClass.NumberOfJobsExecuting - 1, DateTime.Now));
// Decrement the number of Jobs executing in one place!
StaticClass.NumberOfJobsExecuting--;
}
public void WatchFiles(string directory)
{
// Get job name from directory name
string job = directory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
if (StaticClass.NumberOfInputFilesFound[job] == StaticClass.NumberOfInputFilesNeeded[job])
{
// Exiting thread message
StaticClass.Log(string.Format("Input File Watcher completed the scan for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Signal the Run thread that the Input files were found
StaticClass.InputFileScanComplete[job] = true;
return;
}
// Create a new FileSystemWatcher and set its properties
using (FileSystemWatcher watcher = new FileSystemWatcher())
{
if (watcher == null)
{
StaticClass.Logger.LogError("InputFileWatcherThread watcher failed to instantiate");
}
// Watch for file changes in the watched directory
watcher.NotifyFilter =
NotifyFilters.FileName |
NotifyFilters.CreationTime |
NotifyFilters.LastAccess;
// Set the Path to scan for files
watcher.Path = directory;
// Watch for any file to get directory changes
watcher.Filter = "*.*";
// Add event handlers
watcher.Created += OnCreated;
watcher.Changed += OnChanged;
// Begin watching for changes to Input directory
watcher.EnableRaisingEvents = true;
StaticClass.Log(string.Format("Input File Watcher watching {0} at {1:HH:mm:ss.fff}",
directory, DateTime.Now));
// Wait for Input file scan to Complete with a full set of job output files
do
{
if (StaticClass.ShutDownPauseCheck("InputFileWatcherThread") == true)
{
return;
}
Thread.Yield();
}while (StaticClass.InputFileScanComplete[job] == false);
// Signal the Input Job Complete flag for the Job
StaticClass.InputJobScanComplete[job] = true;
// Exiting thread message
StaticClass.Log(string.Format("Input File Watcher completed scan for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
}
}
/// <summary>
/// CopyFolderContents - Copy files and folders from source to destination and optionally remove source files/folders
/// </summary>
/// <param name="sourcePath"></param>
/// <param name="destinationPath"></param>
/// <param name="removeSource"></param>
/// <param name="overwrite"></param>
public static void CopyFolderContents(string sourcePath, string destinationPath,
bool removeSource = false, bool overwrite = false)
{
DirectoryInfo sourceDI = new DirectoryInfo(sourcePath);
if (sourceDI == null)
{
StaticClass.Logger.LogError("FileHandling sourceDI failed to instantiate");
}
DirectoryInfo destinationDI = new DirectoryInfo(destinationPath);
if (destinationDI == null)
{
StaticClass.Logger.LogError("FileHandling destinationDI failed to instantiate");
}
// If the destination directory does not exist, create it
if (!destinationDI.Exists)
{
destinationDI.Create();
}
// Copy files one by one
FileInfo[] sourceFiles = sourceDI.GetFiles();
foreach (FileInfo sourceFile in sourceFiles)
{
// This is the destination folder plus the new filename
FileInfo destFile = new FileInfo(Path.Combine(destinationDI.FullName, sourceFile.Name));
if (destFile == null)
{
StaticClass.Log("FileHandling destFile failed to instantiate");
}
// Delete the destination file if overwrite is true
if (destFile.Exists && overwrite)
{
lock (FileLock)
{
File.SetAttributes(destFile.FullName, FileAttributes.Normal);
destFile.Delete();
}
}
sourceFile.CopyTo(Path.Combine(destinationDI.FullName, sourceFile.Name));
// Delete the source file if removeSource is true
if (removeSource)
{
lock (FileLock)
{
File.SetAttributes(sourceFile.FullName, FileAttributes.Normal);
sourceFile.Delete();
}
}
}
// Delete the source directory if removeSource is true
if (removeSource)
{
lock (FileLock)
{
DirectoryInfo dirInfo = new DirectoryInfo(sourceDI.FullName);
dirInfo.Attributes |= FileAttributes.Normal;
sourceDI.Delete();
}
}
StaticClass.Log(string.Format("Copied Directory {0} -> {1}", sourcePath, destinationPath));
}
/// <summary>
/// Check for new Input Buffer jobs
/// </summary>
public void RunInputJobsFound()
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
if (StaticClass.GetTotalNumberOfJobs() > 0)
{
string job = StaticClass.GetJobFromList(StaticClass.CurrentJobIndex);
if (job != string.Empty)
{
StaticClass.Log(string.Format("Input Job handler got Job {0} from list index {1} at {2:HH:mm:ss.fff}",
job, StaticClass.CurrentJobIndex, DateTime.Now));
// Check for complete jobs and run them first
string jobDirectory = StaticClass.IniData.InputDir + @"\" + job;
if (StaticClass.CheckIfJobFilesComplete(jobDirectory) == true)
{
StaticClass.Log(string.Format("Starting Input Job {0} index {1} at {2:HH:mm:ss.fff}",
jobDirectory, StaticClass.CurrentJobIndex, DateTime.Now));
// Start the new completely ready Job
StartInputJob(jobDirectory);
// Delete the job from the list and display the new list
StaticClass.DeleteJobFromList(StaticClass.CurrentJobIndex);
}
else // Partial Job handling
{
// Skip Partial Job if there are more in the list
int lastIndex = StaticClass.GetLastIndex();
if (StaticClass.CurrentJobIndex < lastIndex)
{
StaticClass.Log(string.Format("Input Job scanner skipping Job {0} index {1} as not ready at {2:HH:mm:ss.fff}",
job, StaticClass.CurrentJobIndex, DateTime.Now));
StaticClass.CurrentJobIndex++;
}
else // Run last job in list
{
// Wait for the job xml file
JobXmlData jobXmlData = StaticClass.GetJobXmlFileInfo(jobDirectory, DirectoryScanType.INPUT_BUFFER);
if (jobXmlData.XmlFileName != string.Empty)
{
StaticClass.Log(string.Format("Starting Partial Input Job {0} index {1} at {2:HH:mm:ss.fff}",
jobDirectory, StaticClass.CurrentJobIndex, DateTime.Now));
// Start the new partial job
StartInputJob(jobDirectory);
// Delete the job from the list
StaticClass.DeleteJobFromList(StaticClass.CurrentJobIndex);
}
// If there is a skipped partial job, start it next by setting index to previous one
int previousJobIndex = StaticClass.GetPreviousIndex();
if ((previousJobIndex > 0) && (StaticClass.CurrentJobIndex > previousJobIndex))
{
StaticClass.CurrentJobIndex = previousJobIndex;
}
}
}
}
}
else
{
if (CurrentJobIndex != StaticClass.CurrentJobIndex)
{
StaticClass.Log(string.Format("Input Jobs Scanner Resetting Current Job Index from {0} to {1} at {2:HH:mm:ss.fff}",
StaticClass.CurrentJobIndex, 1, DateTime.Now));
CurrentJobIndex = StaticClass.CurrentJobIndex = 1;
}
}
}
}
/// <summary>
/// Method to scan for unfinished jobs in the Input Buffer
/// </summary>
public void CheckForUnfinishedInputJobs()
{
// Start the Directory Watcher class to scan for new Input Buffer jobs
DirectoryWatcherThread dirWatch = new DirectoryWatcherThread();
if (dirWatch == null)
{
StaticClass.Logger.LogError("InputJobsScanThread dirWatch failed to instantiate");
}
dirWatch.ThreadProc();
// Register with the Processing Buffer Jobs check completion event and start its thread
ProcessingJobsScanThread unfinishedProcessingJobs = new ProcessingJobsScanThread();
if (unfinishedProcessingJobs == null)
{
StaticClass.Logger.LogError("InputJobsScanThread currentProcessingJobs failed to instantiate");
}
unfinishedProcessingJobs.ThreadProc();
// Wait while scanning for unfinished Processing jobs
do
{
if (StaticClass.ShutDownPauseCheck("CheckForUnfinishedInputJobs") == true)
{
return;
}
Thread.Yield();
}while (StaticClass.UnfinishedProcessingJobsScanComplete == false);
StaticClass.Log("\nChecking for unfinished Input Jobs...");
DirectoryInfo InputDirectoryInfo = new DirectoryInfo(StaticClass.IniData.InputDir);
if (InputDirectoryInfo == null)
{
StaticClass.Logger.LogError("InputJobsScanThread InputDirectoryInfo failed to instantiate");
}
// Get the current list of directories from the Input Buffer
List <DirectoryInfo> inputDirectoryInfoList = InputDirectoryInfo.EnumerateDirectories().ToList();
if (inputDirectoryInfoList == null)
{
StaticClass.Logger.LogError("InputJobsScanThread inputDirectoryInfoList failed to instantiate");
}
if (inputDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nUnfinished Input Jobs waiting...");
}
else
{
StaticClass.Log("\nNo unfinished Input Jobs found...");
}
// Start the jobs in the directory list found for the Input Buffer
foreach (DirectoryInfo dirInfo in inputDirectoryInfoList.Reverse <DirectoryInfo>())
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
string jobDirectory = dirInfo.ToString();
string job = jobDirectory.ToString().Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
// Check if the directory has a full set of Job files
if (StaticClass.CheckIfJobFilesComplete(jobDirectory) == true)
{
StaticClass.Log(string.Format("Starting Input Job {0} at {1:HH:mm:ss.fff}", jobDirectory, DateTime.Now));
// Remove job run from Input Job directory list
lock (RemoveLock)
{
if (inputDirectoryInfoList.Remove(dirInfo) == false)
{
StaticClass.Logger.LogError("InputJobsScanThread failed to remove Job {0} from Input Job list", job);
}
}
StaticClass.Log(string.Format("Input Directory Watcher removed Job {0} from Input Directory Jobs list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Reset Input file scan flag
StaticClass.InputFileScanComplete[job] = false;
// Start an Input Buffer Job
StartInputJob(jobDirectory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
}
if (inputDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nMore unfinished Input Jobs then Execution Slots available...");
// Sort the Input Buffer directory list by older dates first
inputDirectoryInfoList.Sort((x, y) => - x.LastAccessTime.CompareTo(y.LastAccessTime));
// Add the jobs in the directory list to the Input Buffer Jobs to run list
foreach (DirectoryInfo dirInfo in inputDirectoryInfoList)
{
string directory = dirInfo.ToString();
string job = directory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
StaticClass.AddJobToList(job);
}
// Clear the Directory Info List after done with it
inputDirectoryInfoList.Clear();
// Display new job list
StaticClass.DisplayJobList();
}
StaticClass.Log("\nStarted Watching for new Input Jobs...");
// Run new Input Job watch loop here forever
do
{
// Check if the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("CheckForNewInputJobs") == true)
{
return;
}
// Run any unfinished input jobs
RunInputJobsFound();
// Throttle calling new Jobs Found handler
Thread.Sleep(StaticClass.IniData.ScanWaitTime);
}while (true);
}
/// <summary>
/// Connect to TCP/IP Port
/// </summary>
/// <param name="port"></param>
/// <param name="server"></param>
/// <param name="monitorData"></pram>
public void Connect(int port, string server, StatusMonitorData monitorData)
{
ModelerStepState ModelerCurrentStepState = ModelerStepState.NONE;
string job = monitorData.Job;
// Wait about a minute for the Modeler to start execution
Thread.Sleep(StaticClass.TCP_IP_STARTUP_WAIT);
StaticClass.Log(string.Format("\nStarting TCP/IP Scan for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Log starting TCP/IP monitoring entry
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_TCPIP, JobType.TIME_START);
try
{
// Create a TcpClient
TcpClient client = new TcpClient(server, port);
if (client == null)
{
StaticClass.Logger.LogError("TcpIp Connectinon client failed to instantiate");
}
// Get a client stream for reading and writing
NetworkStream stream = client.GetStream();
if (stream == null)
{
StaticClass.Logger.LogError("TcpIp Connection stream handle was not gotten from client");
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Show connection and start sending messages
StaticClass.Log(string.Format("Connected to Modeler TCP/IP for Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now));
bool jobComplete = false;
int numRequestSent = 0;
do
{
// Loop shutdown/Pause check
if (StaticClass.ShutDownPauseCheck("TCP/IP Connect") == true)
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Check if the stream is writable, if not, exit
if (stream.CanWrite)
{
if (StaticClass.ShutDownPauseCheck(job) == false)
{
try
{
// Translate the passed message into ASCII and send it as a byte array
byte[] sendData = Encoding.ASCII.GetBytes(StatusMessage);
stream.Write(sendData, 0, sendData.Length);
numRequestSent++;
}
catch (IOException e)
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} because of Exception {2} at {3:HH:mm:ss.fff}",
job, port, e, DateTime.Now));
// Signal job complete if exception happend in Step 5 or 6
if ((ModelerCurrentStepState == ModelerStepState.STEP_5) ||
(ModelerCurrentStepState == ModelerStepState.STEP_6))
{
// Set the TCP/IP Scan complete flag to signal the RunJob thread
StaticClass.TcpIpScanComplete[job] = true;
}
else
{
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
}
return;
}
StaticClass.Log(string.Format("\nSending {0} msg to Modeler for Job {1} on Port {2} at {3:HH:mm:ss.fff}",
StatusMessage, job, port, DateTime.Now));
}
}
else
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("\nTCP/IP stream closed for Modeler Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Check if TCP/IP stream is readable and data is available
int adjustableSleepTime = StaticClass.STARTING_TCP_IP_WAIT;
int tcpIpRetryCount = 0;
bool messageReceived = false;
do
{
if (stream.CanRead && stream.DataAvailable)
{
// Buffers to store the response
byte[] recvData = new byte[256];
string responseData = string.Empty;
int bytes = stream.Read(recvData, 0, recvData.Length);
responseData = Encoding.ASCII.GetString(recvData, 0, bytes);
StaticClass.Log(string.Format("Received: {0} from Job {1} on Port {2} at {3:HH:mm:ss.fff}",
responseData, job, port, DateTime.Now));
// Readjust sleep time according to Step number
switch (responseData)
{
case "Step 1 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_1;
if (numRequestSent < StaticClass.NUM_REQUESTS_TILL_TCPIP_SLOWDOWN)
{
adjustableSleepTime = 15000;
}
else
{
adjustableSleepTime = 60000;
}
break;
case "Step 2 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_2;
adjustableSleepTime = 10000;
break;
case "Step 3 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_3;
adjustableSleepTime = 7500;
break;
case "Step 4 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_4;
adjustableSleepTime = 5000;
break;
case "Step 5 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_5;
adjustableSleepTime = 2500;
break;
case "Step 6 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_6;
adjustableSleepTime = 250;
break;
case "Whole process done, socket closed.":
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("TCP/IP for Job {0} on Port {1} received Modeler process done at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
ModelerCurrentStepState = ModelerStepState.STEP_COMPLETE;
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the TCP/IP Scan complete flag to signal the RunJob thread
StaticClass.TcpIpScanComplete[job] = true;
return;
default:
StaticClass.Logger.LogWarning("Received Weird Response: {0} from Job {1} on Port {2} at {3:HH:mm:ss.fff}",
responseData, job, port, DateTime.Now);
break;
}
// Backup check of the process complete string, even if it is concatenated with another string
if (responseData.Contains("Whole process done, socket closed."))
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("TCP/IP for Job {0} on Port {1} received Modeler process complete at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
ModelerCurrentStepState = ModelerStepState.STEP_COMPLETE;
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the TCP/IP Scan complete flag to signal the RunJob thread
StaticClass.TcpIpScanComplete[job] = true;
return;
}
// Check for Job timeout
if ((DateTime.Now - monitorData.StartTime).TotalSeconds > StaticClass.MaxJobTimeLimitSeconds)
{
StaticClass.Log(string.Format("Job Execution Timeout for Job {0} in state {1} at {2:HH:mm:ss.fff}",
job, ModelerCurrentStepState, DateTime.Now));
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Create job Timeout status
StaticClass.StatusDataEntry(job, JobStatus.JOB_TIMEOUT, JobType.TIME_START);
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("TCP/IP Receive") == true)
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Set the messge received flag to exit receive loop
messageReceived = true;
}
else
{
// Wait 250 msec between 480 Data Available checks (2 min) CanRead is set for session
Thread.Sleep(StaticClass.READ_AVAILABLE_RETRY_DELAY);
tcpIpRetryCount++;
}
}while ((tcpIpRetryCount < StaticClass.NUM_TCP_IP_RETRIES) && (messageReceived == false));
// Wait for an adjustable time between TCP/IP status requests
Thread.Sleep(adjustableSleepTime);
}while (jobComplete == false);
// Close everything
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} in state {2} at {3:HH:mm:ss.fff}",
job, port, ModelerCurrentStepState, DateTime.Now));
}
catch (SocketException e)
{
StaticClass.Logger.LogError(string.Format("SocketException {0} for Job {1} on Port {2} in state {3} at {4:HH:mm:ss.fff}",
e, job, port, ModelerCurrentStepState, DateTime.Now));
}
}