/// <summary>
/// The Add or Change of files callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string fullDirectory = e.FullPath;
string jobDirectory = fullDirectory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
string jobFile = jobDirectory.Substring(jobDirectory.LastIndexOf('\\') + 1);
string job = jobDirectory.Substring(0, jobDirectory.LastIndexOf('\\'));
// If Number of files is not complete
if (StaticClass.NumberOfInputFilesFound[job] < StaticClass.NumberOfInputFilesNeeded[job])
{
// Wait some for the file creation to complete
Thread.Sleep(StaticClass.FILE_RECEIVE_WAIT);
if (StaticClass.CheckFileReady(fullDirectory) == true)
{
// Increment the number of Input Buffer Job files found
StaticClass.NumberOfInputFilesFound[job]++;
StaticClass.Log(string.Format("Input File Watcher detected {0} for Job {1} file {2} of {3} at {4:HH:mm:ss.fff}",
jobFile, job, StaticClass.NumberOfInputFilesFound[job], StaticClass.NumberOfInputFilesNeeded[job], DateTime.Now));
// If Number of Input files is complete
if (StaticClass.NumberOfInputFilesFound[job] == StaticClass.NumberOfInputFilesNeeded[job])
{
StaticClass.Log(string.Format("\nInput File Watcher detected Job {0} complete set of {1} files at {2:HH:mm:ss.fff}",
job, StaticClass.NumberOfInputFilesNeeded[job], DateTime.Now));
// Signal the Run thread that the Input Buffer files were found
StaticClass.InputFileScanComplete[job] = true;
}
}
}
}
/// <summary>
/// Log a Status and write to csv file
/// </summary>
/// <param name="job"></param>
/// <param name="status"></param>
/// <param name="timeSlot"></param>
public void ListStatus(string job, JobStatus status, JobType timeSlot)
{
StatusData entry = new StatusData();
if (entry == null)
{
StaticClass.Logger.LogError("StatusEntry entry failed to instantiate");
}
entry.Job = job;
entry.JobStatus = status;
switch (timeSlot)
{
case JobType.TIME_RECEIVED:
entry.TimeReceived = DateTime.Now;
break;
case JobType.TIME_START:
entry.TimeStarted = DateTime.Now;
break;
case JobType.TIME_COMPLETE:
entry.TimeCompleted = DateTime.Now;
break;
}
// Add entry to status data list
StaticClass.StatusDataList.Add(entry);
// Output to Console also
StaticClass.Log(string.Format("Status: Job: {0} Job Status: {1} Time Type: {2}",
job, status, timeSlot.ToString()));
}
// Define the event handlers.
/// <summary>
/// The directory created callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string jobDirectory = e.FullPath;
string job = jobDirectory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
StaticClass.Log(string.Format("\nInput Directory Watcher checking new Job {0} for Input Job list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
if (StaticClass.ShutDownPauseCheck("Directory Watcher OnCreated") == false)
{
Thread.Sleep(StaticClass.WAIT_FOR_FILES_TO_COMPLETE);
// Check directory contents complete
if (StaticClass.CheckDirectoryReady(jobDirectory) == true)
{
StaticClass.AddJobToList(job);
if ((StaticClass.IniData.DebugMode & (byte)DebugModeState.JOB_LIST) != 0)
{
StaticClass.DisplayJobList();
}
}
else
{
StaticClass.Logger.LogError("DirectoryWatcherThread Job {0} directory check failed at {1:HH:mm:ss.fff}",
job, DateTime.Now);
}
}
}
/// <summary>
/// Check if the Modeler has deposited the OverallResult entry in the job data.xml file
/// </summary>
/// <param name="dataXmlFileName"></param>
/// <returns></returns>
public bool OverallResultEntryCheck(string dataXmlFileName)
{
int numOfRetries = 0;
do
{
// Check for data.xml file to be ready
if (StaticClass.CheckFileReady(dataXmlFileName))
{
// Check if the OverallResult node exists
XmlDocument dataXmlDoc = new XmlDocument();
dataXmlDoc.Load(dataXmlFileName);
XmlNode OverallResult = dataXmlDoc.DocumentElement.SelectSingleNode("/Data/OverallResult/result");
if (OverallResult != null)
{
return(true);
}
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("Overall Result Entry Check") == true)
{
return(false);
}
Thread.Sleep(StaticClass.FILE_WAIT_DELAY);
}
}while (numOfRetries++ < StaticClass.NUM_RESULTS_ENTRY_RETRIES);
StaticClass.Log(string.Format("File {0} did not have Overall Result entry even after {1} retries at {2:HH:mm:ss.fff}",
dataXmlFileName, StaticClass.NUM_RESULTS_ENTRY_RETRIES, DateTime.Now));
return(false);
}
/// <summary>
/// Method to start new jobs from the Input Buffer
/// </summary>
/// <param name="directory"></param>
public void StartInputJob(string directory)
{
// Get data found in Job xml file
JobXmlData jobXmlData = StaticClass.GetJobXmlFileInfo(directory, DirectoryScanType.INPUT_BUFFER);
if (jobXmlData == null)
{
StaticClass.Logger.LogError("InputJobsScanThread GetJobXmlData failed");
}
// Display job xml data found
StaticClass.Log("Input Job : " + jobXmlData.Job);
StaticClass.Log("Input Job Directory : " + jobXmlData.JobDirectory);
StaticClass.Log("Input Job Serial Number : " + jobXmlData.JobSerialNumber);
StaticClass.Log("Input Job Time Stamp : " + jobXmlData.TimeStamp);
StaticClass.Log("Input Job Xml File : " + jobXmlData.XmlFileName);
StaticClass.Log(string.Format("Started Input Job {0} executing Slot {1} at {2:HH:mm:ss.fff}",
jobXmlData.Job, StaticClass.NumberOfJobsExecuting + 1, DateTime.Now));
// Create a thread to run the job, and then start the thread
JobRunThread jobRunThread = new JobRunThread(jobXmlData, DirectoryScanType.INPUT_BUFFER);
if (jobRunThread == null)
{
StaticClass.Logger.LogError("InputJobsScanThread jobRunThread failed to instantiate");
}
jobRunThread.ThreadProc();
}
/// <summary>
/// Display the Modeler Process information
/// </summary>
/// <param name="modelerProcess"></param>
public void DisplayProcessInfo(string job, Process modelerProcess)
{
// Display Modeler Executable information
StaticClass.Log($"\nJob {job} Modeler execution process data:");
StaticClass.Log($"ProcessName : {modelerProcess.ProcessName}");
StaticClass.Log($"StartTime : {modelerProcess.StartTime}");
StaticClass.Log($"MainWindowTitle : {modelerProcess.MainWindowTitle}");
StaticClass.Log($"MainModule : {modelerProcess.MainModule}");
StaticClass.Log($"StartInfo : {modelerProcess.StartInfo}");
StaticClass.Log($"GetType : {modelerProcess.GetType()}");
StaticClass.Log($"MainWindowHandle : {modelerProcess.MainWindowHandle}");
StaticClass.Log($"Handle : {modelerProcess.Handle}");
StaticClass.Log($"Id : {modelerProcess.Id}");
StaticClass.Log($"PriorityClass : {modelerProcess.PriorityClass}");
StaticClass.Log($"Basepriority : {modelerProcess.BasePriority}");
StaticClass.Log($"PriorityBoostEnabled : {modelerProcess.PriorityBoostEnabled}");
StaticClass.Log($"Responding : {modelerProcess.Responding}");
StaticClass.Log($"ProcessorAffinity : {modelerProcess.ProcessorAffinity}");
StaticClass.Log($"HandleCount : {modelerProcess.HandleCount}");
StaticClass.Log($"MaxWorkingSet : {modelerProcess.MaxWorkingSet}");
StaticClass.Log($"MinWorkingSet : {modelerProcess.MinWorkingSet}");
StaticClass.Log($"NonpagedSystemMemorySize64 : {modelerProcess.NonpagedSystemMemorySize64}");
StaticClass.Log($"PeakVirtualMemorySize64 : {modelerProcess.PeakVirtualMemorySize64}");
StaticClass.Log($"PagedSystemMemorySize64 : {modelerProcess.PagedSystemMemorySize64}");
StaticClass.Log($"PrivateMemorySize64 : {modelerProcess.PrivateMemorySize64}");
StaticClass.Log($"VirtualMemorySize64 : {modelerProcess.VirtualMemorySize64}");
StaticClass.Log($"NonpagedSystemMemorySize64 : {modelerProcess.PagedMemorySize64}");
StaticClass.Log($"WorkingSet64 : {modelerProcess.WorkingSet64}");
StaticClass.Log($"PeakWorkingSet64 : {modelerProcess.PeakWorkingSet64}");
StaticClass.Log($"PrivilegedProcessorTime : {modelerProcess.PrivilegedProcessorTime}");
StaticClass.Log($"TotalProcessorTime : {modelerProcess.TotalProcessorTime}");
StaticClass.Log($"UserProcessorTime : {modelerProcess.UserProcessorTime}");
}
/// <summary>
/// Copy file from source to target
/// </summary>
/// <param name="sourceFile"></param>
/// <param name="targetFile"></param>
public static void CopyFile(string sourceFile, string targetFile)
{
FileInfo Source = new FileInfo(sourceFile);
if (Source == null)
{
StaticClass.Logger.LogError("FileHandling Source failed to instantiate");
}
FileInfo Target = new FileInfo(targetFile);
if (Target == null)
{
StaticClass.Logger.LogError("FileHandling Target failed to instantiate");
}
if (Target.Exists)
{
// Delete the Target file first
lock (FileLock)
{
File.SetAttributes(Target.FullName, FileAttributes.Normal);
Target.Delete();
}
}
// Copy to target file
lock (FileLock)
{
Source.CopyTo(targetFile);
}
StaticClass.Log(string.Format("Copied File {0} -> {1}", sourceFile, targetFile));
}
/// <summary>
/// Returns when a file is ready to access
/// </summary>
/// <param name="fileName"></param>
/// <returns>Returns if file is ready to access</returns>
public static bool CheckFileReady(string fileName)
{
int numOfRetries = 0;
do
{
try
{
// Check that a file is both readable and writeable
using (FileStream fileService = File.Open(fileName, FileMode.Open))
{
if (fileService.CanRead && fileService.CanWrite)
{
Log(string.Format("File {0} is available at {1:HH:mm:ss.fff}", fileName, DateTime.Now));
return(true);
}
}
}
catch (UnauthorizedAccessException e)
{
Log(string.Format("File {0} not available exception {1} at {2:HH:mm:ss.fff}", fileName, e, DateTime.Now));
}
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("IsFileReady") == true)
{
return(false);
}
Thread.Yield();
}while (numOfRetries++ < StaticClass.NUM_XML_ACCESS_RETRIES);
return(false);
}
/// <summary>
/// The Add of files callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string fullDirectory = e.FullPath;
string jobDirectory = fullDirectory.Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
string jobFile = jobDirectory.Substring(jobDirectory.LastIndexOf('\\') + 1);
string job = jobDirectory.Substring(0, jobDirectory.LastIndexOf('\\'));
// If Number of files is not complete
if (StaticClass.NumberOfProcessingFilesFound[job] < StaticClass.NumberOfProcessingFilesNeeded[job])
{
// Increment the number of Processing Buffer Job files found
StaticClass.NumberOfProcessingFilesFound[job]++;
StaticClass.Log(string.Format("Processing File Watcher detected {0} for Job {1} file {2} of {3} at {4:HH:mm:ss.fff}",
jobFile, job, StaticClass.NumberOfProcessingFilesFound[job], StaticClass.NumberOfProcessingFilesNeeded[job], DateTime.Now));
// If Number of Processing files is complete
if (StaticClass.NumberOfProcessingFilesFound[job] == StaticClass.NumberOfProcessingFilesNeeded[job])
{
StaticClass.Log(string.Format("Processing File Watcher detected Job {0} complete set of {1} files at {2:HH:mm:ss.fff}",
job, StaticClass.NumberOfProcessingFilesNeeded[job], DateTime.Now));
// Short pause so that files from the network that are slow
Thread.Sleep(StaticClass.WAIT_FOR_FILES_TO_COMPLETE);
// Signal the Run thread that the Processing Buffer files were found
StaticClass.ProcessingFileScanComplete[job] = true;
}
}
}
/// <summary>
/// Run Input Buffer Scan
/// </summary>
/// <param name="jobDirectory"></param>
public void RunInputBufferPartialJob(string jobDirectory)
{
JobXmlData jobXmlData = StaticClass.GetJobXmlFileInfo(jobDirectory, DirectoryScanType.INPUT_BUFFER);
if (jobXmlData.XmlFileName != string.Empty)
{
// Now start the new job
StartInputJob(jobDirectory);
}
}
/// <summary>
/// Write to Log File
/// </summary>
/// <param name="text"></param>
public void WriteToLogFile(string text)
{
bool tooBig = false;
int MaxFileSize = StaticClass.LogFileSizeLimit * 1024 * 1024;
lock (FileLock)
{
using (FileStream stream = new FileStream(LogFileName, FileMode.Append))
{
using (StreamWriter writer = new StreamWriter(stream))
{
// Check file size before writing
if (stream.Position < MaxFileSize)
{
writer.WriteLine(text);
}
else
{
tooBig = true;
}
}
}
}
if (tooBig)
{
StaticClass.Log("\nProcess log.txt file too big. Reducing 10%...\n");
lock (FileLock)
{
// Remove old data from log file
using (MemoryStream memoryStream = new MemoryStream(StaticClass.LogFileSizeLimit))
{
using (FileStream stream = new FileStream(LogFileName, FileMode.Open, FileAccess.ReadWrite))
{
// Reduce size of log file by 10%
int sizeLimit = (int)(MaxFileSize * 0.9);
stream.Seek(-sizeLimit, SeekOrigin.End);
byte[] bytes = new byte[sizeLimit];
stream.Read(bytes, 0, sizeLimit);
memoryStream.Write(bytes, 0, sizeLimit);
memoryStream.Position = 0;
stream.SetLength(sizeLimit);
stream.Position = 0;
memoryStream.CopyTo(stream);
using (StreamWriter writer = new StreamWriter(stream))
{
writer.WriteLine(text);
}
}
}
}
}
}
/// <summary>
/// A Thread procedure that scans for new jobs
/// </summary>
public void ThreadProc()
{
// Check for expired Input jobs
StaticClass.CheckForInputBufferTimeLimits();
StaticClass.InputJobsScanThreadHandle = new Thread(() => CheckForUnfinishedInputJobs());
if (StaticClass.InputJobsScanThreadHandle == null)
{
StaticClass.Logger.LogError("InputJobsScanThread InputJobsScanThreadHandle thread failed to instantiate");
}
StaticClass.InputJobsScanThreadHandle.Start();
}
public void RunInputBufferScan(string directory, string job, StatusMonitorData monitorData)
{
// Add initial entry to status list
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_INPUT, JobType.TIME_START);
// Monitor the Input Buffer job directory until it has the total number of consumed files
string inputBufferJobDir = StaticClass.IniData.InputDir;
int numberOfFilesNeeded = monitorData.NumFilesConsumed;
if (Directory.Exists(inputBufferJobDir))
{
string inputJobFileDir = inputBufferJobDir + @"\" + job;
// Register with the File Watcher class event and start its thread
InputFileWatcherThread inputFileWatch = new InputFileWatcherThread(inputJobFileDir, numberOfFilesNeeded);
if (inputFileWatch == null)
{
StaticClass.Logger.LogError("Job Run Thread inputFileWatch failed to instantiate");
}
inputFileWatch.ThreadProc();
// Wait for Input file scan to complete
do
{
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
Thread.Yield();
}while (StaticClass.InputFileScanComplete[job] == false);
StaticClass.Log(string.Format("Finished Input file scan for Job {0} at {1:HH:mm:ss.fff}",
inputJobFileDir, DateTime.Now));
// Add copying entry to status list
StaticClass.StatusDataEntry(job, JobStatus.COPYING_TO_PROCESSING, JobType.TIME_START);
// Move files from Input directory to the Processing directory, creating it first if needed
FileHandling.CopyFolderContents(inputJobFileDir, directory, true, true);
}
else
{
StaticClass.Logger.LogError("Could not find Input Buffer Directory");
throw new InvalidOperationException("Could not find Input Buffer Directory");
}
}
/// <summary>
/// Check for unfinished Input Buffer jobs
/// </summary>
public void RunInputJobsFound()
{
// Check if there are unfinished Input jobs waiting to run
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
int index = 0;
string job = string.Empty;
Task AddTask = Task.Run(() =>
{
if (StaticClass.InputJobsToRun.Count > 0)
{
index = StaticClass.InputJobsToRun.Count;
job = StaticClass.InputJobsToRun.Read(index);
StaticClass.InputJobsToRun.Delete(index);
StaticClass.Log(string.Format("\nInput Directory Watcher removed Job {0} from Input Job list index {1} at {2:HH:mm:ss.fff}",
job, index, DateTime.Now));
}
});
TimeSpan timeSpan = TimeSpan.FromMilliseconds(StaticClass.DELETE_TASK_DELAY);
if (!AddTask.Wait(timeSpan))
{
StaticClass.Logger.LogError("InputJobScanThread Delete Job {0} timed out at {1:HH:mm:ss.fff}", job, DateTime.Now);
}
if (job != string.Empty)
{
string directory = StaticClass.IniData.InputDir + @"\" + job;
StaticClass.Log(string.Format("Starting Input Job {0} at {1:HH:mm:ss.fff}", directory, DateTime.Now));
// Reset Input job file scan flag
StaticClass.InputFileScanComplete[job] = false;
// Start an Input Buffer Job
StartInputJob(directory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
}
/// <summary>
/// Display the Modeler Process information
/// </summary>
/// <param name=job"></param>
/// <param name="modelerProcess"></param>
public void DisplayProcessInfo(string job, Process modelerProcess)
{
if ((StaticClass.IniData.DebugMode & (byte)DebugModeState.MODELER) != 0)
{
// Display Modeler Executable information
StaticClass.Log($"\nJob {job} Modeler execution process data:");
StaticClass.Log($"ProcessName : {modelerProcess.ProcessName}");
StaticClass.Log($"StartTime : {modelerProcess.StartTime}");
StaticClass.Log($"MainWindowTitle : {modelerProcess.MainWindowTitle}");
StaticClass.Log($"MainModule : {modelerProcess.MainModule}");
StaticClass.Log($"StartInfo : {modelerProcess.StartInfo}");
StaticClass.Log($"GetType : {modelerProcess.GetType()}");
StaticClass.Log($"MainWindowHandle : {modelerProcess.MainWindowHandle}");
StaticClass.Log($"Handle : {modelerProcess.Handle}");
StaticClass.Log($"Id : {modelerProcess.Id}");
StaticClass.Log($"PriorityClass : {modelerProcess.PriorityClass}");
StaticClass.Log($"Basepriority : {modelerProcess.BasePriority}");
StaticClass.Log($"PriorityBoostEnabled : {modelerProcess.PriorityBoostEnabled}");
StaticClass.Log($"Responding : {modelerProcess.Responding}");
StaticClass.Log($"ProcessorAffinity : {modelerProcess.ProcessorAffinity}");
StaticClass.Log($"HandleCount : {modelerProcess.HandleCount}");
StaticClass.Log($"MaxWorkingSet : {modelerProcess.MaxWorkingSet}");
StaticClass.Log($"MinWorkingSet : {modelerProcess.MinWorkingSet}");
StaticClass.Log($"NonpagedSystemMemorySize64 : {modelerProcess.NonpagedSystemMemorySize64}");
StaticClass.Log($"PeakVirtualMemorySize64 : {modelerProcess.PeakVirtualMemorySize64}");
StaticClass.Log($"PagedSystemMemorySize64 : {modelerProcess.PagedSystemMemorySize64}");
StaticClass.Log($"PrivateMemorySize64 : {modelerProcess.PrivateMemorySize64}");
StaticClass.Log($"VirtualMemorySize64 : {modelerProcess.VirtualMemorySize64}");
StaticClass.Log($"NonpagedSystemMemorySize64 : {modelerProcess.PagedMemorySize64}");
StaticClass.Log($"WorkingSet64 : {modelerProcess.WorkingSet64}");
StaticClass.Log($"PeakWorkingSet64 : {modelerProcess.PeakWorkingSet64}");
StaticClass.Log($"PrivilegedProcessorTime : {modelerProcess.PrivilegedProcessorTime}");
StaticClass.Log($"TotalProcessorTime : {modelerProcess.TotalProcessorTime}");
StaticClass.Log($"UserProcessorTime : {modelerProcess.UserProcessorTime}");
}
else
{
StaticClass.Log(string.Format("\nJob {0} Modeler execution started with base priority {1} at {2:HH:mm:ss.fff}\n",
job, modelerProcess.BasePriority, modelerProcess.StartTime));
}
}
/// <summary>
/// Start the Monitor process by starting the new jobs scan thread
/// </summary>
public void StartMonitorProcess()
{
StaticClass.ShutdownFlag = false;
// Check for pause state and reset it if the Start button is pressed when in Pause mode
if (StaticClass.PauseFlag == true)
{
StaticClass.Log(string.Format("Taking the system out of Pause Mode at {0:HH:mm:ss.fff}", DateTime.Now));
StaticClass.PauseFlag = false;
}
else
{
// Start Modeler job Processing thread
InputJobsScanThread newJobsScanThread = new InputJobsScanThread();
if (newJobsScanThread == null)
{
StaticClass.Logger.LogError("StartMonitorProcess newJobsScanThread failed to instantiate");
}
newJobsScanThread.ThreadProc();
}
}
/// <summary>
/// Deletes a directory after deleting files inside
/// </summary>
/// <param name="targetDirectory"></param>
public static void DeleteDirectory(string targetDirectory)
{
// First delete all files in target directory
string[] files = Directory.GetFiles(targetDirectory);
foreach (string file in files)
{
lock (FileLock)
{
File.SetAttributes(file, FileAttributes.Normal);
File.Delete(file);
}
}
lock (FileLock)
{
// Then delete the directory
File.SetAttributes(targetDirectory, FileAttributes.Normal);
Directory.Delete(targetDirectory, false);
}
StaticClass.Log(string.Format("Deleted Directory {0}", targetDirectory));
}
// Define the event handlers.
/// <summary>
/// The directory created callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string jobDirectory = e.FullPath;
string job = jobDirectory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
int index = 0;
// Check Input Buffer Job directory to see if it is ready if adding the first jobs
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
StaticClass.Log(string.Format("\nInput Directory Watcher checking new Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Wait some for the directory creation and file copies to complete
Thread.Sleep(StaticClass.DIRECTORY_RECEIVE_WAIT);
StaticClass.CheckJobDirectoryComplete(jobDirectory);
}
// Do Shutdown Pause check
if (StaticClass.ShutDownPauseCheck("Directory Watcher OnCreated") == false)
{
// Loop shutdown/Pause check
Task AddTask = Task.Run(() =>
{
index = StaticClass.InputJobsToRun.Count + 1;
StaticClass.InputJobsToRun.Add(index, job);
StaticClass.Log(string.Format("Input Directory Watcher added new Job {0} to Input Job list index {1} at {2:HH:mm:ss.fff}",
job, index, DateTime.Now));
});
TimeSpan timeSpan = TimeSpan.FromMilliseconds(StaticClass.ADD_JOB_DELAY);
if (!AddTask.Wait(timeSpan))
{
StaticClass.Logger.LogError("DirectoryWatcherThread Add Job {0} timed out at {1:HH:mm:ss.fff}", job, DateTime.Now);
}
}
}
/// <summary>
/// Start a Processing directory job
/// </summary>
/// <param name="directory"></param>
public void StartProcessingJob(string directory)
{
// Delete the data.xml file in the Processing directory if found
string dataXmlFile = directory + @"\" + "data.xml";
if (File.Exists(dataXmlFile))
{
File.Delete(dataXmlFile);
}
// Get data found in job Xml file
JobXmlData jobXmlData = StaticClass.GetJobXmlFileInfo(directory, DirectoryScanType.PROCESSING_BUFFER);
if (jobXmlData == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread GetJobXmlData failed");
}
// Display job xml Data found
StaticClass.Log("Processing Job : " + jobXmlData.Job);
StaticClass.Log("Processing Job Directory : " + jobXmlData.JobDirectory);
StaticClass.Log("Processing Job Serial Number : " + jobXmlData.JobSerialNumber);
StaticClass.Log("Processing Job Time Stamp : " + jobXmlData.TimeStamp);
StaticClass.Log("Processing Job Xml File : " + jobXmlData.XmlFileName);
StaticClass.Log(string.Format("Starting Processing directory Job {0} Executing Slot {1} at {2:HH:mm:ss.fff}",
jobXmlData.Job, StaticClass.NumberOfJobsExecuting + 1, DateTime.Now));
// Create a thread to run the job, and then start the thread
JobRunThread jobRunThread = new JobRunThread(jobXmlData, DirectoryScanType.PROCESSING_BUFFER);
if (jobRunThread == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread jobRunThread failed to instantiate");
}
jobRunThread.ThreadProc();
}
public void WatchDirectory()
{
// Create a new FileSystemWatcher and set its properties
using (FileSystemWatcher watcher = new FileSystemWatcher())
{
if (watcher == null)
{
StaticClass.Logger.LogError("DirectoryWatcherThread watcher failed to instantiate");
}
// Watch for changes in the directory list
watcher.NotifyFilter =
NotifyFilters.DirectoryName |
NotifyFilters.CreationTime;
// Set the Input Buffre as path to watch for new directory additions
watcher.Path = StaticClass.IniData.InputDir;
// Watch for any directories names added
watcher.Filter = "*.*";
watcher.IncludeSubdirectories = true;
// Add OnCreated event handler which is the only one needed
watcher.Created += OnCreated;
watcher.Changed += OnChanged;
// Begin watching for directory changes to Input directory
watcher.EnableRaisingEvents = true;
// Scan Input directory forever
do
{
Thread.Yield();
}while (StaticClass.ShutDownPauseCheck("Directory Watcher") == false);
}
}
/// <summary>
/// Execute the Modeler command line
/// </summary>
/// <param name="job"></param>
/// <returns></returns>
public Process ExecuteCommand(string job)
{
ProcessStartInfo startInfo = new ProcessStartInfo(Executable)
{
Arguments = string.Format("{0} {1} {2}", ProcessingDirParam, StartPortParam, CpuCoresParam),
UseShellExecute = true,
WorkingDirectory = ProcessingDirectory,
WindowStyle = ProcessWindowStyle.Minimized
};
// Check object
if (startInfo == null)
{
StaticClass.Logger.LogError("CommandLineGenerator startInfo failed to instantiate");
}
// Start the Modeler process window
Process ModelerProcess = Process.Start(startInfo);
if (ModelerProcess == null)
{
StaticClass.Logger.LogError("CommandLineGeneratorThread ModelerProcess failes to instantiate");
}
// Set Process Priority
ModelerProcess.PriorityClass = ProcessPriorityClass.Normal;
// Store process handle to use for stopping
StaticClass.ProcessHandles[job] = ModelerProcess;
// Give the Modeler time so you can read the Main Window Title parameter for display confirmation
Thread.Sleep(StaticClass.DISPLAY_PROCESS_TITLE_WAIT);
StaticClass.Log(string.Format("{0} {1}", ModelerProcess.MainWindowTitle, ModelerProcess.StartInfo.Arguments));
return(ModelerProcess);
}
/// <summary>
/// Check for unfinished processing jobs after one completes
/// </summary>
public void RunUnfinishedProcessingJobs()
{
// Start Processing jobs currently waiting
foreach (string job in ProcessingJobsToRun.Reverse <string>())
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
string directory = StaticClass.IniData.ProcessingDir + @"\" + job;
StaticClass.Log(string.Format("\nStarting Processing Job {0} at {1:HH:mm:ss.fff}", directory, DateTime.Now));
// Remove job run from Processing Job list
lock (RemoveLock)
{
if (ProcessingJobsToRun.Remove(job) == false)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread failed to remove Job {0} from Processing Job list", job);
}
}
StaticClass.Log(string.Format("Processing Directory Watcher removed Job {0} from Job list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Reset Processing job and file scan flags
StaticClass.ProcessingFileScanComplete[job] = false;
StaticClass.ProcessingJobScanComplete[job] = false;
StaticClass.JobShutdownFlag[job] = false;
// Start a Processing Buffer Job
StartProcessingJob(directory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
}
/// <summary>
/// Method to scan for unfinished jobs in the Input Buffer
/// </summary>
public void CheckForUnfinishedInputJobs()
{
// Start the Directory Watcher class to scan for new Input Buffer jobs
DirectoryWatcherThread dirWatch = new DirectoryWatcherThread();
if (dirWatch == null)
{
StaticClass.Logger.LogError("InputJobsScanThread dirWatch failed to instantiate");
}
dirWatch.ThreadProc();
// Register with the Processing Buffer Jobs check completion event and start its thread
ProcessingJobsScanThread unfinishedProcessingJobs = new ProcessingJobsScanThread();
if (unfinishedProcessingJobs == null)
{
StaticClass.Logger.LogError("InputJobsScanThread currentProcessingJobs failed to instantiate");
}
unfinishedProcessingJobs.ThreadProc();
// Wait while scanning for unfinished Processing jobs
do
{
if (StaticClass.ShutDownPauseCheck("CheckForUnfinishedInputJobs") == true)
{
return;
}
Thread.Yield();
}while (StaticClass.UnfinishedProcessingJobsScanComplete == false);
StaticClass.Log("\nChecking for unfinished Input Jobs...");
DirectoryInfo InputDirectoryInfo = new DirectoryInfo(StaticClass.IniData.InputDir);
if (InputDirectoryInfo == null)
{
StaticClass.Logger.LogError("InputJobsScanThread InputDirectoryInfo failed to instantiate");
}
// Get the current list of directories from the Input Buffer
List <DirectoryInfo> inputDirectoryInfoList = InputDirectoryInfo.EnumerateDirectories().ToList();
if (inputDirectoryInfoList == null)
{
StaticClass.Logger.LogError("InputJobsScanThread inputDirectoryInfoList failed to instantiate");
}
if (inputDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nUnfinished Input Jobs waiting...");
}
else
{
StaticClass.Log("\nNo unfinished Input Jobs found...");
}
// Start the jobs in the directory list found for the Input Buffer
foreach (DirectoryInfo dirInfo in inputDirectoryInfoList.Reverse <DirectoryInfo>())
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
string jobDirectory = dirInfo.ToString();
string job = jobDirectory.ToString().Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
// Check if the directory has a full set of Job files
if (StaticClass.CheckIfJobFilesComplete(jobDirectory) == true)
{
StaticClass.Log(string.Format("Starting Input Job {0} at {1:HH:mm:ss.fff}", jobDirectory, DateTime.Now));
// Remove job run from Input Job directory list
lock (RemoveLock)
{
if (inputDirectoryInfoList.Remove(dirInfo) == false)
{
StaticClass.Logger.LogError("InputJobsScanThread failed to remove Job {0} from Input Job list", job);
}
}
StaticClass.Log(string.Format("Input Directory Watcher removed Job {0} from Input Directory Jobs list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Reset Input file scan flag
StaticClass.InputFileScanComplete[job] = false;
// Start an Input Buffer Job
StartInputJob(jobDirectory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
}
if (inputDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nMore unfinished Input Jobs then Execution Slots available...");
// Sort the Input Buffer directory list by older dates first
inputDirectoryInfoList.Sort((x, y) => - x.LastAccessTime.CompareTo(y.LastAccessTime));
// Add the jobs in the directory list to the Input Buffer Jobs to run list
foreach (DirectoryInfo dirInfo in inputDirectoryInfoList)
{
string directory = dirInfo.ToString();
string job = directory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
StaticClass.AddJobToList(job);
}
// Clear the Directory Info List after done with it
inputDirectoryInfoList.Clear();
// Display new job list
StaticClass.DisplayJobList();
}
StaticClass.Log("\nStarted Watching for new Input Jobs...");
// Run new Input Job watch loop here forever
do
{
// Check if the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("CheckForNewInputJobs") == true)
{
return;
}
// Run any unfinished input jobs
RunInputJobsFound();
// Throttle calling new Jobs Found handler
Thread.Sleep(StaticClass.IniData.ScanWaitTime);
}while (true);
}
/// <summary>
/// Check for new Input Buffer jobs
/// </summary>
public void RunInputJobsFound()
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
if (StaticClass.GetTotalNumberOfJobs() > 0)
{
string job = StaticClass.GetJobFromList(StaticClass.CurrentJobIndex);
if (job != string.Empty)
{
StaticClass.Log(string.Format("Input Job handler got Job {0} from list index {1} at {2:HH:mm:ss.fff}",
job, StaticClass.CurrentJobIndex, DateTime.Now));
// Check for complete jobs and run them first
string jobDirectory = StaticClass.IniData.InputDir + @"\" + job;
if (StaticClass.CheckIfJobFilesComplete(jobDirectory) == true)
{
StaticClass.Log(string.Format("Starting Input Job {0} index {1} at {2:HH:mm:ss.fff}",
jobDirectory, StaticClass.CurrentJobIndex, DateTime.Now));
// Start the new completely ready Job
StartInputJob(jobDirectory);
// Delete the job from the list and display the new list
StaticClass.DeleteJobFromList(StaticClass.CurrentJobIndex);
}
else // Partial Job handling
{
// Skip Partial Job if there are more in the list
int lastIndex = StaticClass.GetLastIndex();
if (StaticClass.CurrentJobIndex < lastIndex)
{
StaticClass.Log(string.Format("Input Job scanner skipping Job {0} index {1} as not ready at {2:HH:mm:ss.fff}",
job, StaticClass.CurrentJobIndex, DateTime.Now));
StaticClass.CurrentJobIndex++;
}
else // Run last job in list
{
// Wait for the job xml file
JobXmlData jobXmlData = StaticClass.GetJobXmlFileInfo(jobDirectory, DirectoryScanType.INPUT_BUFFER);
if (jobXmlData.XmlFileName != string.Empty)
{
StaticClass.Log(string.Format("Starting Partial Input Job {0} index {1} at {2:HH:mm:ss.fff}",
jobDirectory, StaticClass.CurrentJobIndex, DateTime.Now));
// Start the new partial job
StartInputJob(jobDirectory);
// Delete the job from the list
StaticClass.DeleteJobFromList(StaticClass.CurrentJobIndex);
}
// If there is a skipped partial job, start it next by setting index to previous one
int previousJobIndex = StaticClass.GetPreviousIndex();
if ((previousJobIndex > 0) && (StaticClass.CurrentJobIndex > previousJobIndex))
{
StaticClass.CurrentJobIndex = previousJobIndex;
}
}
}
}
}
else
{
if (CurrentJobIndex != StaticClass.CurrentJobIndex)
{
StaticClass.Log(string.Format("Input Jobs Scanner Resetting Current Job Index from {0} to {1} at {2:HH:mm:ss.fff}",
StaticClass.CurrentJobIndex, 1, DateTime.Now));
CurrentJobIndex = StaticClass.CurrentJobIndex = 1;
}
}
}
}
/// <summary>
/// Get the data from the Job xml file
/// </summary>
/// <param name="jobXmlData"></param>
/// <param name="monitorData"></param>
/// <returns></returns>
public StatusMonitorData GetJobXmlData(JobXmlData jobXmlData, StatusMonitorData monitorData)
{
// Wait for Job xml file to be ready
string jobXmlFileName = jobXmlData.JobDirectory + @"\" + jobXmlData.XmlFileName;
if (StaticClass.CheckFileReady(jobXmlFileName))
{
// Read Job xml file and get the top node
XmlDocument jobXmlDoc = new XmlDocument();
jobXmlDoc.Load(jobXmlFileName);
XmlElement root = jobXmlDoc.DocumentElement;
string TopNode = root.LocalName;
// Get nodes for the number of files and names of files to transfer from Job .xml file
XmlNode UnitNumberdNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/listitem/value");
XmlNode ConsumedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Consumed");
XmlNode ProducedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Produced");
XmlNode TransferedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Transfered");
XmlNode ModelerNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Modeler");
// Assign then increment port number for this Job
monitorData.JobPortNumber = StaticClass.IniData.StartPort + StaticClass.JobPortIndex++;
// Get the modeler and number of files to transfer
int NumFilesToTransfer = 0;
monitorData.UnitNumber = UnitNumberdNode.InnerText;
monitorData.Modeler = ModelerNode.InnerText;
monitorData.NumFilesConsumed = Convert.ToInt32(ConsumedNode.InnerText);
monitorData.NumFilesProduced = Convert.ToInt32(ProducedNode.InnerText);
if (TransferedNode != null)
{
NumFilesToTransfer = Convert.ToInt32(TransferedNode.InnerText);
}
monitorData.NumFilesToTransfer = NumFilesToTransfer;
// Get the modeler and number of files to transfer
StaticClass.Log($"Unit Number : {monitorData.UnitNumber}");
StaticClass.Log($"Modeler : {monitorData.Modeler}");
StaticClass.Log($"Num Files Consumed : {monitorData.NumFilesConsumed}");
StaticClass.Log($"Num Files Produced : {monitorData.NumFilesProduced}");
StaticClass.Log($"Num Files To Transfer : {monitorData.NumFilesToTransfer}");
StaticClass.Log($"Job Port Number : {monitorData.JobPortNumber}");
// Create the Transfered file list from the Xml file entries
monitorData.TransferedFileList = new List <string>(NumFilesToTransfer);
for (int i = 1; i < NumFilesToTransfer + 1; i++)
{
string transferFileNodeName = ("/" + TopNode + "/FileConfiguration/Transfered" + i.ToString());
XmlNode TransferedFileXml = jobXmlDoc.DocumentElement.SelectSingleNode(transferFileNodeName);
monitorData.TransferedFileList.Add(TransferedFileXml.InnerText);
StaticClass.Log(string.Format("Transfer File{0} : {1}", i, TransferedFileXml.InnerText));
}
}
else
{
StaticClass.Logger.LogError("File {0} is not available at {1:HH:mm:ss.fff}\n", jobXmlFileName, DateTime.Now);
}
return(monitorData);
}
/// <summary>
/// Get the Config.ini file data from the working directory
/// </summary>
public void GetIniFileData()
{
// Check that Config.ini file exists
string IniFileName = "Config.ini";
if (File.Exists(IniFileName) == false)
{
StaticClass.Logger.LogCritical("Missing Config.ini file");
throw new InvalidOperationException("Config.ini file does not exist in the Handler directory");
}
IniFileHandler IniParser = new IniFileHandler(IniFileName);
StaticClass.IniData.IniFileName = IniFileName;
StaticClass.IniData.InputDir = IniParser.Read("Paths", "Input");
StaticClass.IniData.ProcessingDir = IniParser.Read("Paths", "Processing");
StaticClass.IniData.RepositoryDir = IniParser.Read("Paths", "Repository");
StaticClass.IniData.FinishedDir = IniParser.Read("Paths", "Finished");
StaticClass.IniData.ErrorDir = IniParser.Read("Paths", "Error");
StaticClass.IniData.ModelerRootDir = IniParser.Read("Paths", "ModelerRootDir");
StaticClass.IniData.CPUCores = int.Parse(IniParser.Read("Process", "CPUCores"));
StaticClass.IniData.ExecutionLimit = int.Parse(IniParser.Read("Process", "ExecutionLimit"));
StaticClass.IniData.StartPort = int.Parse(IniParser.Read("Process", "StartPort"));
StaticClass.IniData.StatusLogFile = IniParser.Read("Process", "StatusLogFile");
StaticClass.IniData.ProcessLogFile = IniParser.Read("Process", "ProcessLogFile");
string scanWaitTime = IniParser.Read("Process", "ScanWaitTime");
StaticClass.IniData.ScanWaitTime = int.Parse(scanWaitTime.Substring(0, scanWaitTime.IndexOf("#")));
string timeLimitString = IniParser.Read("Process", "MaxJobTimeLimit");
StaticClass.IniData.MaxJobTimeLimit = double.Parse(timeLimitString.Substring(0, timeLimitString.IndexOf("#")));
string logFileHistoryLimit = IniParser.Read("Process", "LogFileHistoryLimit");
StaticClass.IniData.LogFileHistoryLimit = int.Parse(logFileHistoryLimit.Substring(0, logFileHistoryLimit.IndexOf("#")));
string inputBufferTimeLimit = IniParser.Read("Process", "InputbufferTimeLimit");
StaticClass.IniData.InputBufferTimeLimit = int.Parse(inputBufferTimeLimit.Substring(0, inputBufferTimeLimit.IndexOf("#")));
string logFileMaxSize = IniParser.Read("Process", "logFileMaxSize");
StaticClass.IniData.LogFileMaxSize = int.Parse(logFileMaxSize.Substring(0, logFileMaxSize.IndexOf("#")));
// Set the static class data needed for global use
StaticClass.ScanWaitTime = StaticClass.IniData.ScanWaitTime;
StaticClass.LogFileSizeLimit = StaticClass.IniData.LogFileMaxSize;
StaticClass.MaxJobTimeLimitSeconds = StaticClass.IniData.MaxJobTimeLimit * 60.0 * 60.0;
// Set the file logging object handle only once here
LoggingToFile loggingToFile = new LoggingToFile(StaticClass.IniData.ProcessLogFile);
StaticClass.FileLoggerObject = loggingToFile;
// Output the Data.ini informatino found
StaticClass.Log($"\nConfig.ini data found:\n");
StaticClass.Log($"Input Dir : {StaticClass.IniData.InputDir}");
StaticClass.Log($"Processing Dir : {StaticClass.IniData.ProcessingDir}");
StaticClass.Log($"Repository Dir : {StaticClass.IniData.RepositoryDir}");
StaticClass.Log($"Finished Dir : {StaticClass.IniData.FinishedDir}");
StaticClass.Log($"Error Dir : {StaticClass.IniData.ErrorDir}");
StaticClass.Log($"Modeler Root Dir : {StaticClass.IniData.ModelerRootDir}");
StaticClass.Log($"Status Log File : {StaticClass.IniData.StatusLogFile}");
StaticClass.Log($"Process Log File : {StaticClass.IniData.ProcessLogFile}");
StaticClass.Log($"CPU Cores : {StaticClass.IniData.CPUCores} Cores");
StaticClass.Log($"Execution Limit : {StaticClass.IniData.ExecutionLimit} Jobs");
StaticClass.Log($"Start Port : {StaticClass.IniData.StartPort}");
StaticClass.Log($"Scan Wait Time : {StaticClass.IniData.ScanWaitTime} Miliseconds");
StaticClass.Log($"Max Job Time Limit : {StaticClass.IniData.MaxJobTimeLimit} Hours");
StaticClass.Log($"Log File History Limit : {StaticClass.IniData.LogFileHistoryLimit} Days");
StaticClass.Log($"Log File Max Size : {StaticClass.IniData.LogFileMaxSize} Megabytes");
}
/// <summary>
/// Pause the Monitor System
/// </summary>
public void PauseMonitor()
{
StaticClass.Log(string.Format("Putting the system into Pause Mode at {0:HH:mm:ss.fff}", DateTime.Now));
StaticClass.PauseFlag = true;
}
/// <summary>
/// Read Status Data from CSV File
/// </summary>
/// <returns>Status List from CSV</returns>
public List <StatusData> ReadFromCsvFile()
{
List <StatusData> statusDataTable = new List <StatusData>();
string statusLogFile = StaticClass.IniData.StatusLogFile;
if (File.Exists(statusLogFile) == true)
{
using (CsvFileReader reader = new CsvFileReader(statusLogFile))
{
if (reader == null)
{
StaticClass.Logger.LogError("ReadFromCsvFile reader failed to instantiate");
return(null);
}
CsvRow rowData = new CsvRow();
while (reader.ReadRow(rowData))
{
StatusData rowStatusData = new StatusData();
if (rowStatusData == null)
{
StaticClass.Logger.LogError("ReadFromCsvFile rowStatusData failed to instantiate");
return(null);
}
rowStatusData.Job = rowData[0];
string jobType = rowData[1];
switch (jobType)
{
case "JOB_STARTED":
rowStatusData.JobStatus = JobStatus.JOB_STARTED;
break;
case "EXECUTING":
rowStatusData.JobStatus = JobStatus.EXECUTING;
break;
case "MONITORING_INPUT":
rowStatusData.JobStatus = JobStatus.MONITORING_INPUT;
break;
case "COPYING_TO_PROCESSING":
rowStatusData.JobStatus = JobStatus.COPYING_TO_PROCESSING;
break;
case "MONITORING_PROCESSING":
rowStatusData.JobStatus = JobStatus.MONITORING_PROCESSING;
break;
case "MONITORING_TCPIP":
rowStatusData.JobStatus = JobStatus.MONITORING_TCPIP;
break;
case "COPYING_TO_ARCHIVE":
rowStatusData.JobStatus = JobStatus.COPYING_TO_ARCHIVE;
break;
case "JOB_TIMEOUT":
rowStatusData.JobStatus = JobStatus.JOB_TIMEOUT;
break;
case "COMPLETE":
rowStatusData.JobStatus = JobStatus.COMPLETE;
break;
}
// Get Time Recieved
if (rowData[2] == "1/1/0001 12:00:00 AM")
{
rowStatusData.TimeReceived = DateTime.MinValue;
}
else
{
rowStatusData.TimeReceived = Convert.ToDateTime(rowData[2]);
}
// Get Time Started
if (rowData[3] == "1/1/0001 12:00:00 AM")
{
rowStatusData.TimeStarted = DateTime.MinValue;
}
else
{
rowStatusData.TimeStarted = Convert.ToDateTime(rowData[3]);
}
// Get Time Complete
if (rowData[4] == "1/1/0001 12:00:00 AM")
{
rowStatusData.TimeCompleted = DateTime.MinValue;
}
else
{
rowStatusData.TimeCompleted = Convert.ToDateTime(rowData[4]);
}
// Add data to status table
statusDataTable.Add(rowStatusData);
// If the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("ReadFromCsvFile") == true)
{
return(null);
}
}
}
// Return status table list
return(statusDataTable);
}
return(null);
}
/// <summary>
/// Read, delete old data and rewrite Log File
/// </summary>
public void CheckLogFileHistory()
{
List <StatusData> statusDataTable = new List <StatusData>();
string logFileName = StaticClass.IniData.StatusLogFile;
if (File.Exists(logFileName) == true)
{
using (CsvFileReader reader = new CsvFileReader(logFileName))
{
if (reader == null)
{
StaticClass.Logger.LogError("CsvFileHandler reader failed to instantiate");
return;
}
CsvRow rowData = new CsvRow();
if (rowData == null)
{
StaticClass.Logger.LogError("CsvFileHandler rowData failed to instantiate");
return;
}
while (reader.ReadRow(rowData))
{
StatusData rowStatusData = new StatusData();
if (rowStatusData == null)
{
StaticClass.Logger.LogError("CsvFileHandler rowStatusData failed to instantiate");
return;
}
bool oldRecord = false;
string job = rowStatusData.Job = rowData[0];
string jobType = rowData[1];
switch (jobType)
{
case "JOB_STARTED":
rowStatusData.JobStatus = JobStatus.JOB_STARTED;
break;
case "EXECUTING":
rowStatusData.JobStatus = JobStatus.EXECUTING;
break;
case "MONITORING_INPUT":
rowStatusData.JobStatus = JobStatus.MONITORING_INPUT;
break;
case "COPYING_TO_PROCESSING":
rowStatusData.JobStatus = JobStatus.COPYING_TO_PROCESSING;
break;
case "MONITORING_PROCESSING":
rowStatusData.JobStatus = JobStatus.MONITORING_PROCESSING;
break;
case "MONITORING_TCPIP":
rowStatusData.JobStatus = JobStatus.MONITORING_TCPIP;
break;
case "COPYING_TO_ARCHIVE":
rowStatusData.JobStatus = JobStatus.COPYING_TO_ARCHIVE;
break;
case "JOB_TIMEOUT":
rowStatusData.JobStatus = JobStatus.JOB_TIMEOUT;
break;
case "COMPLETE":
rowStatusData.JobStatus = JobStatus.COMPLETE;
break;
}
// Check Time Received if older than history limit
DateTime timeReceived = Convert.ToDateTime(rowData[2]);
if ((timeReceived < DateTime.Now.AddDays(-StaticClass.IniData.LogFileHistoryLimit)) && (timeReceived != DateTime.MinValue))
{
oldRecord = true;
}
else
{
rowStatusData.TimeReceived = Convert.ToDateTime(rowData[2]);
}
// Check Time Started if older than history limit
DateTime timeStarted = Convert.ToDateTime(rowData[3]);
if ((timeStarted < DateTime.Now.AddDays(-StaticClass.IniData.LogFileHistoryLimit)) && (timeStarted != DateTime.MinValue))
{
oldRecord = true;
}
else
{
rowStatusData.TimeStarted = Convert.ToDateTime(rowData[3]);
}
// Check Time Complete if older than history limit
DateTime timeCompleted = Convert.ToDateTime(rowData[4]);
if ((timeCompleted < DateTime.Now.AddDays(-StaticClass.IniData.LogFileHistoryLimit)) && (timeStarted != DateTime.MinValue))
{
oldRecord = true;
}
else
{
rowStatusData.TimeCompleted = Convert.ToDateTime(rowData[4]);
}
// Add data to status table if not rejected as old
if (oldRecord == false)
{
statusDataTable.Add(rowStatusData);
}
// If the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("CheckLogFileHistory") == true)
{
return;
}
}
}
// Create new csv file with new data
using (TextWriter writer = new StreamWriter(logFileName))
{
for (int i = 0; i < statusDataTable.Count; i++)
{
writer.WriteLine("{0},{1},{2},{3},{4}",
statusDataTable[i].Job,
statusDataTable[i].JobStatus.ToString(),
statusDataTable[i].TimeReceived,
statusDataTable[i].TimeStarted,
statusDataTable[i].TimeCompleted);
}
writer.Close();
}
}
}
/// <summary>
/// Method to scan for unfinished jobs in the Processing Buffer
/// </summary>
public void CheckForUnfinishedProcessingJobs()
{
// Register with the Processing Buffer Jobs completion event and start its thread
DirectoryInfo processingDirectoryInfo = new DirectoryInfo(StaticClass.IniData.ProcessingDir);
if (processingDirectoryInfo == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread processingDirectoryInfo failed to instantiate");
}
// Get the current list of directories from the Job Processing Buffer
List <DirectoryInfo> processingDirectoryInfoList = processingDirectoryInfo.EnumerateDirectories().ToList();
if (processingDirectoryInfoList == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread processingDirectoryInfoList failed to instantiate");
}
StaticClass.Log("\nChecking for unfinished Processing Jobs...");
if (processingDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nUnfinished Processing Jobs waiting...");
}
else
{
StaticClass.Log("\nNo unfinished Processing Jobs waiting...");
}
// Start the jobs in the directory list found for the Processing Buffer
foreach (DirectoryInfo dirInfo in processingDirectoryInfoList.Reverse <DirectoryInfo>())
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
string directory = dirInfo.ToString();
string job = directory.ToString().Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
StaticClass.Log(string.Format("Starting Processing Job {0} at {1:HH:mm:ss.fff}", directory, DateTime.Now));
// Remove job run from Processing Job list
lock (RemoveLock)
{
if (processingDirectoryInfoList.Remove(dirInfo) == false)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread failed to remove Job {0} from Processing Job list", job);
}
}
StaticClass.Log(string.Format("Processing Directory Watcher removed Job {0} from directory list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Reset Processing file scan flag
StaticClass.ProcessingFileScanComplete[job] = false;
// Start a Processing Buffer Job
StartProcessingJob(directory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
if (processingDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nMore unfinished Processing Jobs then Execution Slots available...");
}
// Sort the Processing Buffer directory list by older dates first
processingDirectoryInfoList.Sort((x, y) => - x.LastAccessTime.CompareTo(y.LastAccessTime));
// Add the jobs in the directory list to the Processing Buffer Jobs to run list
foreach (DirectoryInfo dirInfo in processingDirectoryInfoList)
{
string directory = dirInfo.ToString();
string job = directory.Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
ProcessingJobsToRun.Add(job);
int index = ProcessingJobsToRun.IndexOf(job);
StaticClass.Log(string.Format("Unfinished Processing jobs check added new Job {0} to Processing Job List index {1} at {2:HH:mm:ss.fff}",
job, index, DateTime.Now));
}
// Clear the Directory Info List after done with it
processingDirectoryInfoList.Clear();
// Run check loop until all unfinished Processing jobs are complete
do
{
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("Processing Jobs Scan Thread") == true)
{
return;
}
// Run any unfinished Processing jobs
RunUnfinishedProcessingJobs();
Thread.Yield();
}while (ProcessingJobsToRun.Count > 0);
// Scan of the Processing Buffer jobs is complete
StaticClass.UnfinishedProcessingJobsScanComplete = true;
}