/// <summary>
/// Returns when a file is ready to access
/// </summary>
/// <param name="fileName"></param>
/// <returns>Returns if file is ready to access</returns>
public static bool CheckFileReady(string fileName)
{
int numOfRetries = 0;
do
{
try
{
// Check that a file is both readable and writeable
using (FileStream fileService = File.Open(fileName, FileMode.Open))
{
if (fileService.CanRead && fileService.CanWrite)
{
Log(string.Format("File {0} is available at {1:HH:mm:ss.fff}", fileName, DateTime.Now));
return(true);
}
}
}
catch (UnauthorizedAccessException e)
{
Log(string.Format("File {0} not available exception {1} at {2:HH:mm:ss.fff}", fileName, e, DateTime.Now));
}
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("IsFileReady") == true)
{
return(false);
}
Thread.Yield();
}while (numOfRetries++ < StaticClass.NUM_XML_ACCESS_RETRIES);
return(false);
}
/// <summary>
/// Check if the Modeler has deposited the OverallResult entry in the job data.xml file
/// </summary>
/// <param name="dataXmlFileName"></param>
/// <returns></returns>
public bool OverallResultEntryCheck(string dataXmlFileName)
{
int numOfRetries = 0;
do
{
// Check for data.xml file to be ready
if (StaticClass.CheckFileReady(dataXmlFileName))
{
// Check if the OverallResult node exists
XmlDocument dataXmlDoc = new XmlDocument();
dataXmlDoc.Load(dataXmlFileName);
XmlNode OverallResult = dataXmlDoc.DocumentElement.SelectSingleNode("/Data/OverallResult/result");
if (OverallResult != null)
{
return(true);
}
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("Overall Result Entry Check") == true)
{
return(false);
}
Thread.Sleep(StaticClass.FILE_WAIT_DELAY);
}
}while (numOfRetries++ < StaticClass.NUM_RESULTS_ENTRY_RETRIES);
StaticClass.Log(string.Format("File {0} did not have Overall Result entry even after {1} retries at {2:HH:mm:ss.fff}",
dataXmlFileName, StaticClass.NUM_RESULTS_ENTRY_RETRIES, DateTime.Now));
return(false);
}
// Define the event handlers.
/// <summary>
/// The directory created callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string jobDirectory = e.FullPath;
string job = jobDirectory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
StaticClass.Log(string.Format("\nInput Directory Watcher checking new Job {0} for Input Job list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
if (StaticClass.ShutDownPauseCheck("Directory Watcher OnCreated") == false)
{
Thread.Sleep(StaticClass.WAIT_FOR_FILES_TO_COMPLETE);
// Check directory contents complete
if (StaticClass.CheckDirectoryReady(jobDirectory) == true)
{
StaticClass.AddJobToList(job);
if ((StaticClass.IniData.DebugMode & (byte)DebugModeState.JOB_LIST) != 0)
{
StaticClass.DisplayJobList();
}
}
else
{
StaticClass.Logger.LogError("DirectoryWatcherThread Job {0} directory check failed at {1:HH:mm:ss.fff}",
job, DateTime.Now);
}
}
}
public void RunInputBufferScan(string directory, string job, StatusMonitorData monitorData)
{
// Add initial entry to status list
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_INPUT, JobType.TIME_START);
// Monitor the Input Buffer job directory until it has the total number of consumed files
string inputBufferJobDir = StaticClass.IniData.InputDir;
int numberOfFilesNeeded = monitorData.NumFilesConsumed;
if (Directory.Exists(inputBufferJobDir))
{
string inputJobFileDir = inputBufferJobDir + @"\" + job;
// Register with the File Watcher class event and start its thread
InputFileWatcherThread inputFileWatch = new InputFileWatcherThread(inputJobFileDir, numberOfFilesNeeded);
if (inputFileWatch == null)
{
StaticClass.Logger.LogError("Job Run Thread inputFileWatch failed to instantiate");
}
inputFileWatch.ThreadProc();
// Wait for Input file scan to complete
do
{
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
Thread.Yield();
}while (StaticClass.InputFileScanComplete[job] == false);
StaticClass.Log(string.Format("Finished Input file scan for Job {0} at {1:HH:mm:ss.fff}",
inputJobFileDir, DateTime.Now));
// Add copying entry to status list
StaticClass.StatusDataEntry(job, JobStatus.COPYING_TO_PROCESSING, JobType.TIME_START);
// Move files from Input directory to the Processing directory, creating it first if needed
FileHandling.CopyFolderContents(inputJobFileDir, directory, true, true);
}
else
{
StaticClass.Logger.LogError("Could not find Input Buffer Directory");
throw new InvalidOperationException("Could not find Input Buffer Directory");
}
}
// Define the event handlers.
/// <summary>
/// The directory created callback
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
public void OnCreated(object source, FileSystemEventArgs e)
{
string jobDirectory = e.FullPath;
string job = jobDirectory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
int index = 0;
// Check Input Buffer Job directory to see if it is ready if adding the first jobs
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
StaticClass.Log(string.Format("\nInput Directory Watcher checking new Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Wait some for the directory creation and file copies to complete
Thread.Sleep(StaticClass.DIRECTORY_RECEIVE_WAIT);
StaticClass.CheckJobDirectoryComplete(jobDirectory);
}
// Do Shutdown Pause check
if (StaticClass.ShutDownPauseCheck("Directory Watcher OnCreated") == false)
{
// Loop shutdown/Pause check
Task AddTask = Task.Run(() =>
{
index = StaticClass.InputJobsToRun.Count + 1;
StaticClass.InputJobsToRun.Add(index, job);
StaticClass.Log(string.Format("Input Directory Watcher added new Job {0} to Input Job list index {1} at {2:HH:mm:ss.fff}",
job, index, DateTime.Now));
});
TimeSpan timeSpan = TimeSpan.FromMilliseconds(StaticClass.ADD_JOB_DELAY);
if (!AddTask.Wait(timeSpan))
{
StaticClass.Logger.LogError("DirectoryWatcherThread Add Job {0} timed out at {1:HH:mm:ss.fff}", job, DateTime.Now);
}
}
}
public void WatchDirectory()
{
// Create a new FileSystemWatcher and set its properties
using (FileSystemWatcher watcher = new FileSystemWatcher())
{
if (watcher == null)
{
StaticClass.Logger.LogError("DirectoryWatcherThread watcher failed to instantiate");
}
// Watch for changes in the directory list
watcher.NotifyFilter =
NotifyFilters.DirectoryName |
NotifyFilters.CreationTime;
// Set the Input Buffre as path to watch for new directory additions
watcher.Path = StaticClass.IniData.InputDir;
// Watch for any directories names added
watcher.Filter = "*.*";
watcher.IncludeSubdirectories = true;
// Add OnCreated event handler which is the only one needed
watcher.Created += OnCreated;
watcher.Changed += OnChanged;
// Begin watching for directory changes to Input directory
watcher.EnableRaisingEvents = true;
// Scan Input directory forever
do
{
Thread.Yield();
}while (StaticClass.ShutDownPauseCheck("Directory Watcher") == false);
}
}
/// <summary>
/// Method to scan for unfinished jobs in the Input Buffer
/// </summary>
public void CheckForUnfinishedInputJobs()
{
// Start the Directory Watcher class to scan for new Input Buffer jobs
DirectoryWatcherThread dirWatch = new DirectoryWatcherThread();
if (dirWatch == null)
{
StaticClass.Logger.LogError("InputJobsScanThread dirWatch failed to instantiate");
}
dirWatch.ThreadProc();
// Register with the Processing Buffer Jobs check completion event and start its thread
ProcessingJobsScanThread unfinishedProcessingJobs = new ProcessingJobsScanThread();
if (unfinishedProcessingJobs == null)
{
StaticClass.Logger.LogError("InputJobsScanThread currentProcessingJobs failed to instantiate");
}
unfinishedProcessingJobs.ThreadProc();
// Wait while scanning for unfinished Processing jobs
do
{
if (StaticClass.ShutDownPauseCheck("CheckForUnfinishedInputJobs") == true)
{
return;
}
Thread.Yield();
}while (StaticClass.UnfinishedProcessingJobsScanComplete == false);
StaticClass.Log("\nChecking for unfinished Input Jobs...");
DirectoryInfo InputDirectoryInfo = new DirectoryInfo(StaticClass.IniData.InputDir);
if (InputDirectoryInfo == null)
{
StaticClass.Logger.LogError("InputJobsScanThread InputDirectoryInfo failed to instantiate");
}
// Get the current list of directories from the Input Buffer
List <DirectoryInfo> inputDirectoryInfoList = InputDirectoryInfo.EnumerateDirectories().ToList();
if (inputDirectoryInfoList == null)
{
StaticClass.Logger.LogError("InputJobsScanThread inputDirectoryInfoList failed to instantiate");
}
if (inputDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nUnfinished Input Jobs waiting...");
}
else
{
StaticClass.Log("\nNo unfinished Input Jobs found...");
}
// Start the jobs in the directory list found for the Input Buffer
foreach (DirectoryInfo dirInfo in inputDirectoryInfoList.Reverse <DirectoryInfo>())
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
string jobDirectory = dirInfo.ToString();
string job = jobDirectory.ToString().Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
// Check if the directory has a full set of Job files
if (StaticClass.CheckIfJobFilesComplete(jobDirectory) == true)
{
StaticClass.Log(string.Format("Starting Input Job {0} at {1:HH:mm:ss.fff}", jobDirectory, DateTime.Now));
// Remove job run from Input Job directory list
lock (RemoveLock)
{
if (inputDirectoryInfoList.Remove(dirInfo) == false)
{
StaticClass.Logger.LogError("InputJobsScanThread failed to remove Job {0} from Input Job list", job);
}
}
StaticClass.Log(string.Format("Input Directory Watcher removed Job {0} from Input Directory Jobs list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Reset Input file scan flag
StaticClass.InputFileScanComplete[job] = false;
// Start an Input Buffer Job
StartInputJob(jobDirectory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
}
if (inputDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nMore unfinished Input Jobs then Execution Slots available...");
// Sort the Input Buffer directory list by older dates first
inputDirectoryInfoList.Sort((x, y) => - x.LastAccessTime.CompareTo(y.LastAccessTime));
// Add the jobs in the directory list to the Input Buffer Jobs to run list
foreach (DirectoryInfo dirInfo in inputDirectoryInfoList)
{
string directory = dirInfo.ToString();
string job = directory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
StaticClass.AddJobToList(job);
}
// Clear the Directory Info List after done with it
inputDirectoryInfoList.Clear();
// Display new job list
StaticClass.DisplayJobList();
}
StaticClass.Log("\nStarted Watching for new Input Jobs...");
// Run new Input Job watch loop here forever
do
{
// Check if the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("CheckForNewInputJobs") == true)
{
return;
}
// Run any unfinished input jobs
RunInputJobsFound();
// Throttle calling new Jobs Found handler
Thread.Sleep(StaticClass.IniData.ScanWaitTime);
}while (true);
}
/// <summary>
/// Read Status Data from CSV File
/// </summary>
/// <returns>Status List from CSV</returns>
public List <StatusData> ReadFromCsvFile()
{
List <StatusData> statusDataTable = new List <StatusData>();
string statusLogFile = StaticClass.IniData.StatusLogFile;
if (File.Exists(statusLogFile) == true)
{
using (CsvFileReader reader = new CsvFileReader(statusLogFile))
{
if (reader == null)
{
StaticClass.Logger.LogError("ReadFromCsvFile reader failed to instantiate");
return(null);
}
CsvRow rowData = new CsvRow();
while (reader.ReadRow(rowData))
{
StatusData rowStatusData = new StatusData();
if (rowStatusData == null)
{
StaticClass.Logger.LogError("ReadFromCsvFile rowStatusData failed to instantiate");
return(null);
}
rowStatusData.Job = rowData[0];
string jobType = rowData[1];
switch (jobType)
{
case "JOB_STARTED":
rowStatusData.JobStatus = JobStatus.JOB_STARTED;
break;
case "EXECUTING":
rowStatusData.JobStatus = JobStatus.EXECUTING;
break;
case "MONITORING_INPUT":
rowStatusData.JobStatus = JobStatus.MONITORING_INPUT;
break;
case "COPYING_TO_PROCESSING":
rowStatusData.JobStatus = JobStatus.COPYING_TO_PROCESSING;
break;
case "MONITORING_PROCESSING":
rowStatusData.JobStatus = JobStatus.MONITORING_PROCESSING;
break;
case "MONITORING_TCPIP":
rowStatusData.JobStatus = JobStatus.MONITORING_TCPIP;
break;
case "COPYING_TO_ARCHIVE":
rowStatusData.JobStatus = JobStatus.COPYING_TO_ARCHIVE;
break;
case "JOB_TIMEOUT":
rowStatusData.JobStatus = JobStatus.JOB_TIMEOUT;
break;
case "COMPLETE":
rowStatusData.JobStatus = JobStatus.COMPLETE;
break;
}
// Get Time Recieved
if (rowData[2] == "1/1/0001 12:00:00 AM")
{
rowStatusData.TimeReceived = DateTime.MinValue;
}
else
{
rowStatusData.TimeReceived = Convert.ToDateTime(rowData[2]);
}
// Get Time Started
if (rowData[3] == "1/1/0001 12:00:00 AM")
{
rowStatusData.TimeStarted = DateTime.MinValue;
}
else
{
rowStatusData.TimeStarted = Convert.ToDateTime(rowData[3]);
}
// Get Time Complete
if (rowData[4] == "1/1/0001 12:00:00 AM")
{
rowStatusData.TimeCompleted = DateTime.MinValue;
}
else
{
rowStatusData.TimeCompleted = Convert.ToDateTime(rowData[4]);
}
// Add data to status table
statusDataTable.Add(rowStatusData);
// If the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("ReadFromCsvFile") == true)
{
return(null);
}
}
}
// Return status table list
return(statusDataTable);
}
return(null);
}
/// <summary>
/// Read, delete old data and rewrite Log File
/// </summary>
public void CheckLogFileHistory()
{
List <StatusData> statusDataTable = new List <StatusData>();
string logFileName = StaticClass.IniData.StatusLogFile;
if (File.Exists(logFileName) == true)
{
using (CsvFileReader reader = new CsvFileReader(logFileName))
{
if (reader == null)
{
StaticClass.Logger.LogError("CsvFileHandler reader failed to instantiate");
return;
}
CsvRow rowData = new CsvRow();
if (rowData == null)
{
StaticClass.Logger.LogError("CsvFileHandler rowData failed to instantiate");
return;
}
while (reader.ReadRow(rowData))
{
StatusData rowStatusData = new StatusData();
if (rowStatusData == null)
{
StaticClass.Logger.LogError("CsvFileHandler rowStatusData failed to instantiate");
return;
}
bool oldRecord = false;
string job = rowStatusData.Job = rowData[0];
string jobType = rowData[1];
switch (jobType)
{
case "JOB_STARTED":
rowStatusData.JobStatus = JobStatus.JOB_STARTED;
break;
case "EXECUTING":
rowStatusData.JobStatus = JobStatus.EXECUTING;
break;
case "MONITORING_INPUT":
rowStatusData.JobStatus = JobStatus.MONITORING_INPUT;
break;
case "COPYING_TO_PROCESSING":
rowStatusData.JobStatus = JobStatus.COPYING_TO_PROCESSING;
break;
case "MONITORING_PROCESSING":
rowStatusData.JobStatus = JobStatus.MONITORING_PROCESSING;
break;
case "MONITORING_TCPIP":
rowStatusData.JobStatus = JobStatus.MONITORING_TCPIP;
break;
case "COPYING_TO_ARCHIVE":
rowStatusData.JobStatus = JobStatus.COPYING_TO_ARCHIVE;
break;
case "JOB_TIMEOUT":
rowStatusData.JobStatus = JobStatus.JOB_TIMEOUT;
break;
case "COMPLETE":
rowStatusData.JobStatus = JobStatus.COMPLETE;
break;
}
// Check Time Received if older than history limit
DateTime timeReceived = Convert.ToDateTime(rowData[2]);
if ((timeReceived < DateTime.Now.AddDays(-StaticClass.IniData.LogFileHistoryLimit)) && (timeReceived != DateTime.MinValue))
{
oldRecord = true;
}
else
{
rowStatusData.TimeReceived = Convert.ToDateTime(rowData[2]);
}
// Check Time Started if older than history limit
DateTime timeStarted = Convert.ToDateTime(rowData[3]);
if ((timeStarted < DateTime.Now.AddDays(-StaticClass.IniData.LogFileHistoryLimit)) && (timeStarted != DateTime.MinValue))
{
oldRecord = true;
}
else
{
rowStatusData.TimeStarted = Convert.ToDateTime(rowData[3]);
}
// Check Time Complete if older than history limit
DateTime timeCompleted = Convert.ToDateTime(rowData[4]);
if ((timeCompleted < DateTime.Now.AddDays(-StaticClass.IniData.LogFileHistoryLimit)) && (timeStarted != DateTime.MinValue))
{
oldRecord = true;
}
else
{
rowStatusData.TimeCompleted = Convert.ToDateTime(rowData[4]);
}
// Add data to status table if not rejected as old
if (oldRecord == false)
{
statusDataTable.Add(rowStatusData);
}
// If the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("CheckLogFileHistory") == true)
{
return;
}
}
}
// Create new csv file with new data
using (TextWriter writer = new StreamWriter(logFileName))
{
for (int i = 0; i < statusDataTable.Count; i++)
{
writer.WriteLine("{0},{1},{2},{3},{4}",
statusDataTable[i].Job,
statusDataTable[i].JobStatus.ToString(),
statusDataTable[i].TimeReceived,
statusDataTable[i].TimeStarted,
statusDataTable[i].TimeCompleted);
}
writer.Close();
}
}
}
/// <summary>
/// Method to scan for unfinished jobs in the Processing Buffer
/// </summary>
public void CheckForUnfinishedProcessingJobs()
{
// Register with the Processing Buffer Jobs completion event and start its thread
DirectoryInfo processingDirectoryInfo = new DirectoryInfo(StaticClass.IniData.ProcessingDir);
if (processingDirectoryInfo == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread processingDirectoryInfo failed to instantiate");
}
// Get the current list of directories from the Job Processing Buffer
List <DirectoryInfo> processingDirectoryInfoList = processingDirectoryInfo.EnumerateDirectories().ToList();
if (processingDirectoryInfoList == null)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread processingDirectoryInfoList failed to instantiate");
}
StaticClass.Log("\nChecking for unfinished Processing Jobs...");
if (processingDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nUnfinished Processing Jobs waiting...");
}
else
{
StaticClass.Log("\nNo unfinished Processing Jobs waiting...");
}
// Start the jobs in the directory list found for the Processing Buffer
foreach (DirectoryInfo dirInfo in processingDirectoryInfoList.Reverse <DirectoryInfo>())
{
if (StaticClass.NumberOfJobsExecuting < StaticClass.IniData.ExecutionLimit)
{
string directory = dirInfo.ToString();
string job = directory.ToString().Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
StaticClass.Log(string.Format("Starting Processing Job {0} at {1:HH:mm:ss.fff}", directory, DateTime.Now));
// Remove job run from Processing Job list
lock (RemoveLock)
{
if (processingDirectoryInfoList.Remove(dirInfo) == false)
{
StaticClass.Logger.LogError("ProcessingJobsScanThread failed to remove Job {0} from Processing Job list", job);
}
}
StaticClass.Log(string.Format("Processing Directory Watcher removed Job {0} from directory list at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Reset Processing file scan flag
StaticClass.ProcessingFileScanComplete[job] = false;
// Start a Processing Buffer Job
StartProcessingJob(directory);
// Throttle the Job startups
Thread.Sleep(StaticClass.ScanWaitTime);
}
}
if (processingDirectoryInfoList.Count > 0)
{
StaticClass.Log("\nMore unfinished Processing Jobs then Execution Slots available...");
}
// Sort the Processing Buffer directory list by older dates first
processingDirectoryInfoList.Sort((x, y) => - x.LastAccessTime.CompareTo(y.LastAccessTime));
// Add the jobs in the directory list to the Processing Buffer Jobs to run list
foreach (DirectoryInfo dirInfo in processingDirectoryInfoList)
{
string directory = dirInfo.ToString();
string job = directory.Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
ProcessingJobsToRun.Add(job);
int index = ProcessingJobsToRun.IndexOf(job);
StaticClass.Log(string.Format("Unfinished Processing jobs check added new Job {0} to Processing Job List index {1} at {2:HH:mm:ss.fff}",
job, index, DateTime.Now));
}
// Clear the Directory Info List after done with it
processingDirectoryInfoList.Clear();
// Run check loop until all unfinished Processing jobs are complete
do
{
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("Processing Jobs Scan Thread") == true)
{
return;
}
// Run any unfinished Processing jobs
RunUnfinishedProcessingJobs();
Thread.Yield();
}while (ProcessingJobsToRun.Count > 0);
// Scan of the Processing Buffer jobs is complete
StaticClass.UnfinishedProcessingJobsScanComplete = true;
}
/// <summary>
/// Run a job from Input or Processing Buffers
/// </summary>
/// <param name="dirScanType"></param>
/// <param name="jobXmlData"></param>
public void RunJob(JobXmlData jobXmlData, DirectoryScanType dirScanType)
{
// Increment number of Jobs executing in only one place!
StaticClass.NumberOfJobsExecuting++;
// Create the Job Run common strings
string job = jobXmlData.Job;
string xmlJobDirectory = jobXmlData.JobDirectory;
string processingBufferDirectory = StaticClass.IniData.ProcessingDir;
string processingBufferJobDir = processingBufferDirectory + @"\" + job;
// Set the job start time
StaticClass.JobStartTime[job] = DateTime.Now;
// Create new status monitor data and fill it in with the job xml data
StatusMonitorData monitorData = new StatusMonitorData
{
Job = job,
JobDirectory = xmlJobDirectory,
StartTime = DateTime.Now,
JobSerialNumber = jobXmlData.JobSerialNumber,
TimeStamp = jobXmlData.TimeStamp,
XmlFileName = jobXmlData.XmlFileName
};
// Add initial entry to status list
StaticClass.StatusDataEntry(job, JobStatus.JOB_STARTED, JobType.TIME_RECEIVED);
// Get the Job xml data
monitorData = GetJobXmlData(jobXmlData, monitorData);
// If this job comes from the Input directory, run the Input job check and start job
if (dirScanType == DirectoryScanType.INPUT_BUFFER)
{
RunInputBufferScan(processingBufferJobDir, job, monitorData);
}
// If the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
// Add entry to status list
StaticClass.StatusDataEntry(job, JobStatus.EXECUTING, JobType.TIME_START);
StaticClass.Log(string.Format("Starting Job {0} with Modeler {1} on Port {2} with {3} CPU's at {4:HH:mm:ss.fff}",
job, monitorData.Modeler, monitorData.JobPortNumber, StaticClass.IniData.CPUCores, DateTime.Now));
// Execute Modeler using the command line generator
string executable = StaticClass.IniData.ModelerRootDir + @"\" + monitorData.Modeler + @"\" + monitorData.Modeler + ".exe";
string processingBuffer = processingBufferJobDir;
int port = monitorData.JobPortNumber;
int cpuCores = StaticClass.IniData.CPUCores;
CommandLineGenerator cmdLineGenerator = new CommandLineGenerator(
executable, processingBuffer, port, cpuCores);
if (cmdLineGenerator == null)
{
StaticClass.Logger.LogError("JobRunThread cmdLineGenerator failed to instantiate");
}
Process modelerProcess = cmdLineGenerator.ExecuteCommand(job);
// Register with the Processing File Watcher class and start its thread
ProcessingFileWatcherThread processingFileWatcher = new ProcessingFileWatcherThread(processingBufferJobDir, monitorData);
if (processingFileWatcher == null)
{
StaticClass.Logger.LogError("JobRunThread ProcessingFileWatch failed to instantiate");
}
processingFileWatcher.ThreadProc();
// Start the TCP/IP Communications thread before checking for Processing job files
TcpIpListenThread tcpIpThread = new TcpIpListenThread(monitorData);
if (tcpIpThread == null)
{
StaticClass.Logger.LogError("ProcessingFileWatcherThread tcpIpThread failed to instantiate");
}
tcpIpThread.ThreadProc();
// Add entry to status list
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_PROCESSING, JobType.TIME_START);
// Wait 45 seconds for Modeler to get started before reading it's information
Thread.Sleep(StaticClass.DISPLAY_PROCESS_DATA_WAIT);
// Display the Modeler Process information
DisplayProcessInfo(job, modelerProcess);
// Wait for the Processing job scan complete or shut down
do
{
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
Thread.Yield();
}while ((StaticClass.ProcessingJobScanComplete[job] == false) && (StaticClass.JobShutdownFlag[job] == false));
// Wait to make sure the data.xml is done being handled
Thread.Sleep(StaticClass.POST_PROCESS_WAIT);
if (StaticClass.JobShutdownFlag[job] == false)
{
// Wait for the data.xml file to contain a result
string dataXmlFileName = processingBufferJobDir + @"\" + "data.xml";
if (OverallResultEntryCheck(dataXmlFileName) == true)
{
StaticClass.Log(string.Format("Overall Results check confirmed for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
}
else
{
StaticClass.Log(string.Format("Overall Results check failed for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
}
}
// Make sure Modeler Process is stopped
if (StaticClass.ProcessHandles[job].HasExited == false)
{
StaticClass.Log(string.Format("Shutting down Modeler Executable for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
StaticClass.ProcessHandles[job].Kill();
StaticClass.ProcessHandles.Remove(job);
// Wait for Process to end
Thread.Sleep(StaticClass.SHUTDOWN_PROCESS_WAIT);
}
// Run the Job Complete handler
RunJobFileProcessing(job, monitorData);
// Add entry to status list
StaticClass.StatusDataEntry(job, JobStatus.COMPLETE, JobType.TIME_COMPLETE);
// Show Job Complete message
TimeSpan timeSpan = DateTime.Now - StaticClass.JobStartTime[job];
StaticClass.Log(string.Format("Job {0} Complete taking {1:hh\\:mm\\:ss}. Decrementing Job count to {2} at {3:HH:mm:ss.fff}",
job, timeSpan, StaticClass.NumberOfJobsExecuting - 1, DateTime.Now));
// Decrement the number of Jobs executing in one place!
StaticClass.NumberOfJobsExecuting--;
}
/// <summary>
/// Connect to TCP/IP Port
/// </summary>
/// <param name="port"></param>
/// <param name="server"></param>
/// <param name="monitorData"></pram>
public void Connect(int port, string server, StatusMonitorData monitorData)
{
ModelerStepState ModelerCurrentStepState = ModelerStepState.NONE;
string job = monitorData.Job;
// Wait about a minute for the Modeler to start execution
Thread.Sleep(StaticClass.TCP_IP_STARTUP_WAIT);
StaticClass.Log(string.Format("\nStarting TCP/IP Scan for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Log starting TCP/IP monitoring entry
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_TCPIP, JobType.TIME_START);
try
{
// Create a TcpClient
TcpClient client = new TcpClient(server, port);
if (client == null)
{
StaticClass.Logger.LogError("TcpIp Connectinon client failed to instantiate");
}
// Get a client stream for reading and writing
NetworkStream stream = client.GetStream();
if (stream == null)
{
StaticClass.Logger.LogError("TcpIp Connection stream handle was not gotten from client");
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Show connection and start sending messages
StaticClass.Log(string.Format("Connected to Modeler TCP/IP for Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now));
bool jobComplete = false;
int numRequestSent = 0;
do
{
// Loop shutdown/Pause check
if (StaticClass.ShutDownPauseCheck("TCP/IP Connect") == true)
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Check if the stream is writable, if not, exit
if (stream.CanWrite)
{
if (StaticClass.ShutDownPauseCheck(job) == false)
{
try
{
// Translate the passed message into ASCII and send it as a byte array
byte[] sendData = Encoding.ASCII.GetBytes(StatusMessage);
stream.Write(sendData, 0, sendData.Length);
numRequestSent++;
}
catch (IOException e)
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} because of Exception {2} at {3:HH:mm:ss.fff}",
job, port, e, DateTime.Now));
// Signal job complete if exception happend in Step 5 or 6
if ((ModelerCurrentStepState == ModelerStepState.STEP_5) ||
(ModelerCurrentStepState == ModelerStepState.STEP_6))
{
// Set the TCP/IP Scan complete flag to signal the RunJob thread
StaticClass.TcpIpScanComplete[job] = true;
}
else
{
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
}
return;
}
StaticClass.Log(string.Format("\nSending {0} msg to Modeler for Job {1} on Port {2} at {3:HH:mm:ss.fff}",
StatusMessage, job, port, DateTime.Now));
}
}
else
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("\nTCP/IP stream closed for Modeler Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Check if TCP/IP stream is readable and data is available
int adjustableSleepTime = StaticClass.STARTING_TCP_IP_WAIT;
int tcpIpRetryCount = 0;
bool messageReceived = false;
do
{
if (stream.CanRead && stream.DataAvailable)
{
// Buffers to store the response
byte[] recvData = new byte[256];
string responseData = string.Empty;
int bytes = stream.Read(recvData, 0, recvData.Length);
responseData = Encoding.ASCII.GetString(recvData, 0, bytes);
StaticClass.Log(string.Format("Received: {0} from Job {1} on Port {2} at {3:HH:mm:ss.fff}",
responseData, job, port, DateTime.Now));
// Readjust sleep time according to Step number
switch (responseData)
{
case "Step 1 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_1;
if (numRequestSent < StaticClass.NUM_REQUESTS_TILL_TCPIP_SLOWDOWN)
{
adjustableSleepTime = 15000;
}
else
{
adjustableSleepTime = 60000;
}
break;
case "Step 2 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_2;
adjustableSleepTime = 10000;
break;
case "Step 3 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_3;
adjustableSleepTime = 7500;
break;
case "Step 4 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_4;
adjustableSleepTime = 5000;
break;
case "Step 5 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_5;
adjustableSleepTime = 2500;
break;
case "Step 6 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_6;
adjustableSleepTime = 250;
break;
case "Whole process done, socket closed.":
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("TCP/IP for Job {0} on Port {1} received Modeler process done at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
ModelerCurrentStepState = ModelerStepState.STEP_COMPLETE;
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the TCP/IP Scan complete flag to signal the RunJob thread
StaticClass.TcpIpScanComplete[job] = true;
return;
default:
StaticClass.Logger.LogWarning("Received Weird Response: {0} from Job {1} on Port {2} at {3:HH:mm:ss.fff}",
responseData, job, port, DateTime.Now);
break;
}
// Backup check of the process complete string, even if it is concatenated with another string
if (responseData.Contains("Whole process done, socket closed."))
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("TCP/IP for Job {0} on Port {1} received Modeler process complete at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
ModelerCurrentStepState = ModelerStepState.STEP_COMPLETE;
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the TCP/IP Scan complete flag to signal the RunJob thread
StaticClass.TcpIpScanComplete[job] = true;
return;
}
// Check for Job timeout
if ((DateTime.Now - monitorData.StartTime).TotalSeconds > StaticClass.MaxJobTimeLimitSeconds)
{
StaticClass.Log(string.Format("Job Execution Timeout for Job {0} in state {1} at {2:HH:mm:ss.fff}",
job, ModelerCurrentStepState, DateTime.Now));
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Create job Timeout status
StaticClass.StatusDataEntry(job, JobStatus.JOB_TIMEOUT, JobType.TIME_START);
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("TCP/IP Receive") == true)
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Set the messge received flag to exit receive loop
messageReceived = true;
}
else
{
// Wait 250 msec between 480 Data Available checks (2 min) CanRead is set for session
Thread.Sleep(StaticClass.READ_AVAILABLE_RETRY_DELAY);
tcpIpRetryCount++;
}
}while ((tcpIpRetryCount < StaticClass.NUM_TCP_IP_RETRIES) && (messageReceived == false));
// Wait for an adjustable time between TCP/IP status requests
Thread.Sleep(adjustableSleepTime);
}while (jobComplete == false);
// Close everything
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} in state {2} at {3:HH:mm:ss.fff}",
job, port, ModelerCurrentStepState, DateTime.Now));
}
catch (SocketException e)
{
StaticClass.Logger.LogError(string.Format("SocketException {0} for Job {1} on Port {2} in state {3} at {4:HH:mm:ss.fff}",
e, job, port, ModelerCurrentStepState, DateTime.Now));
}
}
public void WatchFiles(string directory)
{
// Get job name from directory name
string job = directory.Replace(StaticClass.IniData.InputDir, "").Remove(0, 1);
if (StaticClass.NumberOfInputFilesFound[job] == StaticClass.NumberOfInputFilesNeeded[job])
{
// Exiting thread message
StaticClass.Log(string.Format("Input File Watcher completed the scan for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Signal the Run thread that the Input files were found
StaticClass.InputFileScanComplete[job] = true;
return;
}
// Create a new FileSystemWatcher and set its properties
using (FileSystemWatcher watcher = new FileSystemWatcher())
{
if (watcher == null)
{
StaticClass.Logger.LogError("InputFileWatcherThread watcher failed to instantiate");
}
// Watch for file changes in the watched directory
watcher.NotifyFilter =
NotifyFilters.FileName |
NotifyFilters.CreationTime |
NotifyFilters.LastAccess;
// Set the Path to scan for files
watcher.Path = directory;
// Watch for any file to get directory changes
watcher.Filter = "*.*";
// Add event handlers
watcher.Created += OnCreated;
watcher.Changed += OnChanged;
// Begin watching for changes to Input directory
watcher.EnableRaisingEvents = true;
StaticClass.Log(string.Format("Input File Watcher watching {0} at {1:HH:mm:ss.fff}",
directory, DateTime.Now));
// Wait for Input file scan to Complete with a full set of job output files
do
{
if (StaticClass.ShutDownPauseCheck("InputFileWatcherThread") == true)
{
return;
}
Thread.Yield();
}while (StaticClass.InputFileScanComplete[job] == false);
// Signal the Input Job Complete flag for the Job
StaticClass.InputJobScanComplete[job] = true;
// Exiting thread message
StaticClass.Log(string.Format("Input File Watcher completed scan for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
}
}
public void WatchFiles(string directory)
{
// Get job name from directory name
string job = directory.Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
// Check to see if the directory is already full
if (StaticClass.NumberOfProcessingFilesFound[job] == StaticClass.NumberOfProcessingFilesNeeded[job])
{
StaticClass.Log(string.Format("Processing File Watcher detected Job {0} has complete file set at {1:HH:mm:ss.fff}",
job, DateTime.Now));
// Signal the Run thread that the Processing files were found
StaticClass.ProcessingFileScanComplete[job] = true;
return;
}
// Create a new FileSystemWatcher and set its properties
using (FileSystemWatcher watcher = new FileSystemWatcher())
{
if (watcher == null)
{
StaticClass.Logger.LogError("ProcessingFileWatcherThread watcher failed to instantiate");
}
// Watch for file changes in the watched directory
watcher.NotifyFilter =
NotifyFilters.FileName |
NotifyFilters.CreationTime |
NotifyFilters.LastAccess;
// Set the Path to scan for files
watcher.Path = directory;
// Watch for any file to get directory changes
watcher.Filter = "*.*";
// Add event handlers
watcher.Created += OnCreated;
watcher.Changed += OnChanged;
// Begin watching for file changes to Processing job directory
watcher.EnableRaisingEvents = true;
StaticClass.Log(string.Format("Processing File Watcher watching {0} at {1:HH:mm:ss.fff}",
directory, DateTime.Now));
// Wait for Processing file scan to Complete with a full set of job output files
do
{
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("ProcessingWatchFiles") == true)
{
return;
}
Thread.Yield();
}while ((StaticClass.ProcessingFileScanComplete[job] == false) || (StaticClass.TcpIpScanComplete[job] == false));
// set the Processing of a Job scan complete flag
StaticClass.ProcessingJobScanComplete[job] = true;
}
}
