public void RunInputBufferScan(string directory, string job, StatusMonitorData monitorData)
{
// Add initial entry to status list
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_INPUT, JobType.TIME_START);
// Monitor the Input Buffer job directory until it has the total number of consumed files
string inputBufferJobDir = StaticClass.IniData.InputDir;
int numberOfFilesNeeded = monitorData.NumFilesConsumed;
if (Directory.Exists(inputBufferJobDir))
{
string inputJobFileDir = inputBufferJobDir + @"\" + job;
// Register with the File Watcher class event and start its thread
InputFileWatcherThread inputFileWatch = new InputFileWatcherThread(inputJobFileDir, numberOfFilesNeeded);
if (inputFileWatch == null)
{
StaticClass.Logger.LogError("Job Run Thread inputFileWatch failed to instantiate");
}
inputFileWatch.ThreadProc();
// Wait for Input file scan to complete
do
{
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
Thread.Yield();
}while (StaticClass.InputFileScanComplete[job] == false);
StaticClass.Log(string.Format("Finished Input file scan for Job {0} at {1:HH:mm:ss.fff}",
inputJobFileDir, DateTime.Now));
// Add copying entry to status list
StaticClass.StatusDataEntry(job, JobStatus.COPYING_TO_PROCESSING, JobType.TIME_START);
// Move files from Input directory to the Processing directory, creating it first if needed
FileHandling.CopyFolderContents(inputJobFileDir, directory, true, true);
}
else
{
StaticClass.Logger.LogError("Could not find Input Buffer Directory");
throw new InvalidOperationException("Could not find Input Buffer Directory");
}
}
/// <summary>
/// Processing directory file watcher thread
/// </summary>
/// <param name="directory"></param>
/// <param name="monitorData"></param>
public ProcessingFileWatcherThread(string directory, StatusMonitorData monitorData)
{
DirectoryName = directory;
Job = directory.Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1);
DirectoryInfo ProcessingJobInfo = new DirectoryInfo(directory);
if (ProcessingJobInfo == null)
{
StaticClass.Logger.LogError("ProcessingFileWatcherThread ProcessingJobInfo failed to instantiate");
}
StaticClass.NumberOfProcessingFilesFound[Job] = ProcessingJobInfo.GetFiles().Length;
StaticClass.NumberOfProcessingFilesNeeded[Job] = monitorData.NumFilesConsumed + monitorData.NumFilesProduced;
StaticClass.TcpIpScanComplete[Job] = false;
StaticClass.ProcessingFileScanComplete[Job] = false;
StaticClass.ProcessingJobScanComplete[Job] = false;
StaticClass.JobShutdownFlag[Job] = false;
}
/// <summary>
/// Get the data from the Job xml file
/// </summary>
/// <param name="jobXmlData"></param>
/// <param name="monitorData"></param>
/// <returns></returns>
public StatusMonitorData GetJobXmlData(JobXmlData jobXmlData, StatusMonitorData monitorData)
{
// Wait for Job xml file to be ready
string jobXmlFileName = jobXmlData.JobDirectory + @"\" + jobXmlData.XmlFileName;
if (StaticClass.CheckFileReady(jobXmlFileName))
{
// Read Job xml file and get the top node
XmlDocument jobXmlDoc = new XmlDocument();
jobXmlDoc.Load(jobXmlFileName);
XmlElement root = jobXmlDoc.DocumentElement;
string TopNode = root.LocalName;
// Get nodes for the number of files and names of files to transfer from Job .xml file
XmlNode UnitNumberdNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/listitem/value");
XmlNode ConsumedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Consumed");
XmlNode ProducedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Produced");
XmlNode TransferedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Transfered");
XmlNode ModelerNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Modeler");
// Assign then increment port number for this Job
monitorData.JobPortNumber = StaticClass.IniData.StartPort + StaticClass.JobPortIndex++;
// Get the modeler and number of files to transfer
int NumFilesToTransfer = 0;
monitorData.UnitNumber = UnitNumberdNode.InnerText;
monitorData.Modeler = ModelerNode.InnerText;
monitorData.NumFilesConsumed = Convert.ToInt32(ConsumedNode.InnerText);
monitorData.NumFilesProduced = Convert.ToInt32(ProducedNode.InnerText);
if (TransferedNode != null)
{
NumFilesToTransfer = Convert.ToInt32(TransferedNode.InnerText);
}
monitorData.NumFilesToTransfer = NumFilesToTransfer;
// Get the modeler and number of files to transfer
StaticClass.Log($"Unit Number : {monitorData.UnitNumber}");
StaticClass.Log($"Modeler : {monitorData.Modeler}");
StaticClass.Log($"Num Files Consumed : {monitorData.NumFilesConsumed}");
StaticClass.Log($"Num Files Produced : {monitorData.NumFilesProduced}");
StaticClass.Log($"Num Files To Transfer : {monitorData.NumFilesToTransfer}");
StaticClass.Log($"Job Port Number : {monitorData.JobPortNumber}");
// Create the Transfered file list from the Xml file entries
monitorData.TransferedFileList = new List <string>(NumFilesToTransfer);
for (int i = 1; i < NumFilesToTransfer + 1; i++)
{
string transferFileNodeName = ("/" + TopNode + "/FileConfiguration/Transfered" + i.ToString());
XmlNode TransferedFileXml = jobXmlDoc.DocumentElement.SelectSingleNode(transferFileNodeName);
monitorData.TransferedFileList.Add(TransferedFileXml.InnerText);
StaticClass.Log(string.Format("Transfer File{0} : {1}", i, TransferedFileXml.InnerText));
}
}
else
{
StaticClass.Logger.LogError("File {0} is not available at {1:HH:mm:ss.fff}\n", jobXmlFileName, DateTime.Now);
}
return(monitorData);
}
/// <summary>
/// Run a job from Input or Processing Buffers
/// </summary>
/// <param name="dirScanType"></param>
/// <param name="jobXmlData"></param>
public void RunJob(JobXmlData jobXmlData, DirectoryScanType dirScanType)
{
// Increment number of Jobs executing in only one place!
StaticClass.NumberOfJobsExecuting++;
// Create the Job Run common strings
string job = jobXmlData.Job;
string xmlJobDirectory = jobXmlData.JobDirectory;
string processingBufferDirectory = StaticClass.IniData.ProcessingDir;
string processingBufferJobDir = processingBufferDirectory + @"\" + job;
// Set the job start time
StaticClass.JobStartTime[job] = DateTime.Now;
// Create new status monitor data and fill it in with the job xml data
StatusMonitorData monitorData = new StatusMonitorData
{
Job = job,
JobDirectory = xmlJobDirectory,
StartTime = DateTime.Now,
JobSerialNumber = jobXmlData.JobSerialNumber,
TimeStamp = jobXmlData.TimeStamp,
XmlFileName = jobXmlData.XmlFileName
};
// Add initial entry to status list
StaticClass.StatusDataEntry(job, JobStatus.JOB_STARTED, JobType.TIME_RECEIVED);
// Get the Job xml data
monitorData = GetJobXmlData(jobXmlData, monitorData);
// If this job comes from the Input directory, run the Input job check and start job
if (dirScanType == DirectoryScanType.INPUT_BUFFER)
{
RunInputBufferScan(processingBufferJobDir, job, monitorData);
}
// If the shutdown flag is set, exit method
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
// Add entry to status list
StaticClass.StatusDataEntry(job, JobStatus.EXECUTING, JobType.TIME_START);
StaticClass.Log(string.Format("Starting Job {0} with Modeler {1} on Port {2} with {3} CPU's at {4:HH:mm:ss.fff}",
job, monitorData.Modeler, monitorData.JobPortNumber, StaticClass.IniData.CPUCores, DateTime.Now));
// Execute Modeler using the command line generator
string executable = StaticClass.IniData.ModelerRootDir + @"\" + monitorData.Modeler + @"\" + monitorData.Modeler + ".exe";
string processingBuffer = processingBufferJobDir;
int port = monitorData.JobPortNumber;
int cpuCores = StaticClass.IniData.CPUCores;
CommandLineGenerator cmdLineGenerator = new CommandLineGenerator(
executable, processingBuffer, port, cpuCores);
if (cmdLineGenerator == null)
{
StaticClass.Logger.LogError("JobRunThread cmdLineGenerator failed to instantiate");
}
Process modelerProcess = cmdLineGenerator.ExecuteCommand(job);
// Register with the Processing File Watcher class and start its thread
ProcessingFileWatcherThread processingFileWatcher = new ProcessingFileWatcherThread(processingBufferJobDir, monitorData);
if (processingFileWatcher == null)
{
StaticClass.Logger.LogError("JobRunThread ProcessingFileWatch failed to instantiate");
}
processingFileWatcher.ThreadProc();
// Start the TCP/IP Communications thread before checking for Processing job files
TcpIpListenThread tcpIpThread = new TcpIpListenThread(monitorData);
if (tcpIpThread == null)
{
StaticClass.Logger.LogError("ProcessingFileWatcherThread tcpIpThread failed to instantiate");
}
tcpIpThread.ThreadProc();
// Add entry to status list
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_PROCESSING, JobType.TIME_START);
// Wait 45 seconds for Modeler to get started before reading it's information
Thread.Sleep(StaticClass.DISPLAY_PROCESS_DATA_WAIT);
// Display the Modeler Process information
DisplayProcessInfo(job, modelerProcess);
// Wait for the Processing job scan complete or shut down
do
{
if (StaticClass.ShutDownPauseCheck("Run Job") == true)
{
return;
}
Thread.Yield();
}while ((StaticClass.ProcessingJobScanComplete[job] == false) && (StaticClass.JobShutdownFlag[job] == false));
// Wait to make sure the data.xml is done being handled
Thread.Sleep(StaticClass.POST_PROCESS_WAIT);
if (StaticClass.JobShutdownFlag[job] == false)
{
// Wait for the data.xml file to contain a result
string dataXmlFileName = processingBufferJobDir + @"\" + "data.xml";
if (OverallResultEntryCheck(dataXmlFileName) == true)
{
StaticClass.Log(string.Format("Overall Results check confirmed for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
}
else
{
StaticClass.Log(string.Format("Overall Results check failed for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
}
}
// Make sure Modeler Process is stopped
if (StaticClass.ProcessHandles[job].HasExited == false)
{
StaticClass.Log(string.Format("Shutting down Modeler Executable for Job {0} at {1:HH:mm:ss.fff}",
job, DateTime.Now));
StaticClass.ProcessHandles[job].Kill();
StaticClass.ProcessHandles.Remove(job);
// Wait for Process to end
Thread.Sleep(StaticClass.SHUTDOWN_PROCESS_WAIT);
}
// Run the Job Complete handler
RunJobFileProcessing(job, monitorData);
// Add entry to status list
StaticClass.StatusDataEntry(job, JobStatus.COMPLETE, JobType.TIME_COMPLETE);
// Show Job Complete message
TimeSpan timeSpan = DateTime.Now - StaticClass.JobStartTime[job];
StaticClass.Log(string.Format("Job {0} Complete taking {1:hh\\:mm\\:ss}. Decrementing Job count to {2} at {3:HH:mm:ss.fff}",
job, timeSpan, StaticClass.NumberOfJobsExecuting - 1, DateTime.Now));
// Decrement the number of Jobs executing in one place!
StaticClass.NumberOfJobsExecuting--;
}
/// <summary>
/// Run the Job Completion file and directory handling
/// </summary>
/// <param name="job"></param>
/// <param name="monitorData"></param>
public void RunJobFileProcessing(string job, StatusMonitorData monitorData)
{
string repositoryDirectory = StaticClass.IniData.RepositoryDir;
string finishedDirectory = StaticClass.IniData.FinishedDir;
string errorDirectory = StaticClass.IniData.ErrorDir;
string processingBufferJobDir = StaticClass.IniData.ProcessingDir + @"\" + job;
string dataXmlFileName = StaticClass.IniData.ProcessingDir + @"\" + job + @"\" + "data.xml";
StaticClass.StatusDataEntry(job, JobStatus.COPYING_TO_ARCHIVE, JobType.TIME_START);
// Get the pass or fail data from the data.xml OverallResult result node
XmlDocument dataXmlDoc = new XmlDocument();
dataXmlDoc.Load(dataXmlFileName);
XmlNode OverallResult = dataXmlDoc.DocumentElement.SelectSingleNode("/Data/OverallResult/result");
string passFail = "Fail";
if (OverallResult != null)
{
passFail = OverallResult.InnerText;
}
string repositoryJobDirectoryName = repositoryDirectory + @"\" + job;
if ((OverallResult != null) && (passFail == "Pass"))
{
// If the Finished directory does not exist, create it
string finishedJobDirectoryName = finishedDirectory + @"\" + monitorData.JobSerialNumber;
if (!Directory.Exists(finishedJobDirectoryName))
{
Directory.CreateDirectory(finishedJobDirectoryName);
}
// Copy the Transfered files to the Finished directory
foreach (string file in monitorData.TransferedFileList)
{
FileHandling.CopyFile(processingBufferJobDir + @"\" + file,
finishedJobDirectoryName + @"\" + file);
}
// Move Processing Buffer Files to the Repository directory when passed
FileHandling.CopyFolderContents(processingBufferJobDir, repositoryJobDirectoryName, true, true);
}
else // Send files to the Error Buffer and repository
{
// If the Error directory does not exist, create it
string errorJobDirectoryName = errorDirectory + @"\" + monitorData.JobSerialNumber;
if (!Directory.Exists(errorJobDirectoryName))
{
Directory.CreateDirectory(errorJobDirectoryName);
}
// Copy the Transfered files to the Error directory
foreach (string file in monitorData.TransferedFileList)
{
if (File.Exists(processingBufferJobDir + @"\" + file))
{
FileHandling.CopyFile(processingBufferJobDir + @"\" + file,
errorJobDirectoryName + @"\" + file);
}
}
// Move Processing Buffer Files to the Repository directory when failed
FileHandling.CopyFolderContents(processingBufferJobDir, repositoryJobDirectoryName, true, true);
}
}
/// <summary>
/// Connect to TCP/IP Port
/// </summary>
/// <param name="port"></param>
/// <param name="server"></param>
/// <param name="monitorData"></pram>
public void Connect(int port, string server, StatusMonitorData monitorData)
{
ModelerStepState ModelerCurrentStepState = ModelerStepState.NONE;
string job = monitorData.Job;
// Wait about a minute for the Modeler to start execution
Thread.Sleep(StaticClass.TCP_IP_STARTUP_WAIT);
StaticClass.Log(string.Format("\nStarting TCP/IP Scan for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Log starting TCP/IP monitoring entry
StaticClass.StatusDataEntry(job, JobStatus.MONITORING_TCPIP, JobType.TIME_START);
try
{
// Create a TcpClient
TcpClient client = new TcpClient(server, port);
if (client == null)
{
StaticClass.Logger.LogError("TcpIp Connectinon client failed to instantiate");
}
// Get a client stream for reading and writing
NetworkStream stream = client.GetStream();
if (stream == null)
{
StaticClass.Logger.LogError("TcpIp Connection stream handle was not gotten from client");
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Show connection and start sending messages
StaticClass.Log(string.Format("Connected to Modeler TCP/IP for Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now));
bool jobComplete = false;
int numRequestSent = 0;
do
{
// Loop shutdown/Pause check
if (StaticClass.ShutDownPauseCheck("TCP/IP Connect") == true)
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Check if the stream is writable, if not, exit
if (stream.CanWrite)
{
if (StaticClass.ShutDownPauseCheck(job) == false)
{
try
{
// Translate the passed message into ASCII and send it as a byte array
byte[] sendData = Encoding.ASCII.GetBytes(StatusMessage);
stream.Write(sendData, 0, sendData.Length);
numRequestSent++;
}
catch (IOException e)
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} because of Exception {2} at {3:HH:mm:ss.fff}",
job, port, e, DateTime.Now));
// Signal job complete if exception happend in Step 5 or 6
if ((ModelerCurrentStepState == ModelerStepState.STEP_5) ||
(ModelerCurrentStepState == ModelerStepState.STEP_6))
{
// Set the TCP/IP Scan complete flag to signal the RunJob thread
StaticClass.TcpIpScanComplete[job] = true;
}
else
{
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
}
return;
}
StaticClass.Log(string.Format("\nSending {0} msg to Modeler for Job {1} on Port {2} at {3:HH:mm:ss.fff}",
StatusMessage, job, port, DateTime.Now));
}
}
else
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("\nTCP/IP stream closed for Modeler Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Check if TCP/IP stream is readable and data is available
int adjustableSleepTime = StaticClass.STARTING_TCP_IP_WAIT;
int tcpIpRetryCount = 0;
bool messageReceived = false;
do
{
if (stream.CanRead && stream.DataAvailable)
{
// Buffers to store the response
byte[] recvData = new byte[256];
string responseData = string.Empty;
int bytes = stream.Read(recvData, 0, recvData.Length);
responseData = Encoding.ASCII.GetString(recvData, 0, bytes);
StaticClass.Log(string.Format("Received: {0} from Job {1} on Port {2} at {3:HH:mm:ss.fff}",
responseData, job, port, DateTime.Now));
// Readjust sleep time according to Step number
switch (responseData)
{
case "Step 1 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_1;
if (numRequestSent < StaticClass.NUM_REQUESTS_TILL_TCPIP_SLOWDOWN)
{
adjustableSleepTime = 15000;
}
else
{
adjustableSleepTime = 60000;
}
break;
case "Step 2 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_2;
adjustableSleepTime = 10000;
break;
case "Step 3 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_3;
adjustableSleepTime = 7500;
break;
case "Step 4 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_4;
adjustableSleepTime = 5000;
break;
case "Step 5 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_5;
adjustableSleepTime = 2500;
break;
case "Step 6 in process.":
ModelerCurrentStepState = ModelerStepState.STEP_6;
adjustableSleepTime = 250;
break;
case "Whole process done, socket closed.":
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("TCP/IP for Job {0} on Port {1} received Modeler process done at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
ModelerCurrentStepState = ModelerStepState.STEP_COMPLETE;
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the TCP/IP Scan complete flag to signal the RunJob thread
StaticClass.TcpIpScanComplete[job] = true;
return;
default:
StaticClass.Logger.LogWarning("Received Weird Response: {0} from Job {1} on Port {2} at {3:HH:mm:ss.fff}",
responseData, job, port, DateTime.Now);
break;
}
// Backup check of the process complete string, even if it is concatenated with another string
if (responseData.Contains("Whole process done, socket closed."))
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("TCP/IP for Job {0} on Port {1} received Modeler process complete at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
ModelerCurrentStepState = ModelerStepState.STEP_COMPLETE;
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Set the TCP/IP Scan complete flag to signal the RunJob thread
StaticClass.TcpIpScanComplete[job] = true;
return;
}
// Check for Job timeout
if ((DateTime.Now - monitorData.StartTime).TotalSeconds > StaticClass.MaxJobTimeLimitSeconds)
{
StaticClass.Log(string.Format("Job Execution Timeout for Job {0} in state {1} at {2:HH:mm:ss.fff}",
job, ModelerCurrentStepState, DateTime.Now));
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}",
job, port, DateTime.Now));
// Create job Timeout status
StaticClass.StatusDataEntry(job, JobStatus.JOB_TIMEOUT, JobType.TIME_START);
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Check for shutdown or pause
if (StaticClass.ShutDownPauseCheck("TCP/IP Receive") == true)
{
// Make sure to close TCP/IP socket
stream.Close();
client.Close();
// Set the Shutdown flag to signal the RunJob thread
StaticClass.JobShutdownFlag[job] = true;
return;
}
// Set the messge received flag to exit receive loop
messageReceived = true;
}
else
{
// Wait 250 msec between 480 Data Available checks (2 min) CanRead is set for session
Thread.Sleep(StaticClass.READ_AVAILABLE_RETRY_DELAY);
tcpIpRetryCount++;
}
}while ((tcpIpRetryCount < StaticClass.NUM_TCP_IP_RETRIES) && (messageReceived == false));
// Wait for an adjustable time between TCP/IP status requests
Thread.Sleep(adjustableSleepTime);
}while (jobComplete == false);
// Close everything
stream.Close();
client.Close();
StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} in state {2} at {3:HH:mm:ss.fff}",
job, port, ModelerCurrentStepState, DateTime.Now));
}
catch (SocketException e)
{
StaticClass.Logger.LogError(string.Format("SocketException {0} for Job {1} on Port {2} in state {3} at {4:HH:mm:ss.fff}",
e, job, port, ModelerCurrentStepState, DateTime.Now));
}
}
/// <summary>
/// Job Tcp/IP thread
/// </summary>
/// <param name="monitorData"></param>
public TcpIpListenThread(StatusMonitorData monitorData)
{
Port = monitorData.JobPortNumber;
MonitorData = monitorData;
}
