public void RunInputBufferScan(string directory, string job, StatusMonitorData monitorData) { // Add initial entry to status list StaticClass.StatusDataEntry(job, JobStatus.MONITORING_INPUT, JobType.TIME_START); // Monitor the Input Buffer job directory until it has the total number of consumed files string inputBufferJobDir = StaticClass.IniData.InputDir; int numberOfFilesNeeded = monitorData.NumFilesConsumed; if (Directory.Exists(inputBufferJobDir)) { string inputJobFileDir = inputBufferJobDir + @"\" + job; // Register with the File Watcher class event and start its thread InputFileWatcherThread inputFileWatch = new InputFileWatcherThread(inputJobFileDir, numberOfFilesNeeded); if (inputFileWatch == null) { StaticClass.Logger.LogError("Job Run Thread inputFileWatch failed to instantiate"); } inputFileWatch.ThreadProc(); // Wait for Input file scan to complete do { if (StaticClass.ShutDownPauseCheck("Run Job") == true) { return; } Thread.Yield(); }while (StaticClass.InputFileScanComplete[job] == false); StaticClass.Log(string.Format("Finished Input file scan for Job {0} at {1:HH:mm:ss.fff}", inputJobFileDir, DateTime.Now)); // Add copying entry to status list StaticClass.StatusDataEntry(job, JobStatus.COPYING_TO_PROCESSING, JobType.TIME_START); // Move files from Input directory to the Processing directory, creating it first if needed FileHandling.CopyFolderContents(inputJobFileDir, directory, true, true); } else { StaticClass.Logger.LogError("Could not find Input Buffer Directory"); throw new InvalidOperationException("Could not find Input Buffer Directory"); } }
/// <summary> /// Processing directory file watcher thread /// </summary> /// <param name="directory"></param> /// <param name="monitorData"></param> public ProcessingFileWatcherThread(string directory, StatusMonitorData monitorData) { DirectoryName = directory; Job = directory.Replace(StaticClass.IniData.ProcessingDir, "").Remove(0, 1); DirectoryInfo ProcessingJobInfo = new DirectoryInfo(directory); if (ProcessingJobInfo == null) { StaticClass.Logger.LogError("ProcessingFileWatcherThread ProcessingJobInfo failed to instantiate"); } StaticClass.NumberOfProcessingFilesFound[Job] = ProcessingJobInfo.GetFiles().Length; StaticClass.NumberOfProcessingFilesNeeded[Job] = monitorData.NumFilesConsumed + monitorData.NumFilesProduced; StaticClass.TcpIpScanComplete[Job] = false; StaticClass.ProcessingFileScanComplete[Job] = false; StaticClass.ProcessingJobScanComplete[Job] = false; StaticClass.JobShutdownFlag[Job] = false; }
/// <summary> /// Get the data from the Job xml file /// </summary> /// <param name="jobXmlData"></param> /// <param name="monitorData"></param> /// <returns></returns> public StatusMonitorData GetJobXmlData(JobXmlData jobXmlData, StatusMonitorData monitorData) { // Wait for Job xml file to be ready string jobXmlFileName = jobXmlData.JobDirectory + @"\" + jobXmlData.XmlFileName; if (StaticClass.CheckFileReady(jobXmlFileName)) { // Read Job xml file and get the top node XmlDocument jobXmlDoc = new XmlDocument(); jobXmlDoc.Load(jobXmlFileName); XmlElement root = jobXmlDoc.DocumentElement; string TopNode = root.LocalName; // Get nodes for the number of files and names of files to transfer from Job .xml file XmlNode UnitNumberdNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/listitem/value"); XmlNode ConsumedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Consumed"); XmlNode ProducedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Produced"); XmlNode TransferedNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Transfered"); XmlNode ModelerNode = jobXmlDoc.DocumentElement.SelectSingleNode("/" + TopNode + "/FileConfiguration/Modeler"); // Assign then increment port number for this Job monitorData.JobPortNumber = StaticClass.IniData.StartPort + StaticClass.JobPortIndex++; // Get the modeler and number of files to transfer int NumFilesToTransfer = 0; monitorData.UnitNumber = UnitNumberdNode.InnerText; monitorData.Modeler = ModelerNode.InnerText; monitorData.NumFilesConsumed = Convert.ToInt32(ConsumedNode.InnerText); monitorData.NumFilesProduced = Convert.ToInt32(ProducedNode.InnerText); if (TransferedNode != null) { NumFilesToTransfer = Convert.ToInt32(TransferedNode.InnerText); } monitorData.NumFilesToTransfer = NumFilesToTransfer; // Get the modeler and number of files to transfer StaticClass.Log($"Unit Number : {monitorData.UnitNumber}"); StaticClass.Log($"Modeler : {monitorData.Modeler}"); StaticClass.Log($"Num Files Consumed : {monitorData.NumFilesConsumed}"); StaticClass.Log($"Num Files Produced : {monitorData.NumFilesProduced}"); StaticClass.Log($"Num Files To Transfer : {monitorData.NumFilesToTransfer}"); StaticClass.Log($"Job Port Number : {monitorData.JobPortNumber}"); // Create the Transfered file list from the Xml file entries monitorData.TransferedFileList = new List <string>(NumFilesToTransfer); for (int i = 1; i < NumFilesToTransfer + 1; i++) { string transferFileNodeName = ("/" + TopNode + "/FileConfiguration/Transfered" + i.ToString()); XmlNode TransferedFileXml = jobXmlDoc.DocumentElement.SelectSingleNode(transferFileNodeName); monitorData.TransferedFileList.Add(TransferedFileXml.InnerText); StaticClass.Log(string.Format("Transfer File{0} : {1}", i, TransferedFileXml.InnerText)); } } else { StaticClass.Logger.LogError("File {0} is not available at {1:HH:mm:ss.fff}\n", jobXmlFileName, DateTime.Now); } return(monitorData); }
/// <summary> /// Run a job from Input or Processing Buffers /// </summary> /// <param name="dirScanType"></param> /// <param name="jobXmlData"></param> public void RunJob(JobXmlData jobXmlData, DirectoryScanType dirScanType) { // Increment number of Jobs executing in only one place! StaticClass.NumberOfJobsExecuting++; // Create the Job Run common strings string job = jobXmlData.Job; string xmlJobDirectory = jobXmlData.JobDirectory; string processingBufferDirectory = StaticClass.IniData.ProcessingDir; string processingBufferJobDir = processingBufferDirectory + @"\" + job; // Set the job start time StaticClass.JobStartTime[job] = DateTime.Now; // Create new status monitor data and fill it in with the job xml data StatusMonitorData monitorData = new StatusMonitorData { Job = job, JobDirectory = xmlJobDirectory, StartTime = DateTime.Now, JobSerialNumber = jobXmlData.JobSerialNumber, TimeStamp = jobXmlData.TimeStamp, XmlFileName = jobXmlData.XmlFileName }; // Add initial entry to status list StaticClass.StatusDataEntry(job, JobStatus.JOB_STARTED, JobType.TIME_RECEIVED); // Get the Job xml data monitorData = GetJobXmlData(jobXmlData, monitorData); // If this job comes from the Input directory, run the Input job check and start job if (dirScanType == DirectoryScanType.INPUT_BUFFER) { RunInputBufferScan(processingBufferJobDir, job, monitorData); } // If the shutdown flag is set, exit method if (StaticClass.ShutDownPauseCheck("Run Job") == true) { return; } // Add entry to status list StaticClass.StatusDataEntry(job, JobStatus.EXECUTING, JobType.TIME_START); StaticClass.Log(string.Format("Starting Job {0} with Modeler {1} on Port {2} with {3} CPU's at {4:HH:mm:ss.fff}", job, monitorData.Modeler, monitorData.JobPortNumber, StaticClass.IniData.CPUCores, DateTime.Now)); // Execute Modeler using the command line generator string executable = StaticClass.IniData.ModelerRootDir + @"\" + monitorData.Modeler + @"\" + monitorData.Modeler + ".exe"; string processingBuffer = processingBufferJobDir; int port = monitorData.JobPortNumber; int cpuCores = StaticClass.IniData.CPUCores; CommandLineGenerator cmdLineGenerator = new CommandLineGenerator( executable, processingBuffer, port, cpuCores); if (cmdLineGenerator == null) { StaticClass.Logger.LogError("JobRunThread cmdLineGenerator failed to instantiate"); } Process modelerProcess = cmdLineGenerator.ExecuteCommand(job); // Register with the Processing File Watcher class and start its thread ProcessingFileWatcherThread processingFileWatcher = new ProcessingFileWatcherThread(processingBufferJobDir, monitorData); if (processingFileWatcher == null) { StaticClass.Logger.LogError("JobRunThread ProcessingFileWatch failed to instantiate"); } processingFileWatcher.ThreadProc(); // Start the TCP/IP Communications thread before checking for Processing job files TcpIpListenThread tcpIpThread = new TcpIpListenThread(monitorData); if (tcpIpThread == null) { StaticClass.Logger.LogError("ProcessingFileWatcherThread tcpIpThread failed to instantiate"); } tcpIpThread.ThreadProc(); // Add entry to status list StaticClass.StatusDataEntry(job, JobStatus.MONITORING_PROCESSING, JobType.TIME_START); // Wait 45 seconds for Modeler to get started before reading it's information Thread.Sleep(StaticClass.DISPLAY_PROCESS_DATA_WAIT); // Display the Modeler Process information DisplayProcessInfo(job, modelerProcess); // Wait for the Processing job scan complete or shut down do { if (StaticClass.ShutDownPauseCheck("Run Job") == true) { return; } Thread.Yield(); }while ((StaticClass.ProcessingJobScanComplete[job] == false) && (StaticClass.JobShutdownFlag[job] == false)); // Wait to make sure the data.xml is done being handled Thread.Sleep(StaticClass.POST_PROCESS_WAIT); if (StaticClass.JobShutdownFlag[job] == false) { // Wait for the data.xml file to contain a result string dataXmlFileName = processingBufferJobDir + @"\" + "data.xml"; if (OverallResultEntryCheck(dataXmlFileName) == true) { StaticClass.Log(string.Format("Overall Results check confirmed for Job {0} at {1:HH:mm:ss.fff}", job, DateTime.Now)); } else { StaticClass.Log(string.Format("Overall Results check failed for Job {0} at {1:HH:mm:ss.fff}", job, DateTime.Now)); } } // Make sure Modeler Process is stopped if (StaticClass.ProcessHandles[job].HasExited == false) { StaticClass.Log(string.Format("Shutting down Modeler Executable for Job {0} at {1:HH:mm:ss.fff}", job, DateTime.Now)); StaticClass.ProcessHandles[job].Kill(); StaticClass.ProcessHandles.Remove(job); // Wait for Process to end Thread.Sleep(StaticClass.SHUTDOWN_PROCESS_WAIT); } // Run the Job Complete handler RunJobFileProcessing(job, monitorData); // Add entry to status list StaticClass.StatusDataEntry(job, JobStatus.COMPLETE, JobType.TIME_COMPLETE); // Show Job Complete message TimeSpan timeSpan = DateTime.Now - StaticClass.JobStartTime[job]; StaticClass.Log(string.Format("Job {0} Complete taking {1:hh\\:mm\\:ss}. Decrementing Job count to {2} at {3:HH:mm:ss.fff}", job, timeSpan, StaticClass.NumberOfJobsExecuting - 1, DateTime.Now)); // Decrement the number of Jobs executing in one place! StaticClass.NumberOfJobsExecuting--; }
/// <summary> /// Run the Job Completion file and directory handling /// </summary> /// <param name="job"></param> /// <param name="monitorData"></param> public void RunJobFileProcessing(string job, StatusMonitorData monitorData) { string repositoryDirectory = StaticClass.IniData.RepositoryDir; string finishedDirectory = StaticClass.IniData.FinishedDir; string errorDirectory = StaticClass.IniData.ErrorDir; string processingBufferJobDir = StaticClass.IniData.ProcessingDir + @"\" + job; string dataXmlFileName = StaticClass.IniData.ProcessingDir + @"\" + job + @"\" + "data.xml"; StaticClass.StatusDataEntry(job, JobStatus.COPYING_TO_ARCHIVE, JobType.TIME_START); // Get the pass or fail data from the data.xml OverallResult result node XmlDocument dataXmlDoc = new XmlDocument(); dataXmlDoc.Load(dataXmlFileName); XmlNode OverallResult = dataXmlDoc.DocumentElement.SelectSingleNode("/Data/OverallResult/result"); string passFail = "Fail"; if (OverallResult != null) { passFail = OverallResult.InnerText; } string repositoryJobDirectoryName = repositoryDirectory + @"\" + job; if ((OverallResult != null) && (passFail == "Pass")) { // If the Finished directory does not exist, create it string finishedJobDirectoryName = finishedDirectory + @"\" + monitorData.JobSerialNumber; if (!Directory.Exists(finishedJobDirectoryName)) { Directory.CreateDirectory(finishedJobDirectoryName); } // Copy the Transfered files to the Finished directory foreach (string file in monitorData.TransferedFileList) { FileHandling.CopyFile(processingBufferJobDir + @"\" + file, finishedJobDirectoryName + @"\" + file); } // Move Processing Buffer Files to the Repository directory when passed FileHandling.CopyFolderContents(processingBufferJobDir, repositoryJobDirectoryName, true, true); } else // Send files to the Error Buffer and repository { // If the Error directory does not exist, create it string errorJobDirectoryName = errorDirectory + @"\" + monitorData.JobSerialNumber; if (!Directory.Exists(errorJobDirectoryName)) { Directory.CreateDirectory(errorJobDirectoryName); } // Copy the Transfered files to the Error directory foreach (string file in monitorData.TransferedFileList) { if (File.Exists(processingBufferJobDir + @"\" + file)) { FileHandling.CopyFile(processingBufferJobDir + @"\" + file, errorJobDirectoryName + @"\" + file); } } // Move Processing Buffer Files to the Repository directory when failed FileHandling.CopyFolderContents(processingBufferJobDir, repositoryJobDirectoryName, true, true); } }
/// <summary> /// Connect to TCP/IP Port /// </summary> /// <param name="port"></param> /// <param name="server"></param> /// <param name="monitorData"></pram> public void Connect(int port, string server, StatusMonitorData monitorData) { ModelerStepState ModelerCurrentStepState = ModelerStepState.NONE; string job = monitorData.Job; // Wait about a minute for the Modeler to start execution Thread.Sleep(StaticClass.TCP_IP_STARTUP_WAIT); StaticClass.Log(string.Format("\nStarting TCP/IP Scan for Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now)); // Log starting TCP/IP monitoring entry StaticClass.StatusDataEntry(job, JobStatus.MONITORING_TCPIP, JobType.TIME_START); try { // Create a TcpClient TcpClient client = new TcpClient(server, port); if (client == null) { StaticClass.Logger.LogError("TcpIp Connectinon client failed to instantiate"); } // Get a client stream for reading and writing NetworkStream stream = client.GetStream(); if (stream == null) { StaticClass.Logger.LogError("TcpIp Connection stream handle was not gotten from client"); StaticClass.JobShutdownFlag[job] = true; return; } // Show connection and start sending messages StaticClass.Log(string.Format("Connected to Modeler TCP/IP for Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now)); bool jobComplete = false; int numRequestSent = 0; do { // Loop shutdown/Pause check if (StaticClass.ShutDownPauseCheck("TCP/IP Connect") == true) { // Make sure to close TCP/IP socket stream.Close(); client.Close(); StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now)); // Set the shutdown flag to signal the RunJob thread StaticClass.JobShutdownFlag[job] = true; return; } // Check if the stream is writable, if not, exit if (stream.CanWrite) { if (StaticClass.ShutDownPauseCheck(job) == false) { try { // Translate the passed message into ASCII and send it as a byte array byte[] sendData = Encoding.ASCII.GetBytes(StatusMessage); stream.Write(sendData, 0, sendData.Length); numRequestSent++; } catch (IOException e) { // Make sure to close TCP/IP socket stream.Close(); client.Close(); StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} because of Exception {2} at {3:HH:mm:ss.fff}", job, port, e, DateTime.Now)); // Signal job complete if exception happend in Step 5 or 6 if ((ModelerCurrentStepState == ModelerStepState.STEP_5) || (ModelerCurrentStepState == ModelerStepState.STEP_6)) { // Set the TCP/IP Scan complete flag to signal the RunJob thread StaticClass.TcpIpScanComplete[job] = true; } else { // Set the Shutdown flag to signal the RunJob thread StaticClass.JobShutdownFlag[job] = true; } return; } StaticClass.Log(string.Format("\nSending {0} msg to Modeler for Job {1} on Port {2} at {3:HH:mm:ss.fff}", StatusMessage, job, port, DateTime.Now)); } } else { // Make sure to close TCP/IP socket stream.Close(); client.Close(); StaticClass.Log(string.Format("\nTCP/IP stream closed for Modeler Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now)); // Set the Shutdown flag to signal the RunJob thread StaticClass.JobShutdownFlag[job] = true; return; } // Check if TCP/IP stream is readable and data is available int adjustableSleepTime = StaticClass.STARTING_TCP_IP_WAIT; int tcpIpRetryCount = 0; bool messageReceived = false; do { if (stream.CanRead && stream.DataAvailable) { // Buffers to store the response byte[] recvData = new byte[256]; string responseData = string.Empty; int bytes = stream.Read(recvData, 0, recvData.Length); responseData = Encoding.ASCII.GetString(recvData, 0, bytes); StaticClass.Log(string.Format("Received: {0} from Job {1} on Port {2} at {3:HH:mm:ss.fff}", responseData, job, port, DateTime.Now)); // Readjust sleep time according to Step number switch (responseData) { case "Step 1 in process.": ModelerCurrentStepState = ModelerStepState.STEP_1; if (numRequestSent < StaticClass.NUM_REQUESTS_TILL_TCPIP_SLOWDOWN) { adjustableSleepTime = 15000; } else { adjustableSleepTime = 60000; } break; case "Step 2 in process.": ModelerCurrentStepState = ModelerStepState.STEP_2; adjustableSleepTime = 10000; break; case "Step 3 in process.": ModelerCurrentStepState = ModelerStepState.STEP_3; adjustableSleepTime = 7500; break; case "Step 4 in process.": ModelerCurrentStepState = ModelerStepState.STEP_4; adjustableSleepTime = 5000; break; case "Step 5 in process.": ModelerCurrentStepState = ModelerStepState.STEP_5; adjustableSleepTime = 2500; break; case "Step 6 in process.": ModelerCurrentStepState = ModelerStepState.STEP_6; adjustableSleepTime = 250; break; case "Whole process done, socket closed.": // Make sure to close TCP/IP socket stream.Close(); client.Close(); StaticClass.Log(string.Format("TCP/IP for Job {0} on Port {1} received Modeler process done at {2:HH:mm:ss.fff}", job, port, DateTime.Now)); ModelerCurrentStepState = ModelerStepState.STEP_COMPLETE; StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now)); // Set the TCP/IP Scan complete flag to signal the RunJob thread StaticClass.TcpIpScanComplete[job] = true; return; default: StaticClass.Logger.LogWarning("Received Weird Response: {0} from Job {1} on Port {2} at {3:HH:mm:ss.fff}", responseData, job, port, DateTime.Now); break; } // Backup check of the process complete string, even if it is concatenated with another string if (responseData.Contains("Whole process done, socket closed.")) { // Make sure to close TCP/IP socket stream.Close(); client.Close(); StaticClass.Log(string.Format("TCP/IP for Job {0} on Port {1} received Modeler process complete at {2:HH:mm:ss.fff}", job, port, DateTime.Now)); ModelerCurrentStepState = ModelerStepState.STEP_COMPLETE; StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now)); // Set the TCP/IP Scan complete flag to signal the RunJob thread StaticClass.TcpIpScanComplete[job] = true; return; } // Check for Job timeout if ((DateTime.Now - monitorData.StartTime).TotalSeconds > StaticClass.MaxJobTimeLimitSeconds) { StaticClass.Log(string.Format("Job Execution Timeout for Job {0} in state {1} at {2:HH:mm:ss.fff}", job, ModelerCurrentStepState, DateTime.Now)); // Make sure to close TCP/IP socket stream.Close(); client.Close(); StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} at {2:HH:mm:ss.fff}", job, port, DateTime.Now)); // Create job Timeout status StaticClass.StatusDataEntry(job, JobStatus.JOB_TIMEOUT, JobType.TIME_START); // Set the Shutdown flag to signal the RunJob thread StaticClass.JobShutdownFlag[job] = true; return; } // Check for shutdown or pause if (StaticClass.ShutDownPauseCheck("TCP/IP Receive") == true) { // Make sure to close TCP/IP socket stream.Close(); client.Close(); // Set the Shutdown flag to signal the RunJob thread StaticClass.JobShutdownFlag[job] = true; return; } // Set the messge received flag to exit receive loop messageReceived = true; } else { // Wait 250 msec between 480 Data Available checks (2 min) CanRead is set for session Thread.Sleep(StaticClass.READ_AVAILABLE_RETRY_DELAY); tcpIpRetryCount++; } }while ((tcpIpRetryCount < StaticClass.NUM_TCP_IP_RETRIES) && (messageReceived == false)); // Wait for an adjustable time between TCP/IP status requests Thread.Sleep(adjustableSleepTime); }while (jobComplete == false); // Close everything stream.Close(); client.Close(); StaticClass.Log(string.Format("Closed TCP/IP Socket for Job {0} on Port {1} in state {2} at {3:HH:mm:ss.fff}", job, port, ModelerCurrentStepState, DateTime.Now)); } catch (SocketException e) { StaticClass.Logger.LogError(string.Format("SocketException {0} for Job {1} on Port {2} in state {3} at {4:HH:mm:ss.fff}", e, job, port, ModelerCurrentStepState, DateTime.Now)); } }
/// <summary> /// Job Tcp/IP thread /// </summary> /// <param name="monitorData"></param> public TcpIpListenThread(StatusMonitorData monitorData) { Port = monitorData.JobPortNumber; MonitorData = monitorData; }