/// <summary> /// DoInitialize initializes the Aspen Engine if specified in the job contract. /// </summary> /// <param name="job"></param> /// <param name="stest"></param> /// <param name="defaultsDict"></param> private void DoInitialize(IJobConsumerContract job, sinter.ISimulation stest, JObject defaultsDict) { if (!job.Initialize()) { return; } Debug.WriteLine(String.Format("Start warm-up run on job {0}", job.Id), "SinterConsumer.Run"); try { stest.sendInputs(defaultsDict); } catch (Exception ex) { job.Error(String.Format( "Error sending Defaults to Aspen for warm-up run: {0}, traceback {1}", ex.Message, ex.StackTrace)); Debug.WriteLine(String.Format("Moved Job {0} to Error", job.Id), "SinterConsumer.Run"); throw; } job.Message("Warm-up inputs sent, running simulation"); Debug.WriteLine(String.Format("Calling run Simulation for warm-up on Job {0}", job.Id), "SinterConsumer.Run"); try { running = true; lock (this) { stest.sendInputsToSim(); stest.runSim(); stest.recvOutputsFromSim(); } running = false; if (isTerminated) { throw new TerminateException( "Monitor Terminated Initializing Simulation"); } } catch (Exception ex) { Debug.WriteLine(String.Format("Moving Job {0} to Error", job.Id), "SinterConsumer.Run"); job.Error(String.Format( "Error running warm-up simulation: {0}, traceback {1}", ex.Message, ex.StackTrace)); throw; } job.Message("Warm-up complete and successful. Reseting Sim for real run."); Debug.WriteLine(String.Format("Warm-up complete and successful. Reseting Sim for real run {0}", job.Id), "SinterConsumer.Run"); }
/// <summary> /// DoRun sends inputs specified in job contract to aspen simulation and runs /// the simulation. /// </summary> /// <param name="job"></param> /// <param name="stest"></param> /// <param name="combinedDict"></param> protected void DoRun(IJobConsumerContract job, sinter.ISimulation stest, JObject defaultsDict, JObject inputDict) { Debug.WriteLine(String.Format("Send Inputs to Job {0}", job.Id), "SinterConsumer.DoRun"); try { if (defaultsDict != null) { stest.sendDefaults(defaultsDict); } stest.sendInputs(inputDict); } catch (Exception ex) { job.Error(String.Format( "Error sending Inputs to Aspen: {0}, traceback {1}", ex.Message, ex.StackTrace)); Debug.WriteLine(String.Format("Moved Job {0} to Error", job.Id), "SinterConsumer.DoRun"); throw; } job.Message("sinter inputs sent, running simulation"); Debug.WriteLine(String.Format("Job {0}: sendInputsToSim ", job.Id), "SinterConsumer.DoRun"); try { running = true; lock (this) { stest.sendInputsToSim(); Debug.WriteLine(String.Format("Job {0}: runSim ", job.Id), "SinterConsumer.DoRun"); stest.runSim(); Debug.WriteLine(String.Format("Job {0}: recvOutputsFromSim ", job.Id), "SinterConsumer.DoRun"); stest.recvOutputsFromSim(); } running = false; if (isTerminated) { throw new TerminateException( "Monitor thread Terminated Simulation"); } } catch (Exception ex) { Debug.WriteLine(String.Format("Moving Job {0} to Error", job.Id), "SinterConsumerRun.DoRun"); Debug.WriteLine(ex.Message, "SinterConsumerRun.DoRun"); Debug.WriteLine(ex.StackTrace, "SinterConsumerRun.DoRun"); job.Error(String.Format( "Error running simulation: {0}, traceback {1}", ex.Message, ex.StackTrace)); throw; } }
/// <summary> /// Run pops a job descriptor off the queue and manages the underlying AspenSinter /// </summary> /// <returns></returns> public bool Run() { // RESET ALL instance variables except sim bool prevJobIsTerminated = isTerminated; isSetup = false; isTerminated = false; running = false; var job = GetNextJob(); this.job = job; if (job == null) { Debug.WriteLine(DateTime.Now.ToString() + " - No submitted jobs in queue", "SinterConsumer.Run"); return(false); } visible = job.Visible; IProcess process = null; if (job.Reset == false && sim != null && prevJobIsTerminated == false && RunNoReset(job)) { Debug.WriteLine(String.Format("{0} - Found JOB(noreset): {1}", DateTime.Now.ToString(), job.Id), "SinterConsumer.Run"); return(true); } Debug.WriteLine(String.Format("{0} - Found JOB(reset): {1}", DateTime.Now.ToString(), job.Id), "SinterConsumer.Run"); CleanUp(); sim = null; Debug.WriteLine(String.Format("Setup Sinter Process {0}", job.Id), "SinterConsumer.Run"); process = DoSetup(job); var maxAttemptsReadSetup = 5; DoConfigure(job, process, maxAttemptsReadSetup); Debug.WriteLine(String.Format("Move Job {0} to state Running", job.Id), "SinterConsumer.Run"); job.Running(); IDictionary <String, Object> inputDict = null; String data = null; JObject defaults = null; JObject inputs = null; // NOTE: Quick Fix. Serializing Dicts into string json // and back to JObject because of API change to sinter. // Better to refactor interfaces, etc to hand back strings try { inputDict = process.Input; data = JsonConvert.SerializeObject(inputDict); inputs = JObject.Parse(data); } catch (System.Data.SqlClient.SqlException ex) { Debug.WriteLine("SqlException: " + ex.Message, "SinterConsumer.Run"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run"); sim.closeSim(); sim = null; throw; } catch (Exception ex) { sim.closeSim(); job.Error("Failed to Create Inputs to Simulation (Bad Simulation defaults or Job inputs)"); job.Message(ex.StackTrace.ToString()); throw; } Debug.WriteLine("Initialize", "SinterConsumer.Run"); try { DoInitialize(job, sim, defaults); } catch (System.Data.SqlClient.SqlException ex) { Debug.WriteLine("SqlException: " + ex.Message, "SinterConsumer.Run"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run"); sim.closeSim(); sim = null; throw; } catch (Exception ex) { Debug.WriteLine("DoInitialize Exception Close: " + ex.Message, "SinterConsumer.Run"); sim.closeSim(); sim = null; throw; } Debug.WriteLine("Run", "SinterConsumer.Run"); try { DoRun(job, sim, defaults, inputs); } catch (System.Data.SqlClient.SqlException ex) { Debug.WriteLine("SqlException: " + ex.Message, "SinterConsumer.Run"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run"); sim.closeSim(); sim = null; throw; } catch (Exception ex) { Debug.WriteLine("DoRun Exception Close", "SinterConsumer.Run"); job.Error("Exception: " + ex.Message); sim.closeSim(); sim = null; throw; } Debug.WriteLine("Finalize", "SinterConsumer.Run"); try { DoFinalize(sim, job, process); last_simulation_name = job.SimulationId; last_simulation_success = job.IsSuccess(); } catch (System.Data.SqlClient.SqlException ex) { Debug.WriteLine("SqlException: " + ex.Message, "SinterConsumer.Run"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run"); sim.closeSim(); sim = null; throw; } catch (Exception ex) { Debug.WriteLine("DoFinalize Exception Close", "SinterConsumerRun.Run"); Debug.WriteLine(ex.Message, "SinterConsumerRun.Run"); Debug.WriteLine(ex.StackTrace, "SinterConsumerRun.Run"); job.Error("Exception: " + ex.Message); sim.closeSim(); sim = null; throw; } return(true); }
/// <summary> /// DoConfigure starts up COM process server. Occasionally readSetup fail, this /// will attempt maxAttemptsReadSetup times to open the /// simulation. Also readSetup will hang in unmanaged code /// </summary> /// <param name="job"></param> /// <param name="process"></param> /// <param name="maxAttemptsReadSetup"></param> /// <returns></returns> private void DoConfigure(IJobConsumerContract job, IProcess process, int maxAttemptsReadSetup) { int attempts = 1; int timeout = 1000; string setupString = null; var configFilePath = Path.Combine(process.WorkingDirectory, configFileName); while (true) { Debug.WriteLine(String.Format("Attempt to Read {0}", configFilePath), "SinterConsumer.DoConfigure"); setupString = ""; try { StreamReader inFileStream = new StreamReader(configFilePath); setupString = inFileStream.ReadToEnd(); inFileStream.Close(); } catch (Exception ex) { Debug.WriteLine(ex.Message, "SinterConsumer.DoConfigure"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.DoConfigure"); job.Error(String.Format("Failed to read configuration file: {0}", ex.StackTrace)); throw; } try { InitializeSinter(job, process.WorkingDirectory, setupString); } catch (Exception ex) { Debug.WriteLine(ex.Message, "SinterConsumer.DoConfigure.InitializeSinter"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.DoConfigure.InitializeSinter"); job.Error(String.Format("Failed to InitializeSinter: {0}", ex.StackTrace)); throw; } Debug.WriteLine(String.Format("readSetup {0}", configFilePath), "SinterConsumer.DoConfigure"); Debug.WriteLine(String.Format("model {0}", sim.setupFile.aspenFilename), "SinterConsumer.DoConfigure"); try { lock (this) { sim.openSim(); isSetup = true; if (isTerminated) { throw new TerminateException( "APWN was terminated, try readSetup again"); } } } catch (Exception ex) { //System.Exception: Cannot create ActiveX component var msg = String.Format("Failed to open Simulation ({0}: {1}", attempts, ex.Message); Debug.WriteLine(msg, "SinterConsumer"); Debug.WriteLine(ex.StackTrace, "SinterConsumer"); try { sim.closeSim(); } catch (NullReferenceException e2) { Debug.WriteLine(e2.Message, "SinterConsumer"); Debug.WriteLine(ex.StackTrace, "SinterConsumer"); } if (job.IsTerminated()) { throw new TerminateException("Job was terminated by producer"); } else if (attempts >= maxAttemptsReadSetup) { job.Error(msg); throw; } job.Message(msg); isTerminated = false; attempts += 1; System.Threading.Thread.Sleep(timeout); timeout *= 2; continue; } job.Message("sinter read setup finished"); // One of these commands hangs infrequently.. // However this job will not be retired as above, just abandoned and left in setup. Debug.WriteLine("set simulation layout", "SinterConsumer.DoConfigure"); try { sim.Vis = visible; // operates on Aspen instance } catch (Exception ex) { sim.closeSim(); var msg = String.Format("Failed to Layout Simulation ({0}: {1}", attempts, ex.Message); Debug.WriteLine(msg, "SinterConsumer.DoConfigure"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.DoConfigure"); if (attempts >= maxAttemptsReadSetup) { job.Error(msg); throw; } job.Message(msg); attempts += 1; System.Threading.Thread.Sleep(timeout); timeout *= 2; continue; } Debug.WriteLine("set simulation reset", "SinterConsumer.DoConfigure"); try { sim.dialogSuppress = true; // operates on Aspen instance //sim.resetSim(); // operates on Aspen instance } catch (Exception ex) { sim.closeSim(); var msg = String.Format("Failed to Reset Simulation ({0}: {1}", attempts, ex.Message); Debug.WriteLine(msg, "SinterConsumer.DoConfigure"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.DoConfigure"); if (attempts >= maxAttemptsReadSetup) { job.Error(msg); throw; } job.Message(msg); attempts += 1; System.Threading.Thread.Sleep(timeout); timeout *= 2; continue; } break; } }
/// <summary> /// DoSetup setup up the working directory, returns an IProcess interface. /// </summary> /// <param name="job"></param> /// <param name="configFileName"></param> /// <returns></returns> private IProcess DoSetup(IJobConsumerContract job) { Debug.WriteLine("DoSetup Called on " + job.Id, "SinterConsumer"); // NOTE: expect OptimisticConcurrencyException // to occur ONLY here when running multiple consumers IProcess process = null; try { process = job.Setup(); } catch (Exception ex) { Debug.WriteLine(ex.Message, "SinterConsumer.DoSetup"); throw; } //Sets the dir in the function to MyDocs, but doesn't actually chdir try { SetupWorkingDir(process); } catch (Exception ex) { Debug.WriteLine("Failed to setup working directory", "SinterConsumer.Run"); Debug.WriteLine(ex.Message); Debug.WriteLine(ex.StackTrace); job.Error(String.Format("Failed to setup working directory: {0}", ex.StackTrace)); throw; } bool isCached = false; try { isCached = copyFromCache(job); } catch (Exception ex) { Debug.WriteLine("copyFromCache Failed to copy files to working directory", "SinterConsumer.Run"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run"); job.Error(String.Format("copyFromCache Failed to copy files to working directory: {0}", ex.StackTrace)); throw; } if (isCached == false) { try { copyFilesToDisk(job); } catch (Exception ex) { Debug.WriteLine("copyFilesToDisk Failed to copy files to working directory", "SinterConsumer.Run"); Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run"); job.Error(String.Format("copyFilesToDisk Failed to copy files to working directory: {0}", ex.StackTrace)); throw; } } job.Message("working directory setup finished"); return(process); }
/// <summary> /// RunNoReset doesn't close underlying COM object, doesn't change Sinter working directory. /// Grabs new inputs and sends them via sinter, and runs again. /// </summary> /// <param name="job"></param> /// <returns></returns> protected override bool RunNoReset(IJobConsumerContract job) { if (job.SimulationId != last_simulation_name) { return(false); } if ((sim.GetType() == typeof(sinter_simGPROMS))) { // hack to see whether Aspen has already been initialized // should implement an interface "closed" try { bool test = ((sinter_simGPROMS)sim).Vis; } catch (NullReferenceException) { return(false); } } if (last_simulation_success == false) { return(false); } IProcess process = null; Debug.WriteLine(String.Format("Reusing Sinter({0}) : Job {1}", job.ApplicationName, job.Id), GetType().Name); try { process = job.Setup(); } catch (Exception ex) { Debug.WriteLine(ex.Message, GetType().Name); throw; } job.Message("Setup: Reusing Sinter GProms"); Debug.WriteLine(String.Format("Move Job {0} to state Running", job.Id), GetType().Name); job.Running(); IDictionary <String, Object> inputDict = null; String data = null; JObject defaults = null; JObject inputs = null; // NOTE: Quick Fix. Serializing Dicts into string json // and back to JObject because of API change to sinter. // Better to refactor interfaces, etc to hand back strings try { inputDict = process.Input; data = JsonConvert.SerializeObject(inputDict); inputs = JObject.Parse(data); } catch (Exception ex) { sim.closeSim(); job.Error("Failed to Create Inputs to Simulation (Bad Simulation defaults or Job inputs)"); job.Message(ex.StackTrace.ToString()); throw; } Debug.WriteLine("Run", GetType().Name); try { DoRun(job, sim, defaults, inputs); } catch (System.Data.SqlClient.SqlException ex) { Debug.WriteLine("SqlException: " + ex.Message, GetType().Name); Debug.WriteLine(ex.StackTrace, GetType().Name); sim.closeSim(); sim = null; throw; } catch (Exception ex) { Debug.WriteLine("DoRun Exception Close", GetType().Name); //if (CheckTerminate() == false) if (isTerminated == false) { job.Error("Exception: " + ex.Message); sim.closeSim(); } sim = null; throw; } Debug.WriteLine("Finalize", GetType().Name); try { DoFinalize(sim, job, process); } catch (System.Data.SqlClient.SqlException ex) { Debug.WriteLine("SqlException: " + ex.Message, GetType().Name); Debug.WriteLine(ex.StackTrace, GetType().Name); sim.closeSim(); sim = null; throw; } catch (DbEntityValidationException dbEx) { Debug.WriteLine("DbEntityValidationException: " + dbEx.Message, GetType().Name); Debug.WriteLine(dbEx.StackTrace, GetType().Name); var msg = String.Empty; foreach (var validationErrors in dbEx.EntityValidationErrors) { foreach (var validationError in validationErrors.ValidationErrors) { msg += String.Format("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage); Debug.WriteLine(msg, GetType().Name); Trace.TraceInformation("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage); msg += ". \n"; } } job.Error("Exception: " + msg); sim.closeSim(); sim = null; throw; } catch (Exception ex) { Debug.WriteLine("DoFinalize Exception Close", GetType().Name); job.Error("Exception: " + ex.Message); sim.closeSim(); sim = null; throw; } return(true); }
/// <summary> /// DoFinalize sends outputs to the database, sets final run status and /// warnings and/or errors in the job and process. /// </summary> /// <param name="stest"></param> /// <param name="job"></param> /// <param name="process"></param> protected virtual void DoFinalize(sinter.ISimulation stest, IJobConsumerContract job, IProcess process) { int runStatus = (int)stest.runStatus; process.SetStatus(runStatus); IDictionary <string, Object> myDict = null; JObject outputDict; try { if (stest.runStatus == sinter.sinter_AppError.si_OKAY || stest.runStatus == sinter.sinter_AppError.si_SIMULATION_WARNING) { var superDict = stest.getOutputs(); outputDict = (JObject)superDict["outputs"]; // HACK: Inefficient Just making it work w/o covariance issues string data = outputDict.ToString(Newtonsoft.Json.Formatting.None); myDict = JsonConvert.DeserializeObject <IDictionary <string, Object> >(data); } else { Debug.WriteLine(String.Format("Sinter runstatus={0}, did not retrieve outputs.", stest.runStatus), "SinterConsumer.DoFinalize"); } } catch (Exception ex) { Debug.WriteLine(ex.Message, "SinterConsumer.DoFinalize"); process.Output = new Dictionary <string, Object>() { { "exception", ex.Message } }; if (job.IsTerminated()) { throw new TerminateException("Job was terminated by producer"); } job.Error(String.Format("Failed to retrieve the outputs: {0} {1}", ex.Message, ex.StackTrace)); return; } Debug.WriteLine("Outputs: " + myDict, this.GetType()); process.Output = myDict; if (stest.runStatus == sinter.sinter_AppError.si_OKAY) { job.Success(); } else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_ERROR) { var msg = String.Join(",", stest.errorsBasic()); Debug.WriteLine("Error: si_SIMULATION_ERROR", "SinterConsumerRun.DoFinalize"); Debug.WriteLine(msg, "SinterConsumerRun.DoFinalize"); // Truncate the msg if it's more than 4000 characters. // The last 4000 characters are more interesting for debugging. if (msg.Length > 4000) { msg = msg.Substring(msg.Length - 4000); } job.Error(String.Format("Error: si_SIMULATION_ERROR: {0}", msg)); if (sim != null) { Debug.WriteLine("2) Simulation is not Null", "SinterConsumerRun.DoFinalize()"); sim.closeSim(); if (sim != null) { KillProcessAndChildren(sim.ProcessID); } sim = null; //sim.terminate(); } } else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_WARNING) { job.Message(String.Join(",", stest.warningsBasic())); job.Success(); } else if (stest.runStatus == sinter.sinter_AppError.si_COM_EXCEPTION) { Debug.WriteLine("Error: si_COM_EXCEPTION", "SinterConsumerRun.DoFinalize"); job.Error("COM Exception, Server was Terminated"); } else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_NOT_RUN) { Debug.WriteLine("Error: si_SIMULATION_NOT_RUN", "SinterConsumerRun.DoFinalize"); job.Error("Simulation was not Run"); } else { Debug.WriteLine("Error: Unknown", "SinterConsumerRun.DoFinalize"); job.Error(string.Format("Aspen Process Status unknown code {0}.", (int)stest.runStatus)); } }
/// <summary> /// DoRun sends inputs specified in job contract to aspen simulation and runs /// the simulation. /// </summary> /// <param name="job"></param> /// <param name="stest"></param> /// <param name="combinedDict"></param> protected void DoRun(IJobConsumerContract job, sinter.ISimulation stest, JObject defaultsDict, JObject inputDict) { Debug.WriteLine(String.Format("Send Inputs to Job {0}", job.Id), "SinterConsumer.DoRun"); try { Debug.WriteLine("Default: " + JsonConvert.SerializeObject(defaultsDict), "SinterConsumerRun.DoRun"); Debug.WriteLine("Input: " + JsonConvert.SerializeObject(inputDict), "SinterConsumerRun.DoRun"); if (defaultsDict != null) { stest.sendDefaults(defaultsDict); } stest.sendInputs(inputDict); } catch (Exception ex) { Debug.WriteLine(String.Format( "Error sending Inputs: {0}, traceback {1}", ex.Message, ex.StackTrace), "SinterConsumerRun.DoRun"); if (job.IsTerminated()) { throw new TerminateException("Job was terminated by producer"); } job.Error(String.Format( "Error sending Inputs: {0}, traceback {1}", ex.Message, ex.StackTrace)); Debug.WriteLine(String.Format("Moved Job {0} to Error", job.Id), "SinterConsumer.DoRun"); throw; } job.Message("sinter inputs sent, running simulation"); Debug.WriteLine(String.Format("Job {0}: sendInputsToSim ", job.Id), "SinterConsumer.DoRun"); try { running = true; lock (this) { stest.sendInputsToSim(); Debug.WriteLine(String.Format("Job {0}: runSim ", job.Id), "SinterConsumer.DoRun"); stest.runSim(); Debug.WriteLine(String.Format("Job {0}: recvOutputsFromSim ", job.Id), "SinterConsumer.DoRun"); if (stest.runStatus == sinter.sinter_AppError.si_OKAY) { stest.recvOutputsFromSim(); job.Message("Real Run was complete and successful."); Debug.WriteLine(String.Format("Real Run was complete and successful. Job Id: {0}", job.Id), "SinterConsumer.Run"); } else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_WARNING) { stest.recvOutputsFromSim(); job.Message("Real Run was completed with warning."); Debug.WriteLine(String.Format("Real Run was completed with warning. Job Id: {0}", job.Id), "SinterConsumer.Run"); } else { job.Message(String.Format("Real Run failed, runStatus={0}", stest.runStatus)); Debug.WriteLine(String.Format("Real Run was complete and failed. Job Id: {0}, runStatus={1}", job.Id, stest.runStatus), "SinterConsumer.Run"); } } running = false; if (isTerminated) { throw new TerminateException( "Monitor thread Terminated Simulation"); } } catch (System.Runtime.InteropServices.COMException ex) { Debug.WriteLine(String.Format("Moving Job {0} to Error", job.Id), "SinterConsumerRun.DoRun"); Debug.WriteLine(ex.Message, "SinterConsumerRun.DoRun"); Debug.WriteLine(ex.StackTrace, "SinterConsumerRun.DoRun"); if (this.terminateSimAndJob() == false) { Debug.WriteLine("Failed to terminate the simulation", "SinterConsumerRun.DoRun()"); if (sim != null) { Debug.WriteLine("1) Simulation is not Null", "SinterConsumerRun.DoRun()"); sim.closeSim(); if (sim != null) { KillProcessAndChildren(sim.ProcessID); } sim = null; //sim.terminate(); } } job.Error(String.Format( "Error running simulation: {0}, traceback {1}", ex.Message, ex.StackTrace)); } catch (Exception ex) { Debug.WriteLine(String.Format("Moving Job {0} to Error", job.Id), "SinterConsumerRun.DoRun"); Debug.WriteLine(ex.Message, "SinterConsumerRun.DoRun"); Debug.WriteLine(ex.StackTrace, "SinterConsumerRun.DoRun"); if (job.IsTerminated()) { throw new TerminateException("Job was terminated by producer"); } job.Error(String.Format( "Error running simulation: {0}, traceback {1}", ex.Message, ex.StackTrace)); throw; } }