/// <summary>
/// DoRun sends inputs specified in job contract to aspen simulation and runs
/// the simulation.
/// </summary>
/// <param name="job"></param>
/// <param name="stest"></param>
/// <param name="combinedDict"></param>
protected void DoRun(IJobConsumerContract job, sinter.ISimulation stest, JObject defaultsDict, JObject inputDict)
{
Debug.WriteLine(String.Format("Send Inputs to Job {0}", job.Id),
"SinterConsumer.DoRun");
try
{
if (defaultsDict != null)
{
stest.sendDefaults(defaultsDict);
}
stest.sendInputs(inputDict);
}
catch (Exception ex)
{
job.Error(String.Format(
"Error sending Inputs to Aspen: {0}, traceback {1}",
ex.Message, ex.StackTrace));
Debug.WriteLine(String.Format("Moved Job {0} to Error", job.Id),
"SinterConsumer.DoRun");
throw;
}
job.Message("sinter inputs sent, running simulation");
Debug.WriteLine(String.Format("Job {0}: sendInputsToSim ", job.Id),
"SinterConsumer.DoRun");
try
{
running = true;
lock (this)
{
stest.sendInputsToSim();
Debug.WriteLine(String.Format("Job {0}: runSim ", job.Id),
"SinterConsumer.DoRun");
stest.runSim();
Debug.WriteLine(String.Format("Job {0}: recvOutputsFromSim ", job.Id),
"SinterConsumer.DoRun");
stest.recvOutputsFromSim();
}
running = false;
if (isTerminated)
{
throw new TerminateException(
"Monitor thread Terminated Simulation");
}
}
catch (Exception ex)
{
Debug.WriteLine(String.Format("Moving Job {0} to Error", job.Id),
"SinterConsumerRun.DoRun");
Debug.WriteLine(ex.Message, "SinterConsumerRun.DoRun");
Debug.WriteLine(ex.StackTrace, "SinterConsumerRun.DoRun");
job.Error(String.Format(
"Error running simulation: {0}, traceback {1}",
ex.Message, ex.StackTrace));
throw;
}
}
/// <summary>
/// DoInitialize initializes the Aspen Engine if specified in the job contract.
/// </summary>
/// <param name="job"></param>
/// <param name="stest"></param>
/// <param name="defaultsDict"></param>
private void DoInitialize(IJobConsumerContract job, sinter.ISimulation stest, JObject defaultsDict)
{
if (!job.Initialize())
{
return;
}
Debug.WriteLine(String.Format("Start warm-up run on job {0}", job.Id),
"SinterConsumer.Run");
try
{
stest.sendInputs(defaultsDict);
}
catch (Exception ex)
{
job.Error(String.Format(
"Error sending Defaults to Aspen for warm-up run: {0}, traceback {1}",
ex.Message, ex.StackTrace));
Debug.WriteLine(String.Format("Moved Job {0} to Error", job.Id),
"SinterConsumer.Run");
throw;
}
job.Message("Warm-up inputs sent, running simulation");
Debug.WriteLine(String.Format("Calling run Simulation for warm-up on Job {0}", job.Id),
"SinterConsumer.Run");
try
{
running = true;
lock (this)
{
stest.sendInputsToSim();
stest.runSim();
stest.recvOutputsFromSim();
}
running = false;
if (isTerminated)
{
throw new TerminateException(
"Monitor Terminated Initializing Simulation");
}
}
catch (Exception ex)
{
Debug.WriteLine(String.Format("Moving Job {0} to Error", job.Id),
"SinterConsumer.Run");
job.Error(String.Format(
"Error running warm-up simulation: {0}, traceback {1}",
ex.Message, ex.StackTrace));
throw;
}
job.Message("Warm-up complete and successful. Reseting Sim for real run.");
Debug.WriteLine(String.Format("Warm-up complete and successful. Reseting Sim for real run {0}", job.Id),
"SinterConsumer.Run");
}
/// <summary>
/// Run pops a job descriptor off the queue and manages the underlying AspenSinter
/// </summary>
/// <returns></returns>
public bool Run()
{
// RESET ALL instance variables except sim
bool prevJobIsTerminated = isTerminated;
isSetup = false;
isTerminated = false;
running = false;
var job = GetNextJob();
this.job = job;
if (job == null)
{
Debug.WriteLine(DateTime.Now.ToString() + " - No submitted jobs in queue",
"SinterConsumer.Run");
return(false);
}
visible = job.Visible;
IProcess process = null;
if (job.Reset == false && sim != null && prevJobIsTerminated == false &&
RunNoReset(job))
{
Debug.WriteLine(String.Format("{0} - Found JOB(noreset): {1}", DateTime.Now.ToString(), job.Id),
"SinterConsumer.Run");
return(true);
}
Debug.WriteLine(String.Format("{0} - Found JOB(reset): {1}", DateTime.Now.ToString(), job.Id),
"SinterConsumer.Run");
CleanUp();
sim = null;
Debug.WriteLine(String.Format("Setup Sinter Process {0}", job.Id),
"SinterConsumer.Run");
process = DoSetup(job);
var maxAttemptsReadSetup = 5;
DoConfigure(job, process, maxAttemptsReadSetup);
Debug.WriteLine(String.Format("Move Job {0} to state Running", job.Id),
"SinterConsumer.Run");
job.Running();
IDictionary <String, Object> inputDict = null;
String data = null;
JObject defaults = null;
JObject inputs = null;
// NOTE: Quick Fix. Serializing Dicts into string json
// and back to JObject because of API change to sinter.
// Better to refactor interfaces, etc to hand back strings
try
{
inputDict = process.Input;
data = JsonConvert.SerializeObject(inputDict);
inputs = JObject.Parse(data);
}
catch (System.Data.SqlClient.SqlException ex)
{
Debug.WriteLine("SqlException: " + ex.Message, "SinterConsumer.Run");
Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run");
sim.closeSim();
sim = null;
throw;
}
catch (Exception ex)
{
sim.closeSim();
job.Error("Failed to Create Inputs to Simulation (Bad Simulation defaults or Job inputs)");
job.Message(ex.StackTrace.ToString());
throw;
}
Debug.WriteLine("Initialize", "SinterConsumer.Run");
try
{
DoInitialize(job, sim, defaults);
}
catch (System.Data.SqlClient.SqlException ex)
{
Debug.WriteLine("SqlException: " + ex.Message, "SinterConsumer.Run");
Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run");
sim.closeSim();
sim = null;
throw;
}
catch (Exception ex)
{
Debug.WriteLine("DoInitialize Exception Close: " + ex.Message, "SinterConsumer.Run");
sim.closeSim();
sim = null;
throw;
}
Debug.WriteLine("Run", "SinterConsumer.Run");
try
{
DoRun(job, sim, defaults, inputs);
}
catch (System.Data.SqlClient.SqlException ex)
{
Debug.WriteLine("SqlException: " + ex.Message, "SinterConsumer.Run");
Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run");
sim.closeSim();
sim = null;
throw;
}
catch (Exception ex)
{
Debug.WriteLine("DoRun Exception Close", "SinterConsumer.Run");
job.Error("Exception: " + ex.Message);
sim.closeSim();
sim = null;
throw;
}
Debug.WriteLine("Finalize", "SinterConsumer.Run");
try
{
DoFinalize(sim, job, process);
last_simulation_name = job.SimulationId;
last_simulation_success = job.IsSuccess();
}
catch (System.Data.SqlClient.SqlException ex)
{
Debug.WriteLine("SqlException: " + ex.Message, "SinterConsumer.Run");
Debug.WriteLine(ex.StackTrace, "SinterConsumer.Run");
sim.closeSim();
sim = null;
throw;
}
catch (Exception ex)
{
Debug.WriteLine("DoFinalize Exception Close", "SinterConsumerRun.Run");
Debug.WriteLine(ex.Message, "SinterConsumerRun.Run");
Debug.WriteLine(ex.StackTrace, "SinterConsumerRun.Run");
job.Error("Exception: " + ex.Message);
sim.closeSim();
sim = null;
throw;
}
return(true);
}
/// <summary>
/// DoFinalize sends outputs to the database, sets final run status and
/// warnings and/or errors in the job and process.
/// </summary>
/// <param name="stest"></param>
/// <param name="job"></param>
/// <param name="process"></param>
protected void DoFinalize(sinter.ISimulation stest,
IJobConsumerContract job,
IProcess process)
{
int runStatus = (int)stest.runStatus;
process.SetStatus(runStatus);
IDictionary <string, Object> myDict = null;
JObject outputDict;
try
{
var superDict = stest.getOutputs();
outputDict = (JObject)superDict["outputs"];
// HACK: Inefficient Just making it work w/o covariance issues
string data = outputDict.ToString(Newtonsoft.Json.Formatting.None);
myDict = JsonConvert.DeserializeObject <IDictionary <string, Object> >(data);
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message, "SinterConsumer.DoFinalize");
process.Output = new Dictionary <string, Object>()
{
{ "exception", ex.Message }
};
job.Error(String.Format("Failed to retrieve the outputs: {0} {1}", ex.Message, ex.StackTrace));
return;
}
Debug.WriteLine("Outputs: " + myDict, this.GetType());
process.Output = myDict;
if (stest.runStatus == sinter.sinter_AppError.si_OKAY)
{
job.Success();
}
else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_ERROR)
{
Debug.WriteLine("Error: si_SIMULATION_ERROR", "SinterConsumerRun.DoFinalize");
sim.closeSim();
sim = null;
job.Error(String.Join(",", stest.errorsBasic()));
}
//else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_WARNING)
//{
// job.Error(String.Join(",", stest.warningsBasic()));
//}
else if (stest.runStatus == sinter.sinter_AppError.si_COM_EXCEPTION)
{
Debug.WriteLine("Error: si_COM_EXCEPTION", "SinterConsumerRun.DoFinalize");
job.Error("COM Exception, Server was Terminated");
}
else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_NOT_RUN)
{
Debug.WriteLine("Error: si_SIMULATION_NOT_RUN", "SinterConsumerRun.DoFinalize");
job.Error("Simulation was not Run");
}
else
{
Debug.WriteLine("Error: Unknown", "SinterConsumerRun.DoFinalize");
job.Error(string.Format("Aspen Process Status unknown code {0}.", (int)stest.runStatus));
}
}
/// <summary>
/// DoFinalize sends outputs to the database, sets final run status and
/// warnings and/or errors in the job and process.
/// </summary>
/// <param name="stest"></param>
/// <param name="job"></param>
/// <param name="process"></param>
protected virtual void DoFinalize(sinter.ISimulation stest,
IJobConsumerContract job,
IProcess process)
{
int runStatus = (int)stest.runStatus;
process.SetStatus(runStatus);
IDictionary <string, Object> myDict = null;
JObject outputDict;
try
{
if (stest.runStatus == sinter.sinter_AppError.si_OKAY ||
stest.runStatus == sinter.sinter_AppError.si_SIMULATION_WARNING)
{
var superDict = stest.getOutputs();
outputDict = (JObject)superDict["outputs"];
// HACK: Inefficient Just making it work w/o covariance issues
string data = outputDict.ToString(Newtonsoft.Json.Formatting.None);
myDict = JsonConvert.DeserializeObject <IDictionary <string, Object> >(data);
}
else
{
Debug.WriteLine(String.Format("Sinter runstatus={0}, did not retrieve outputs.", stest.runStatus),
"SinterConsumer.DoFinalize");
}
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message, "SinterConsumer.DoFinalize");
process.Output = new Dictionary <string, Object>()
{
{ "exception", ex.Message }
};
if (job.IsTerminated())
{
throw new TerminateException("Job was terminated by producer");
}
job.Error(String.Format("Failed to retrieve the outputs: {0} {1}", ex.Message, ex.StackTrace));
return;
}
Debug.WriteLine("Outputs: " + myDict, this.GetType());
process.Output = myDict;
if (stest.runStatus == sinter.sinter_AppError.si_OKAY)
{
job.Success();
}
else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_ERROR)
{
var msg = String.Join(",", stest.errorsBasic());
Debug.WriteLine("Error: si_SIMULATION_ERROR", "SinterConsumerRun.DoFinalize");
Debug.WriteLine(msg, "SinterConsumerRun.DoFinalize");
// Truncate the msg if it's more than 4000 characters.
// The last 4000 characters are more interesting for debugging.
if (msg.Length > 4000)
{
msg = msg.Substring(msg.Length - 4000);
}
job.Error(String.Format("Error: si_SIMULATION_ERROR: {0}", msg));
if (sim != null)
{
Debug.WriteLine("2) Simulation is not Null",
"SinterConsumerRun.DoFinalize()");
sim.closeSim();
if (sim != null)
{
KillProcessAndChildren(sim.ProcessID);
}
sim = null;
//sim.terminate();
}
}
else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_WARNING)
{
job.Message(String.Join(",", stest.warningsBasic()));
job.Success();
}
else if (stest.runStatus == sinter.sinter_AppError.si_COM_EXCEPTION)
{
Debug.WriteLine("Error: si_COM_EXCEPTION", "SinterConsumerRun.DoFinalize");
job.Error("COM Exception, Server was Terminated");
}
else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_NOT_RUN)
{
Debug.WriteLine("Error: si_SIMULATION_NOT_RUN", "SinterConsumerRun.DoFinalize");
job.Error("Simulation was not Run");
}
else
{
Debug.WriteLine("Error: Unknown", "SinterConsumerRun.DoFinalize");
job.Error(string.Format("Aspen Process Status unknown code {0}.", (int)stest.runStatus));
}
}
/// <summary>
/// DoRun sends inputs specified in job contract to aspen simulation and runs
/// the simulation.
/// </summary>
/// <param name="job"></param>
/// <param name="stest"></param>
/// <param name="combinedDict"></param>
protected void DoRun(IJobConsumerContract job, sinter.ISimulation stest, JObject defaultsDict, JObject inputDict)
{
Debug.WriteLine(String.Format("Send Inputs to Job {0}", job.Id),
"SinterConsumer.DoRun");
try
{
Debug.WriteLine("Default: " + JsonConvert.SerializeObject(defaultsDict), "SinterConsumerRun.DoRun");
Debug.WriteLine("Input: " + JsonConvert.SerializeObject(inputDict), "SinterConsumerRun.DoRun");
if (defaultsDict != null)
{
stest.sendDefaults(defaultsDict);
}
stest.sendInputs(inputDict);
}
catch (Exception ex)
{
Debug.WriteLine(String.Format(
"Error sending Inputs: {0}, traceback {1}",
ex.Message, ex.StackTrace), "SinterConsumerRun.DoRun");
if (job.IsTerminated())
{
throw new TerminateException("Job was terminated by producer");
}
job.Error(String.Format(
"Error sending Inputs: {0}, traceback {1}",
ex.Message, ex.StackTrace));
Debug.WriteLine(String.Format("Moved Job {0} to Error", job.Id),
"SinterConsumer.DoRun");
throw;
}
job.Message("sinter inputs sent, running simulation");
Debug.WriteLine(String.Format("Job {0}: sendInputsToSim ", job.Id),
"SinterConsumer.DoRun");
try
{
running = true;
lock (this)
{
stest.sendInputsToSim();
Debug.WriteLine(String.Format("Job {0}: runSim ", job.Id),
"SinterConsumer.DoRun");
stest.runSim();
Debug.WriteLine(String.Format("Job {0}: recvOutputsFromSim ", job.Id),
"SinterConsumer.DoRun");
if (stest.runStatus == sinter.sinter_AppError.si_OKAY)
{
stest.recvOutputsFromSim();
job.Message("Real Run was complete and successful.");
Debug.WriteLine(String.Format("Real Run was complete and successful. Job Id: {0}", job.Id),
"SinterConsumer.Run");
}
else if (stest.runStatus == sinter.sinter_AppError.si_SIMULATION_WARNING)
{
stest.recvOutputsFromSim();
job.Message("Real Run was completed with warning.");
Debug.WriteLine(String.Format("Real Run was completed with warning. Job Id: {0}", job.Id),
"SinterConsumer.Run");
}
else
{
job.Message(String.Format("Real Run failed, runStatus={0}", stest.runStatus));
Debug.WriteLine(String.Format("Real Run was complete and failed. Job Id: {0}, runStatus={1}",
job.Id, stest.runStatus), "SinterConsumer.Run");
}
}
running = false;
if (isTerminated)
{
throw new TerminateException(
"Monitor thread Terminated Simulation");
}
}
catch (System.Runtime.InteropServices.COMException ex)
{
Debug.WriteLine(String.Format("Moving Job {0} to Error", job.Id),
"SinterConsumerRun.DoRun");
Debug.WriteLine(ex.Message, "SinterConsumerRun.DoRun");
Debug.WriteLine(ex.StackTrace, "SinterConsumerRun.DoRun");
if (this.terminateSimAndJob() == false)
{
Debug.WriteLine("Failed to terminate the simulation",
"SinterConsumerRun.DoRun()");
if (sim != null)
{
Debug.WriteLine("1) Simulation is not Null",
"SinterConsumerRun.DoRun()");
sim.closeSim();
if (sim != null)
{
KillProcessAndChildren(sim.ProcessID);
}
sim = null;
//sim.terminate();
}
}
job.Error(String.Format(
"Error running simulation: {0}, traceback {1}",
ex.Message, ex.StackTrace));
}
catch (Exception ex)
{
Debug.WriteLine(String.Format("Moving Job {0} to Error", job.Id),
"SinterConsumerRun.DoRun");
Debug.WriteLine(ex.Message, "SinterConsumerRun.DoRun");
Debug.WriteLine(ex.StackTrace, "SinterConsumerRun.DoRun");
if (job.IsTerminated())
{
throw new TerminateException("Job was terminated by producer");
}
job.Error(String.Format(
"Error running simulation: {0}, traceback {1}",
ex.Message, ex.StackTrace));
throw;
}
}