private void AuthorizeConsumer(Job obj)
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerGUID = ctx.Id;
Guid jobGUID = obj.Consumer.Id;
if (jobGUID != Guid.Empty && jobGUID != consumerGUID)
{
throw new IllegalAccessException(String.Format(
"job({0}) consumer access denied {1} != {2}", obj.Id, jobGUID, consumerGUID)
);
}
}
public void Error()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Debug.WriteLine(String.Format("Error as {0}, {1}", consumerId), this.GetType().Name);
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = (JobConsumer)container.JobConsumers.Single <Turbine.Data.JobConsumer>(s => s.Id == consumerId);
consumer.status = "error";
container.SaveChanges();
}
}
public void Error()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Debug.WriteLine(String.Format("Error as {0}, {1}", consumerId), this.GetType().Name);
/*
* using (TurbineModelContainer container = new TurbineModelContainer())
* {
* JobConsumer consumer = (JobConsumer)container.Consumers.First<Turbine.Data.Consumer>(s => s.guid == consumerId);
* consumer.status = "error";
* container.SaveChanges();
* }
* */
}
public void KeepAlive()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Debug.WriteLine(String.Format("KeepAlive Consumer {0}", consumerId), this.GetType().Name);
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = (JobConsumer)container.JobConsumers.Single <Turbine.Data.JobConsumer>(s => s.Id == consumerId);
if (consumer.status == "up")
{
consumer.keepalive = DateTime.UtcNow;
}
container.SaveChanges();
}
}
/* UnRegister
* If state is "up" move to "down", else leave alone.
*/
public void UnRegister()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Debug.WriteLine(String.Format("UnRegister Consumer {0}", consumerId), this.GetType().Name);
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = (JobConsumer)container.JobConsumers.First <Turbine.Data.JobConsumer>(s => s.Id == consumerId);
if (consumer.status == "up")
{
consumer.status = "down";
}
container.SaveChanges();
}
}
/* Setup: Moves job from submit to setup, concurrency issues to be aware of.
* Consumer must be registered before calling this method.
*
* throws OptimisticConcurrencyException if job has already been
* grabbed by another consumer.
*
*/
public IProcess Setup()
{
string baseDir = AppUtility.GetAppContext().BaseWorkingDirectory;
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Guid pid = Guid.Empty;
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = container.JobConsumers
.SingleOrDefault <Turbine.Data.JobConsumer>(c => c.Id == consumerId);
Debug.WriteLine(String.Format("Setup consumer GUID {0}", consumerId), this.GetType().Name);
if (consumer == null)
{
throw new IllegalAccessException("Setup failed, Consumer is not registered");
}
if (consumer.status != "up")
{
throw new IllegalAccessException(String.Format(
"Setup failed, Consumer status {0} != up", consumer.status));
}
Job obj = container.Jobs.Single <Job>(s => s.Id == id);
if (obj.State != "locked")
{
throw new InvalidStateChangeException("Violation of state machine");
}
if (obj.Process == null)
{
throw new InvalidStateChangeException("Process configuration missing");
}
obj.State = "setup";
obj.Setup = DateTime.UtcNow;
obj.Process.WorkingDir = Path.Combine(baseDir, id.ToString());
obj.Consumer = (JobConsumer)consumer;
pid = obj.Process.Id;
simulationName = obj.Simulation.Name;
container.SaveChanges();
}
return(SinterProcessContract.Get(pid));
}
public void Register()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
String hostname = ctx.Hostname;
Debug.WriteLine(String.Format("Register as {0}, {1}", consumerId, hostname), this.GetType().Name);
using (NHibernate.ISession session = SessionFactory.OpenSession())
{
using (NHibernate.ITransaction trans = session.BeginTransaction())
{
var consumer = new JobConsumer()
{
Id = consumerId, hostname = hostname, status = "up"
};
}
}
}
public void Register()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
String hostname = ctx.Hostname;
Debug.WriteLine(String.Format("Register as {0}, {1}", consumerId, hostname), this.GetType().Name);
using (TurbineModelContainer container = new TurbineModelContainer())
{
container.Consumers.AddObject(new JobConsumer()
{
guid = consumerId,
hostname = hostname,
status = "up"
}
);
container.SaveChanges();
}
}
/* Setup: Moves job from submit to setup, concurrency issues to be aware of.
* Consumer must be registered before calling this method.
*
* throws OptimisticConcurrencyException if job has already been
* grabbed by another consumer.
*
*/
public IProcess Setup()
{
string baseDir = AppUtility.GetAppContext().BaseWorkingDirectory;
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
using (TurbineModelContainer container = new TurbineModelContainer())
{
Consumer consumer = container.Consumers
.SingleOrDefault <Consumer>(c => c.guid == consumerId);
if (consumer == null)
{
throw new IllegalAccessException("Setup failed, Consumer is not registered");
}
Job obj = container.Jobs.Single <Job>(s => s.Id == id);
if (obj.State != "submit")
{
throw new InvalidStateChangeException("Violation of state machine");
}
if (obj.Process == null)
{
throw new InvalidStateChangeException("Process configuration missing");
}
obj.State = "setup";
obj.Setup = DateTime.UtcNow;
obj.Process.WorkingDir = Path.Combine(baseDir, id.ToString());
obj.JobConsumer = (JobConsumer)consumer;
container.SaveChanges();
}
return(SinterProcessContract.Get(id));
}
public IJobConsumerContract GetNext(IConsumerRun run)
{
apps = run.SupportedApplications;
IJobConsumerContract contract = null;
if (apps.Length == 0)
{
throw new JobQueueException("Invalid JobQueue configuration: no applications specified");
}
IConsumerContext appContext = AppUtility.GetConsumerContext();
string simulationName = appContext.BindSimulationName;
using (ProducerContext db = new ProducerContext())
{
Job job = null;
foreach (var appName in apps)
{
Debug.WriteLine("Get Job for Application: " + appName, "DBJobQueue.GetNext");
if (String.IsNullOrEmpty(simulationName))
{
job = db.Jobs.OrderByDescending(j => j.Submit).
Where(j => j.State == "submit" && j.ConsumerId == run.ConsumerId &&
j.Simulation.ApplicationName == appName).FirstOrDefault();
if (job == null)
{
job = db.Jobs.OrderByDescending(j => j.Submit).
Where(j => j.State == "submit" && j.ConsumerId == null &&
j.Simulation.ApplicationName == appName).FirstOrDefault();
}
}
else
{
job = db.Jobs.OrderByDescending(j => j.Submit).
Where(j => j.State == "submit" && j.ConsumerId == run.ConsumerId &&
j.Simulation.Name == simulationName && j.Simulation.ApplicationName == appName).FirstOrDefault();
if (job == null)
{
job = db.Jobs.OrderByDescending(j => j.Submit).
Where(j => j.State == "submit" && j.ConsumerId == null &&
j.Simulation.Name == simulationName && j.Simulation.ApplicationName == appName).FirstOrDefault();
}
}
var consumer = db.Consumers.Single(c => c.Id == run.ConsumerId);
if (job != null)
{
job.State = "locked";
job.Consumer = consumer;
db.SaveChanges();
contract = new ConsumerJobContract(job.Count, run.ConsumerId);
Debug.WriteLine(String.Format("Found Job({0}): {1},{2}", job.Id, job.Simulation.Name, job.Simulation.ApplicationName), "DBJobQueue.GetNext");
break;
}
}
}
return(contract);
}
// Consumer grabs job in submit state off queue
public IJobConsumerContract GetNext()
{
IJobConsumerContract contract = new AspenJobConsumerContract();;
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = container.JobConsumers
.SingleOrDefault <Turbine.Data.JobConsumer>(c => c.Id == consumerId);
if (consumer == null)
{
throw new IllegalAccessException("GetNext failed, Consumer is not registered");
}
if (consumer.status != "up")
{
throw new IllegalAccessException(String.Format(
"GetNext failed, Consumer status {0} != up", consumer.status));
}
//Job obj = container.Jobs.OrderBy("it.Submit").
// FirstOrDefault<Job>(s => s.State == "submit");
//obj.Simulation.ApplicationName
Job obj = null;
// TODO: CHange the WHERE clause to do an OR between all available applications
if (applications == null)
{
System.Data.Objects.ObjectQuery <Turbine.Data.Job> query = container.Jobs;
obj = query.OrderBy("it.Submit").FirstOrDefault <Job>(s => s.State == "submit");
}
else
{
foreach (string appName in applications)
{
System.Data.Objects.ObjectQuery <Turbine.Data.Job> query = container.Jobs;
query = query.Where("it.Simulation.ApplicationName = @appName",
new System.Data.Objects.ObjectParameter("appName", appName));
obj = query.OrderBy("it.Submit").FirstOrDefault <Job>(s => s.State == "submit");
if (obj != null)
{
break;
}
}
}
if (obj == null)
{
return(null);
}
// Need to Check that Consumer is up
obj.State = "locked";
try
{
container.SaveChanges();
}
catch (System.Data.OptimisticConcurrencyException)
{
Debug.WriteLine("OptimisticConcurrencyException: Failed attempt to lock job", GetType().Name);
return(null);
}
contract.Id = obj.Id;
}
return(contract);
}
//[ClassInitialize()]
public static void ClassInit(TestContext context)
{
using (TurbineModelContainer container = new TurbineModelContainer())
{
foreach (User obj in container.Users)
{
container.Users.DeleteObject(obj);
}
foreach (Simulation obj in container.Simulations)
{
container.Simulations.DeleteObject(obj);
}
foreach (Job obj in container.Jobs)
{
container.Jobs.DeleteObject(obj);
}
foreach (AspenProcessError obj in container.AspenProcessErrors)
{
container.AspenProcessErrors.DeleteObject(obj);
}
foreach (SinterProcess obj in container.SinterProcesses)
{
container.SinterProcesses.DeleteObject(obj);
}
foreach (Consumer obj in container.Consumers)
{
container.Consumers.DeleteObject(obj);
}
foreach (Session obj in container.Sessions)
{
container.Sessions.DeleteObject(obj);
}
foreach (Message obj in container.Messages)
{
container.Messages.DeleteObject(obj);
}
container.SaveChanges();
byte[] bt = System.Text.Encoding.ASCII.GetBytes(
File.ReadAllText(simulationBackupFile)
);
byte[] cb = System.Text.Encoding.ASCII.GetBytes(
File.ReadAllText(simulationConfigFile)
);
simulationBackupMD5 = GetMd5Hash(bt);
simulationConfigMD5 = GetMd5Hash(cb);
var jsonDefaults = @"{""P_abs_top"":15.0,""abs_ht_wash"":5.0,""abs_ht_mid"":10.7,""abs_ic_dT"":-11.3,""P_sol_pump"":30.0,""lean_load"":0.178,""P_regen_top"":21.2,""cond_T_regen"":121.0,""ht_regen"":13.0,""slv_cool_01"":130.0,""lr_rich_T"":207.0,""input_s[0,0]"":134.4,""input_s[0,1]"":16.0,""input_s[0,2]"":959553.0,""input_s[0,3]"":0.071,""input_s[0,4]"":0.0,""input_s[0,5]"":0.0,""input_s[0,6]"":0.211,""input_s[0,7]"":0.0,""input_s[0,8]"":0.0,""input_s[0,9]"":0.0,""input_s[0,10]"":0.0,""input_s[0,11]"":0.0,""input_s[0,12]"":0.0,""input_s[0,13]"":0.0,""input_s[0,14]"":0.0,""input_s[0,15]"":0.718,""input_s[1,0]"":126.0,""input_s[1,1]"":30.0,""input_s[1,2]"":4319500.4116,""input_s[1,3]"":0.66207726067,""input_s[1,4]"":0.28374739743,""input_s[1,5]"":0.0,""input_s[1,6]"":0.0541753419,""input_s[1,7]"":0.0,""input_s[1,8]"":0.0,""input_s[1,9]"":0.0,""input_s[1,10]"":0.0,""input_s[1,11]"":0.0,""input_s[1,12]"":0.0,""input_s[1,13]"":0.0,""input_s[1,14]"":0.0,""input_s[1,15]"":0.0,""input_s[2,0]"":120.0,""input_s[2,1]"":30.0,""input_s[2,2]"":10000.0,""input_s[2,3]"":1.0,""input_s[2,4]"":0.0,""input_s[2,5]"":0.0,""input_s[2,6]"":0.0,""input_s[2,7]"":0.0,""input_s[2,8]"":0.0,""input_s[2,9]"":0.0,""input_s[2,10]"":0.0,""input_s[2,11]"":0.0,""input_s[2,12]"":0.0,""input_s[2,13]"":0.0,""input_s[2,14]"":0.0,""input_s[2,15]"":0.0,""eq_par[0,0]"":0.7996,""eq_par[0,1]"":-8094.81,""eq_par[0,2]"":0.0,""eq_par[0,3]"":-0.007484,""eq_par[1,0]"":98.566,""eq_par[1,1]"":1353.8,""eq_par[1,2]"":-14.3043,""eq_par[1,3]"":0.0,""eq_par[2,0]"":216.049,""eq_par[2,1]"":-12431.7,""eq_par[2,2]"":-35.4819,""eq_par[2,3]"":0.0,""eq_par[3,0]"":1.282562,""eq_par[3,1]"":-3456.179,""eq_par[3,2]"":0.0,""eq_par[3,3]"":0.0,""eq_par[4,0]"":132.899,""eq_par[4,1]"":-13445.9,""eq_par[4,2]"":-22.4773,""eq_par[4,3]"":0.0}";
System.Text.ASCIIEncoding encoding = new System.Text.ASCIIEncoding();
Byte[] defaultBytes = encoding.GetBytes(jsonDefaults);
User u = null;
Simulation s = null;
Job job = null;
SinterProcess p = null;
Session session = null;
//var consumer = new JobConsumer();
//consumer.guid = Guid.NewGuid();
//consumer.hostname = "test.host";
//container.Consumers.AddObject(consumer);
//container.SaveChanges();
IConsumerRegistrationContract reg = new ConsumerRegistrationContract();
reg.Register();
IConsumerContext ctx = AppUtility.GetConsumerContext();
JobConsumer consumer = (JobConsumer)container.Consumers.First <Consumer>(r => r.guid == ctx.Id);
for (int i = 1; i < 11; i++)
{
Debug.WriteLine("{0}", i);
u = new User()
{
Name = "test" + i, Token = "qwerty"
};
container.Users.AddObject(u);
s = new Simulation()
{
Name = "testSim" + i,
Backup = bt,
Configuration = cb,
Defaults = defaultBytes,
User = u
};
session = new Session();
var guid = Guid.NewGuid();
Debug.WriteLine("GUID: " + guid);
session.guid = guid;
session.Create = DateTime.UtcNow;
session.User = u;
container.Sessions.AddObject(session);
p = new SinterProcess();
job = new Job()
{
State = "create",
Create = DateTime.UtcNow,
Simulation = s,
User = u,
Process = p,
Session = session,
JobConsumer = consumer
};
var msg = new Message();
msg.Value = "test harness";
job.Messages.Add(msg);
for (int j = 1; j < 4; j++)
{
p.Errors.Add(new AspenProcessError()
{
Name = String.Format("Block{0}{1}", i, j),
Type = "block",
Error = "TEST" + i
});
}
}
container.SaveChanges();
}
}
public static void Run()
{
Debug.WriteLine("SinterCosnumerConsole : Running");
int timeSleepInterval = 1000;
//int iterations = 60 * 30;
//int setupIterations = 60 * 10;
//int postInitIterations = 60 + setupIterations;
bool finish = false;
String dir = AppUtility.GetAppContext().BaseWorkingDirectory;
IConsumerContext consumerCtx = AppUtility.GetConsumerContext();
// Register as a consumer, else can't use JobContract
cc = AppUtility.GetConsumerRegistrationContract();
consumerRun = AppUtility.GetConsumerRunContract();
consumerMonitor = AppUtility.GetConsumerMonitorContract();
consumerMonitor.setConsumerRun(consumerRun);
cc.Register(consumerRun);
//consumer = AppUtility.GetConsumerRunContract();
// Run KeepAlive task
var ktask = new Task <bool>(() => KeepAlive());
ktask.Start();
bool stop = false;
bool cancelKeyPressed = false;
var obj = new Object();
System.Console.CancelKeyPress += delegate
{
lock (obj)
{
stop = true;
cancelKeyPressed = true;
//consumer.CleanUp();
Debug.WriteLine("Exit Unregister Consumer ", "SinterConsumerConsole");
try
{
cc.UnRegister();
}
catch (Exception ex)
{
Debug.WriteLine("Exception caught from UnRegister: " + ex.Message, "SinterConsumerConsole");
Debug.WriteLine(ex.StackTrace, "SinterConsumerConsole");
}
// lock will block if "SinterConsumer.DoConfigure --> sim.openSim()" is being called
Debug.WriteLine("lock consumer", "SinterConsumerConsole");
int wait_seconds = 30;
if (Monitor.TryEnter(consumerRun, new TimeSpan(0, 0, wait_seconds)))
{
consumerMonitor.Monitor(cancelKeyPressed);
Debug.WriteLine("Obtained ConsumerRun Lock: Monitor Terminate", "SinterConsumerConsole");
}
else
{
consumerMonitor.Monitor(cancelKeyPressed);
Debug.WriteLine("Failed to Obtain ConsumerRun Lock: Monitor Terminate", "SinterConsumerConsole");
}
Debug.WriteLine("Consumer Exit", "SinterConsumerConsole");
}
};
Task <int> taskMonitor = null;
bool taskMonitorFinished = true;
while (stop == false)
{
if (!Directory.Exists(dir))
{
Debug.WriteLine(String.Format("Base Directory does not exist: {0}", dir), "SinterConsumerConsole");
OnExit(ERROR_PATH_NOT_FOUND);
}
try
{
var acl = Directory.GetAccessControl(dir);
}
catch (UnauthorizedAccessException)
{
Debug.WriteLine(String.Format("Base Directory is not writable: {0}", dir), "SinterConsumerConsole");
OnExit(ERROR_ACCESS_DENIED);
}
Debug.WriteLine("> New Task", "SinterConsumerConsole");
var taskRun = new Task <Boolean>(() => consumerRun.Run());
if (taskMonitorFinished)
{
Debug.WriteLine("> New Monitor Task", "SinterConsumerConsole.Run");
taskMonitorFinished = false;
taskMonitor = new Task <int>(() => consumerMonitor.Monitor(false));
taskMonitor.Start();
}
finish = false;
int i = 0;
bool hasInitialized = false;
bool hasRun = false;
taskRun.Start();
while (stop == false)
{
if (taskMonitor.IsCompleted)
{
Debug.WriteLine("taskMonitor.IsCompleted", "SinterConsumerConsole.Run");
taskMonitorFinished = true;
System.Threading.Thread.Sleep(2000);
break;
}
if (cc.GetStatus() == "down")
{
Debug.WriteLine("ConsumerContract: Consumer is Down", "SinterConsumerConsole.Run");
stop = true;
Debug.WriteLine("Exit Unregister Consumer ", "SinterConsumerConsole.Run");
try
{
cc.UnRegister();
}
catch (Exception ex)
{
Debug.WriteLine("Exception caught from UnRegister: " + ex.Message, "SinterConsumerConsole.Run");
Debug.WriteLine(ex.StackTrace, "SinterConsumerConsole.Run");
}
// lock will block if "SinterConsumer.DoConfigure --> sim.openSim()" is being called
Debug.WriteLine("lock consumer", "SinterConsumerConsole.Run");
int wait_seconds = 30;
if (Monitor.TryEnter(consumerRun, new TimeSpan(0, 0, wait_seconds)))
{
consumerMonitor.Monitor(true);
Debug.WriteLine("Obtained ConsumerRun Lock: Monitor Terminate", "SinterConsumerConsole.Run");
}
else
{
consumerMonitor.Monitor(true);
Debug.WriteLine("Failed to Obtain ConsumerRun Lock: Monitor Terminate", "SinterConsumerConsole.Run");
}
break;
}
i++;
Debug.WriteLine("top");
try
{
finish = taskRun.Wait(timeSleepInterval);
}
catch (Exception ex)
{
Debug.WriteLine("Exception caught from Run: " + ex.Message, "SinterConsumerConsole");
Debug.WriteLine(ex.StackTrace, "SinterConsumerConsole");
Debug.WriteLine("InnerException: " + ex.InnerException, "SinterConsumerConsole");
System.Threading.Thread.Sleep(2000);
/*// check if job was terminated, InnerException would be InvalidStateChangeException
* if (ex.InnerException is InvalidStateChangeException)
* {
* consumer.CheckTerminate();
* }
*/
break;
}
//Debug.WriteLine(">> Waiting!!!! " + i + " " + iterations + " " + setupIterations + " " + postInitIterations, "SinterConsumerConsole");
Debug.WriteLine(">> Waiting " + i, "SinterConsumerConsole");
if (finish)
{
if (taskRun.IsFaulted)
{
Debug.WriteLine("Faulted: " + taskRun.Exception, "SinterConsumerConsole");
}
else if (taskRun.IsCompleted)
{
Debug.WriteLine("Completed", "SinterConsumerConsole");
}
else
{
throw new Exception("Task Unexpected status?");
}
if (!taskRun.Result)
{
// Nothing on Job Queue
System.Threading.Thread.Sleep(4000);
}
break;
}
// DETECT HANGS
if (consumerMonitor.IsSimulationInitializing)
{
hasInitialized = true;
Debug.WriteLine("> Simulation Initializing: " + setupIterations);
if (i >= setupIterations)
{
// Allows SinterConsumer to attempt to reopen the simulation
// IF attempts max out in SinterConsumer, throw Exception
// and task.IsFaulted and job will move to error
Debug.WriteLine("HANG SETUP, TRY AGAIN", "SinterConsumerConsole");
consumerMonitor.MonitorTerminate();
Debug.WriteLine("Killed Aspen Process", "SinterConsumerConsole");
i = 0;
continue;
}
}
else if (consumerMonitor.IsEngineRunning)
{
hasRun = true;
Debug.WriteLine("> Engine Running: " + iterations);
if (i >= iterations)
{
// Assume Simulation is done Initializing
Debug.WriteLine("HANG RUNNING EXIT", "SinterConsumerConsole");
//consumer.MonitorTerminateAPWN();
try
{
//consumer.MonitorTerminateAspenEngine();
//consumer.MonitorTerminateRunning();
consumerMonitor.MonitorTerminate();
}
catch (TerminateException ex)
{
Debug.WriteLine(ex.Message, "SinterConsumerConsole");
}
}
}
else if (hasInitialized && !hasRun)
{
// Stuck
if (i >= postInitIterations)
{
// Assume Simulation is done Initializing
Debug.WriteLine("HANG POST INIT, TRY AGAIN", "SinterConsumerConsole");
consumerMonitor.MonitorTerminate();
Debug.WriteLine("Killed Aspen instances", "SinterConsumerConsole");
i = 0;
continue;
}
}
else if (i % 60 == 0)
{
Debug.WriteLine(String.Format("Waiting {0}", i), "SinterConsumerConsole");
}
} // end of while
Debug.WriteLine("Stop called", "SinterConsumerConsole");
} // end of while
lock (obj)
{
Debug.WriteLine("Stop and cleanup", "SinterConsumerConsole");
consumerMonitor.MonitorTerminate();
}
//consumer.CleanUp();
//OnExit(ERROR_SUCCESS);
}
