/* Terminate: Administrator function, for killing jobs.
         * IJobProducerContract
         */
        public void Terminate()
        {
            string user = AppUtility.GetAppContext().UserName;

            using (TurbineModelContainer container = new TurbineModelContainer())
            {
                Job obj = container.Jobs.Single <Job>(s => s.Id == id);

                if (obj.User.Name != user)
                {
                    throw new AuthorizationError(
                              String.Format("Termination Authorization Denied For {0}", user)
                              );
                }

                if (obj.State == "running" || obj.State == "setup")
                {
                    obj.State    = "terminate";
                    obj.Finished = DateTime.UtcNow;
                }
                else
                {
                    throw new InvalidStateChangeException(
                              String.Format("Cannot terminate a job {0} in state {1}", id, obj.State)
                              );
                }
                container.SaveChanges();
            }
        }
Exemple #2
0
        public List <SampleItem> GetCollection()
        {
            //Debug.Listeners.Add(new TextWriterTraceListener(new System.IO.FileStream(@"C:\logs\hi.txt", System.IO.FileMode.Append)));
            //Debug.WriteLine("GetCollection: " + OperationContext.Current.ServiceSecurityContext.PrimaryIdentity.GetHashCode());
            tsource.TraceEvent(TraceEventType.Start, 0, "GetCollection Trace Event");
            tsource.TraceEvent(TraceEventType.Stop, 0, "GetCollection Trace Event");

            //String username = OperationContext.Current.ServiceSecurityContext.PrimaryIdentity.Name;
            //String username = AppUtility.context.UserName;
            String username = "******";

            try
            {
                username = AppUtility.GetAppContext().UserName;
            }
            catch (Exception ex)
            {
                return(new List <SampleItem>()
                {
                    new SampleItem()
                    {
                        Id = 1,
                        StringValue = ex.ToString()
                    }
                });
            }
            return(new List <SampleItem>()
            {
                new SampleItem()
                {
                    Id = 1, StringValue = "HelloX: " + username
                }
            });
        }
        /// <summary>
        /// copyFromCache:  copies simulation files from cache, return true if cache exists.
        /// </summary>
        /// <param name="job"></param>
        /// <returns></returns>
        private bool copyFromCache(IJobConsumerContract job)
        {
            string cacheDir = Path.Combine(AppUtility.GetAppContext().BaseWorkingDirectory, job.SimulationId.ToString());

            //string shortConfigFileName = "sinter_configuration.txt";
            //configFileName = Path.Combine(job.Process.WorkingDirectory, shortConfigFileName);

            if (Directory.Exists(cacheDir))
            {
                // Shallow Copy cacheDir to Working Directory
                var source = new DirectoryInfo(cacheDir);
                var target = new DirectoryInfo(job.Process.WorkingDirectory);

                /*foreach (FileInfo finfo in source.GetFiles())
                 * {
                 *  Debug.WriteLine(String.Format("Found Cached file {0}: {1}", job.SimulationId, finfo.FullName),
                 *      "SinterConsumer.copyFromCache");
                 *  finfo.CopyTo(Path.Combine(target.ToString(), finfo.Name), true);
                 * }*/

                Debug.WriteLine("Copying Cached files", "SinterConsumerRun.CopyFromCache");
                DirectoryCopy(source.ToString(), target.ToString(), true);
                return(true);
            }
            return(false);
        }
Exemple #4
0
        /// <summary>
        /// copyFilesToDisk copies files from data contract to working directory.  Assumes the sinter configuration file
        /// is named 'configuration' in the job description.  Currently handles all the mapping from 'resource' names
        /// specified in the database schema to file names.
        /// </summary>
        /// <param name="job"></param>
        protected override void copyFilesToDisk(IJobConsumerContract job)
        {
            string cacheDir = Path.Combine(AppUtility.GetAppContext().BaseWorkingDirectory, job.SimulationId.ToString());
            // NOTE: Aspen implementations only
            // Find 'aspenfile'
            // configuration file
            SimpleFile config  = job.GetSimulationInputFiles().Single(f => f.name == "configuration");
            string     content = System.Text.Encoding.ASCII.GetString(config.content);

            var cacheFile = Path.Combine(cacheDir, configFileName);
            var filepath  = Path.Combine(job.Process.WorkingDirectory, configFileName);

            File.WriteAllBytes(cacheFile, config.content);
            File.Copy(cacheFile, filepath);

            Dictionary <String, Object> jsonConfig = JsonConvert.DeserializeObject <Dictionary <String, Object> >(content);
            string modelfilename = (String)jsonConfig["model"];

            SimpleFile model = job.GetSimulationInputFiles().Single(g => g.name == "model");

            cacheFile = Path.Combine(cacheDir, modelfilename);
            filepath  = Path.Combine(job.Process.WorkingDirectory, modelfilename);
            File.WriteAllBytes(cacheFile, model.content);
            File.Copy(cacheFile, filepath);
        }
        /* Submit
         *
         * IJobProducerContract event requires user name check
         * Better solution to use DIJ for Producer.CheckUser
         * and State Checks, throw Application Specific
         * Exceptions.  Move to POCO Data Model first.
         */
        public void Submit()
        {
            string user = AppUtility.GetAppContext().UserName;

            using (TurbineModelContainer container = new TurbineModelContainer())
            {
                Job obj = container.Jobs.Single <Job>(s => s.Id == id);  //Pulls the job that matches id
                if (obj.User.Name != user)
                {
                    throw new ArgumentException(
                              String.Format("Authorization Denied For {0}, Owner {1}", user, obj.User.Name)
                              );
                }
                if (obj.State != "create")
                {
                    throw new InvalidStateChangeException("Violation of state machine");
                }

                // Backup and Config must be valid to submit/run
                ValidateBackup(obj.Simulation);
                ValidateConfig(obj.Simulation);

                obj.State  = "submit";
                obj.Submit = DateTime.UtcNow;

                container.SaveChanges();
            }
        }
        public Turbine.Data.Contract.Behaviors.IProcess Setup()
        {
            string baseDir = AppUtility.GetAppContext().BaseWorkingDirectory;
            Guid   pid     = Guid.Empty;

            using (ProducerContext db = new ProducerContext())
            {
                var job      = db.Jobs.Single(j => j.Count == id);
                var consumer = job.Consumer;

                if (consumer == null)
                {
                    throw new IllegalAccessException("Setup failed, Consumer is not registered");
                }
                if (consumer.status != "up")
                {
                    throw new IllegalAccessException(String.Format(
                                                         "Setup failed, Consumer status {0} != up", consumer.status));
                }
                if (job.ConsumerId != ConsumerId)
                {
                    throw new IllegalAccessException(String.Format(
                                                         "job({0}) consumer access denied {1} != {2}", job.Id, job.ConsumerId, ConsumerId)
                                                     );
                }
                Debug.WriteLine(String.Format("Setup consumer GUID {0}", consumer.Id), this.GetType().Name);

                if (job.State != "locked")
                {
                    throw new InvalidStateChangeException("Violation of state machine");
                }

                if (job.Process == null)
                {
                    throw new InvalidStateChangeException("Process configuration missing");
                }

                processId = job.Process.Id;
                job.State = "setup";
                job.Setup = DateTime.UtcNow;
                job.Process.WorkingDir = System.IO.Path.Combine(baseDir, id.ToString());
                job.Consumer           = (JobConsumer)consumer;
                job.Messages.Add(new Turbine.Data.Entities.Message {
                    Id     = Guid.NewGuid(),
                    Create = DateTime.UtcNow,
                    Value  = String.Format("event=setup,consumer={0}", consumer.Id)
                });
                pid            = job.Process.Id;
                simulationName = job.Simulation.Name;
                db.SaveChanges();
            }
            state = States.SETUP;
            return(Turbine.DataEF6.Contract.ProcessContract.Get(pid));
        }
        public bool UpdateConfiguration(byte[] data)
        {
            string owner = AppUtility.GetAppContext().UserName;

            using (TurbineModelContainer container = new TurbineModelContainer())
            {
                Simulation obj = container.Simulations.Single <Simulation>(s => s.Name == name);
                if (obj.User.Name != owner)
                {
                    throw new ArgumentException("Only owner {0} can update simulation", obj.User.Name);
                }
                obj.Configuration = data;
                container.SaveChanges();
            }
            return(true);
        }
        public Guid Create()
        {
            Guid   session = Guid.NewGuid();
            string owner   = AppUtility.GetAppContext().UserName;

            using (TurbineModelContainer container = new TurbineModelContainer())
            {
                User user = container.Users.Single <User>(s => s.Name == owner);
                container.Sessions.AddObject(new Session()
                {
                    guid = session, User = user, Create = DateTime.UtcNow
                });
                container.SaveChanges();
            }
            return(session);
        }
Exemple #9
0
        /*  Setup: Moves job from submit to setup, concurrency issues to be aware of.
         *     Consumer must be registered before calling this method.
         *
         *      throws OptimisticConcurrencyException if job has already been
         *      grabbed by another consumer.
         *
         */
        public IProcess Setup()
        {
            string           baseDir    = AppUtility.GetAppContext().BaseWorkingDirectory;
            IConsumerContext ctx        = AppUtility.GetConsumerContext();
            Guid             consumerId = ctx.Id;
            Guid             pid        = Guid.Empty;

            using (TurbineModelContainer container = new TurbineModelContainer())
            {
                var consumer = container.JobConsumers
                               .SingleOrDefault <Turbine.Data.JobConsumer>(c => c.Id == consumerId);

                Debug.WriteLine(String.Format("Setup consumer GUID {0}", consumerId), this.GetType().Name);

                if (consumer == null)
                {
                    throw new IllegalAccessException("Setup failed, Consumer is not registered");
                }
                if (consumer.status != "up")
                {
                    throw new IllegalAccessException(String.Format(
                                                         "Setup failed, Consumer status {0} != up", consumer.status));
                }

                Job obj = container.Jobs.Single <Job>(s => s.Id == id);
                if (obj.State != "locked")
                {
                    throw new InvalidStateChangeException("Violation of state machine");
                }

                if (obj.Process == null)
                {
                    throw new InvalidStateChangeException("Process configuration missing");
                }

                obj.State = "setup";
                obj.Setup = DateTime.UtcNow;
                obj.Process.WorkingDir = Path.Combine(baseDir, id.ToString());
                obj.Consumer           = (JobConsumer)consumer;
                pid            = obj.Process.Id;
                simulationName = obj.Simulation.Name;
                container.SaveChanges();
            }
            return(SinterProcessContract.Get(pid));
        }
Exemple #10
0
        public static IJobProducerContract Create(string simulationName, Guid sessionID, bool initialize)
        {
            int      id       = -1;
            IContext ctx      = AppUtility.GetAppContext();
            String   userName = ctx.UserName;

            using (TurbineModelContainer container = new TurbineModelContainer())
            {
                Simulation obj     = container.Simulations.Single <Simulation>(s => s.Name == simulationName);
                User       user    = container.Users.Single <User>(u => u.Name == userName);
                Session    session = container.Sessions.SingleOrDefault <Session>(i => i.guid == sessionID);
                if (session == null)
                {
                    session = new Session()
                    {
                        guid = sessionID, Create = DateTime.UtcNow, User = user
                    };
                    container.Sessions.AddObject(session);
                    container.SaveChanges();
                }

                Job job = new Job()
                {
                    Simulation = obj,
                    State      = "create",
                    Create     = DateTime.UtcNow,
                    Process    = new SinterProcess(),
                    User       = user,
                    Session    = session,
                    Initialize = initialize
                };
                container.SaveChanges();
                id = job.Id;
            }
            return(new AspenJobProducerContract(id));
        }
        public static ISimulationProducerContract Create(string name)
        {
            string owner = AppUtility.GetAppContext().UserName;

            using (TurbineModelContainer container = new TurbineModelContainer())
            {
                User       user = container.Users.Single <User>(s => s.Name == owner);
                Simulation obj  = container.Simulations.SingleOrDefault <Simulation>(s => s.Name == name);
                if (obj != null)
                {
                    throw new InvalidStateChangeException("Simulation with Name {0} already exists");
                }

                container.Simulations.AddObject(new Simulation()
                {
                    Name = name, User = user
                });
                container.SaveChanges();
            }
            AspenSimulationContract sc = new AspenSimulationContract();

            sc.name = name;
            return(sc);
        }
Exemple #12
0
        /*  Setup: Moves job from submit to setup, concurrency issues to be aware of.
         *     Consumer must be registered before calling this method.
         *
         *      throws OptimisticConcurrencyException if job has already been
         *      grabbed by another consumer.
         *
         */
        public IProcess Setup()
        {
            string           baseDir    = AppUtility.GetAppContext().BaseWorkingDirectory;
            IConsumerContext ctx        = AppUtility.GetConsumerContext();
            Guid             consumerId = ctx.Id;

            using (TurbineModelContainer container = new TurbineModelContainer())
            {
                Consumer consumer = container.Consumers
                                    .SingleOrDefault <Consumer>(c => c.guid == consumerId);

                if (consumer == null)
                {
                    throw new IllegalAccessException("Setup failed, Consumer is not registered");
                }

                Job obj = container.Jobs.Single <Job>(s => s.Id == id);
                if (obj.State != "submit")
                {
                    throw new InvalidStateChangeException("Violation of state machine");
                }

                if (obj.Process == null)
                {
                    throw new InvalidStateChangeException("Process configuration missing");
                }

                obj.State = "setup";
                obj.Setup = DateTime.UtcNow;
                obj.Process.WorkingDir = Path.Combine(baseDir, id.ToString());
                obj.JobConsumer        = (JobConsumer)consumer;

                container.SaveChanges();
            }
            return(SinterProcessContract.Get(id));
        }
Exemple #13
0
        /*protected override void copyFilesToDisk(Data.Contract.Behaviors.IJobConsumerContract job)
         * {
         *  Debug.WriteLine("Copying files to disk", GetType().Name);
         *  string cacheDir = Path.Combine(AppUtility.GetAppContext().BaseWorkingDirectory, job.SimulationId.ToString());
         *  Directory.CreateDirectory(cacheDir);
         *
         *  // NOTE: Excel implementations only
         *  // Find 'spreadsheet'
         *  // configuration file
         *  SimpleFile config = job.GetSimulationInputFiles().Single(f => f.name == "configuration");
         *  string content = System.Text.Encoding.ASCII.GetString(config.content);
         *
         *  var cacheFile = Path.Combine(cacheDir, configFileName);
         *  var filepath = Path.Combine(job.Process.WorkingDirectory, configFileName);
         *  Debug.WriteLine("filePath: " + filepath, GetType().Name);
         *  Debug.WriteLine("cacheFile: " + cacheFile, GetType().Name);
         *  File.WriteAllBytes(cacheFile, config.content);
         *  File.Copy(cacheFile, filepath);
         *
         *  Dictionary<String, Object> jsonConfig = JsonConvert.DeserializeObject<Dictionary<String, Object>>(content);
         *  object file_version;
         *  string modelfilename;
         *  // Handling the new format of SinterConfig file
         *  if (jsonConfig.TryGetValue("filetype-version", out file_version) && (double)file_version >= 0.3)
         *  {
         *      Debug.WriteLine("file_version: " + ((double)file_version).ToString(), "ExcelSinterConsumer.copyFilesToDisk");
         *      var modelfileObject = JObject.FromObject(jsonConfig["model"]);
         *      modelfilename = (String)modelfileObject["file"];
         *  }
         *  else
         *  {
         *      Debug.WriteLine("Key 'filetype-version' wasn't specified or < 0.3", "ExcelSinterConsumer.copyFilesToDisk");
         *      modelfilename = (String)jsonConfig["spreadsheet"];
         *  }
         *
         *  SimpleFile model = job.GetSimulationInputFiles().Single(g => g.name == "spreadsheet");
         *  cacheFile = Path.Combine(cacheDir, modelfilename);
         *  filepath = Path.Combine(job.Process.WorkingDirectory, modelfilename);
         *  File.WriteAllBytes(cacheFile, model.content);
         *  File.Copy(cacheFile, filepath);
         * }*/

        protected override void copyFilesToDisk(IJobConsumerContract job)
        {
            string cacheDir = Path.Combine(AppUtility.GetAppContext().BaseWorkingDirectory, job.SimulationId.ToString());

            Directory.CreateDirectory(cacheDir);
            // NOTE: Excel implementations only
            // Find 'spreadsheet'
            // configuration file

            var input_files = job.GetSimulationInputFiles();

            SimpleFile config  = input_files.Single(f => f.name == "configuration");
            string     content = System.Text.Encoding.ASCII.GetString(config.content);

            var cacheFile = Path.Combine(cacheDir, configFileName);
            var filepath  = Path.Combine(job.Process.WorkingDirectory, configFileName);

            File.WriteAllBytes(cacheFile, config.content);
            File.Copy(cacheFile, filepath);

            Dictionary <String, Object> jsonConfig = JsonConvert.DeserializeObject <Dictionary <String, Object> >(content);
            object file_version;
            string excelfilename;

            // Handling the new format of SinterConfig file
            if (jsonConfig.TryGetValue("filetype-version", out file_version) && (double)file_version >= 0.3)
            {
                Debug.WriteLine("file_version: " + ((double)file_version).ToString(), "ExcelSinterConsumer.copyFilesToDisk");
                var excelfileObject = JObject.FromObject(jsonConfig["model"]);
                excelfilename = (String)excelfileObject["file"];
            }
            else
            {
                Debug.WriteLine("Key 'filetype-version' wasn't specified or < 0.3", "ExcelSinterConsumer.copyFilesToDisk");
                excelfilename = (String)jsonConfig["spreadsheet"];
            }

            SimpleFile excelfile = input_files.Single(g => g.name == "spreadsheet");

            cacheFile = Path.Combine(cacheDir, excelfilename);
            filepath  = Path.Combine(job.Process.WorkingDirectory, excelfilename);
            File.WriteAllBytes(cacheFile, excelfile.content);
            File.Copy(cacheFile, filepath);

            foreach (SimpleFile sf in input_files)
            {
                if (sf.name == "configuration" || sf.name == "spreadsheet")
                {
                    continue;
                }
                cacheFile = Path.Combine(cacheDir, sf.name);
                filepath  = Path.Combine(job.Process.WorkingDirectory, sf.name);
                try
                {
                    File.WriteAllBytes(cacheFile, sf.content);
                }
                catch (DirectoryNotFoundException)
                {
                    string[] dirs    = cacheFile.Split('/');
                    string[] d       = dirs.Take <string>(dirs.Length - 1).ToArray <string>();
                    string   dirpath = Path.Combine(d);
                    Debug.WriteLine(String.Format("copyFilesToDisk: create cache directory {0}", dirpath), GetType().Name);
                    Directory.CreateDirectory(dirpath);
                    File.WriteAllBytes(cacheFile, sf.content);
                }
                try
                {
                    File.Copy(cacheFile, filepath);
                }
                catch (DirectoryNotFoundException)
                {
                    string[] dirs    = filepath.Split('/');
                    string[] d       = dirs.Take <string>(dirs.Length - 1).ToArray <string>();
                    string   dirpath = Path.Combine(d);
                    Debug.WriteLine(String.Format("copyFilesToDisk: create directory {0}", dirpath), GetType().Name);
                    Directory.CreateDirectory(dirpath);
                    File.Copy(cacheFile, filepath);
                }
            }
        }
Exemple #14
0
        public static void Run()
        {
            Debug.WriteLine("SinterCosnumerConsole : Running");
            int timeSleepInterval = 1000;
            //int iterations = 60 * 30;
            //int setupIterations = 60 * 10;
            //int postInitIterations = 60 + setupIterations;
            bool             finish      = false;
            String           dir         = AppUtility.GetAppContext().BaseWorkingDirectory;
            IConsumerContext consumerCtx = AppUtility.GetConsumerContext();

            // Register as a consumer, else can't use JobContract
            cc              = AppUtility.GetConsumerRegistrationContract();
            consumerRun     = AppUtility.GetConsumerRunContract();
            consumerMonitor = AppUtility.GetConsumerMonitorContract();

            consumerMonitor.setConsumerRun(consumerRun);

            cc.Register(consumerRun);

            //consumer = AppUtility.GetConsumerRunContract();
            // Run KeepAlive task
            var ktask = new Task <bool>(() => KeepAlive());

            ktask.Start();
            bool stop             = false;
            bool cancelKeyPressed = false;
            var  obj = new Object();

            System.Console.CancelKeyPress += delegate
            {
                lock (obj)
                {
                    stop             = true;
                    cancelKeyPressed = true;
                    //consumer.CleanUp();
                    Debug.WriteLine("Exit Unregister Consumer ", "SinterConsumerConsole");
                    try
                    {
                        cc.UnRegister();
                    }
                    catch (Exception ex)
                    {
                        Debug.WriteLine("Exception caught from UnRegister: " + ex.Message, "SinterConsumerConsole");
                        Debug.WriteLine(ex.StackTrace, "SinterConsumerConsole");
                    }
                    // lock will block if "SinterConsumer.DoConfigure --> sim.openSim()" is being called
                    Debug.WriteLine("lock consumer", "SinterConsumerConsole");
                    int wait_seconds = 30;
                    if (Monitor.TryEnter(consumerRun, new TimeSpan(0, 0, wait_seconds)))
                    {
                        consumerMonitor.Monitor(cancelKeyPressed);
                        Debug.WriteLine("Obtained ConsumerRun Lock: Monitor Terminate", "SinterConsumerConsole");
                    }
                    else
                    {
                        consumerMonitor.Monitor(cancelKeyPressed);
                        Debug.WriteLine("Failed to Obtain ConsumerRun Lock: Monitor Terminate", "SinterConsumerConsole");
                    }
                    Debug.WriteLine("Consumer Exit", "SinterConsumerConsole");
                }
            };

            Task <int> taskMonitor         = null;
            bool       taskMonitorFinished = true;

            while (stop == false)
            {
                if (!Directory.Exists(dir))
                {
                    Debug.WriteLine(String.Format("Base Directory does not exist: {0}", dir), "SinterConsumerConsole");
                    OnExit(ERROR_PATH_NOT_FOUND);
                }
                try
                {
                    var acl = Directory.GetAccessControl(dir);
                }
                catch (UnauthorizedAccessException)
                {
                    Debug.WriteLine(String.Format("Base Directory is not writable: {0}", dir), "SinterConsumerConsole");
                    OnExit(ERROR_ACCESS_DENIED);
                }

                Debug.WriteLine("> New Task", "SinterConsumerConsole");

                var taskRun = new Task <Boolean>(() => consumerRun.Run());

                if (taskMonitorFinished)
                {
                    Debug.WriteLine("> New Monitor Task", "SinterConsumerConsole.Run");
                    taskMonitorFinished = false;
                    taskMonitor         = new Task <int>(() => consumerMonitor.Monitor(false));
                    taskMonitor.Start();
                }

                finish = false;
                int  i = 0;
                bool hasInitialized = false;
                bool hasRun         = false;
                taskRun.Start();

                while (stop == false)
                {
                    if (taskMonitor.IsCompleted)
                    {
                        Debug.WriteLine("taskMonitor.IsCompleted", "SinterConsumerConsole.Run");
                        taskMonitorFinished = true;
                        System.Threading.Thread.Sleep(2000);
                        break;
                    }

                    if (cc.GetStatus() == "down")
                    {
                        Debug.WriteLine("ConsumerContract: Consumer is Down", "SinterConsumerConsole.Run");
                        stop = true;
                        Debug.WriteLine("Exit Unregister Consumer ", "SinterConsumerConsole.Run");
                        try
                        {
                            cc.UnRegister();
                        }
                        catch (Exception ex)
                        {
                            Debug.WriteLine("Exception caught from UnRegister: " + ex.Message, "SinterConsumerConsole.Run");
                            Debug.WriteLine(ex.StackTrace, "SinterConsumerConsole.Run");
                        }
                        // lock will block if "SinterConsumer.DoConfigure --> sim.openSim()" is being called
                        Debug.WriteLine("lock consumer", "SinterConsumerConsole.Run");
                        int wait_seconds = 30;

                        if (Monitor.TryEnter(consumerRun, new TimeSpan(0, 0, wait_seconds)))
                        {
                            consumerMonitor.Monitor(true);
                            Debug.WriteLine("Obtained ConsumerRun Lock: Monitor Terminate", "SinterConsumerConsole.Run");
                        }
                        else
                        {
                            consumerMonitor.Monitor(true);
                            Debug.WriteLine("Failed to Obtain ConsumerRun Lock: Monitor Terminate", "SinterConsumerConsole.Run");
                        }
                        break;
                    }

                    i++;
                    Debug.WriteLine("top");

                    try
                    {
                        finish = taskRun.Wait(timeSleepInterval);
                    }
                    catch (Exception ex)
                    {
                        Debug.WriteLine("Exception caught from Run: " + ex.Message, "SinterConsumerConsole");
                        Debug.WriteLine(ex.StackTrace, "SinterConsumerConsole");
                        Debug.WriteLine("InnerException: " + ex.InnerException, "SinterConsumerConsole");
                        System.Threading.Thread.Sleep(2000);

                        /*// check if job was terminated, InnerException would be InvalidStateChangeException
                         * if (ex.InnerException is InvalidStateChangeException)
                         * {
                         *  consumer.CheckTerminate();
                         * }
                         */
                        break;
                    }
                    //Debug.WriteLine(">> Waiting!!!! " + i + " " + iterations + " " + setupIterations + " " + postInitIterations, "SinterConsumerConsole");
                    Debug.WriteLine(">> Waiting " + i, "SinterConsumerConsole");
                    if (finish)
                    {
                        if (taskRun.IsFaulted)
                        {
                            Debug.WriteLine("Faulted: " + taskRun.Exception, "SinterConsumerConsole");
                        }
                        else if (taskRun.IsCompleted)
                        {
                            Debug.WriteLine("Completed", "SinterConsumerConsole");
                        }
                        else
                        {
                            throw new Exception("Task Unexpected status?");
                        }

                        if (!taskRun.Result)
                        {
                            // Nothing on Job Queue
                            System.Threading.Thread.Sleep(4000);
                        }
                        break;
                    }
                    // DETECT HANGS
                    if (consumerMonitor.IsSimulationInitializing)
                    {
                        hasInitialized = true;
                        Debug.WriteLine("> Simulation Initializing: " + setupIterations);
                        if (i >= setupIterations)
                        {
                            // Allows SinterConsumer to attempt to reopen the simulation
                            // IF attempts max out in SinterConsumer, throw Exception
                            // and task.IsFaulted and job will move to error
                            Debug.WriteLine("HANG SETUP, TRY AGAIN", "SinterConsumerConsole");

                            consumerMonitor.MonitorTerminate();

                            Debug.WriteLine("Killed Aspen Process", "SinterConsumerConsole");
                            i = 0;
                            continue;
                        }
                    }
                    else if (consumerMonitor.IsEngineRunning)
                    {
                        hasRun = true;
                        Debug.WriteLine("> Engine Running: " + iterations);
                        if (i >= iterations)
                        {
                            // Assume Simulation is done Initializing
                            Debug.WriteLine("HANG RUNNING EXIT", "SinterConsumerConsole");
                            //consumer.MonitorTerminateAPWN();
                            try
                            {
                                //consumer.MonitorTerminateAspenEngine();
                                //consumer.MonitorTerminateRunning();
                                consumerMonitor.MonitorTerminate();
                            }
                            catch (TerminateException ex)
                            {
                                Debug.WriteLine(ex.Message, "SinterConsumerConsole");
                            }
                        }
                    }
                    else if (hasInitialized && !hasRun)
                    {
                        // Stuck
                        if (i >= postInitIterations)
                        {
                            // Assume Simulation is done Initializing
                            Debug.WriteLine("HANG POST INIT, TRY AGAIN", "SinterConsumerConsole");
                            consumerMonitor.MonitorTerminate();
                            Debug.WriteLine("Killed Aspen instances", "SinterConsumerConsole");
                            i = 0;
                            continue;
                        }
                    }
                    else if (i % 60 == 0)
                    {
                        Debug.WriteLine(String.Format("Waiting {0}", i), "SinterConsumerConsole");
                    }
                } // end of while
                Debug.WriteLine("Stop called", "SinterConsumerConsole");
            }     // end of while

            lock (obj)
            {
                Debug.WriteLine("Stop and cleanup", "SinterConsumerConsole");

                consumerMonitor.MonitorTerminate();
            }
            //consumer.CleanUp();
            //OnExit(ERROR_SUCCESS);
        }
        /// <summary>
        /// copyFilesToDisk copies files from data contract to working directory.  Assumes the sinter configuration file
        /// is named 'configuration' in the job description.  Currently handles all the mapping from 'resource' names
        /// specified in the database schema to file names.
        /// </summary>
        /// <param name="job"></param>
        protected override void copyFilesToDisk(IJobConsumerContract job)
        {
            string cacheDir = Path.Combine(AppUtility.GetAppContext().BaseWorkingDirectory, job.SimulationId.ToString());

            Directory.CreateDirectory(cacheDir);
            // NOTE: Aspen implementations only
            // Find 'aspenfile'
            // configuration file

            var input_files = job.GetSimulationInputFiles();

            SimpleFile config  = input_files.Single(f => f.name == "configuration");
            string     content = System.Text.Encoding.ASCII.GetString(config.content);

            var cacheFile = Path.Combine(cacheDir, configFileName);
            var filepath  = Path.Combine(job.Process.WorkingDirectory, configFileName);

            File.WriteAllBytes(cacheFile, config.content);
            File.Copy(cacheFile, filepath);

            Dictionary <String, Object> jsonConfig = JsonConvert.DeserializeObject <Dictionary <String, Object> >(content);
            string aspenfilename = (String)jsonConfig["aspenfile"];

            SimpleFile aspenfile = input_files.Single(g => g.name == "aspenfile");

            cacheFile = Path.Combine(cacheDir, aspenfilename);
            filepath  = Path.Combine(job.Process.WorkingDirectory, aspenfilename);
            File.WriteAllBytes(cacheFile, aspenfile.content);
            File.Copy(cacheFile, filepath);

            foreach (SimpleFile sf in input_files)
            {
                if (sf.name == "configuration" || sf.name == "aspenfile")
                {
                    continue;
                }
                cacheFile = Path.Combine(cacheDir, sf.name);
                filepath  = Path.Combine(job.Process.WorkingDirectory, sf.name);
                try
                {
                    File.WriteAllBytes(cacheFile, sf.content);
                }
                catch (DirectoryNotFoundException)
                {
                    string[] dirs    = cacheFile.Split('/');
                    string[] d       = dirs.Take <string>(dirs.Length - 1).ToArray <string>();
                    string   dirpath = Path.Combine(d);
                    Debug.WriteLine(String.Format("copyFilesToDisk: create cache directory {0}", dirpath), GetType().Name);
                    Directory.CreateDirectory(dirpath);
                    File.WriteAllBytes(cacheFile, sf.content);
                }
                try
                {
                    File.Copy(cacheFile, filepath);
                }
                catch (DirectoryNotFoundException)
                {
                    string[] dirs    = filepath.Split('/');
                    string[] d       = dirs.Take <string>(dirs.Length - 1).ToArray <string>();
                    string   dirpath = Path.Combine(d);
                    Debug.WriteLine(String.Format("copyFilesToDisk: create directory {0}", dirpath), GetType().Name);
                    Directory.CreateDirectory(dirpath);
                    File.Copy(cacheFile, filepath);
                }
            }
        }