Esempio n. 1
0
        public void GetMessageStopTestShouldPass()
        {
            AutomationMessage message = Automation.GetMessage("*2*0*31##");

            Assert.AreEqual <WHAT>(message.What, WHAT.AutomationStop);
            Assert.AreEqual <int>(message.Where.A, 3);
            Assert.AreEqual <int>(message.Where.PL, 1);
        }
        public void TestNullISOs()
        {
            sut.EmulateServiceStart(null);
            DirectoryInfo tempDir = new DirectoryInfo(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tempdll"));

            tempDir.Create();
            try
            {
                List <ExecutablePackage> packages = new List <ExecutablePackage>();
                ExecutablePackage        package  = new ExecutablePackage("mock.xml", tempDir.FullName, "TestJobManager.dll", "TestJobManager.MockJobRunner", new SerializableDictionary <string, string>(), new SerializableDictionary <string, string>());
                packages.Add(package);

                Job j1 = new Job(null, "MockVMConfig1", null, packages, new SerializableDictionary <string, string>());
                MessageSendRecieve msr = new MessageSendRecieve(new DirectoryInfo(inboxPath), new DirectoryInfo(outboxPath));
                DateTime           queuedJobsDateTime = DateTime.Now;
                string             job1msgID          = msr.QueueJob(j1);

                //wait for job 1 to start
                Assert.True(WaitForVMAction(vmHash["MockVMName1"], VMActionType.Start, queuedJobsDateTime, TimeSpan.FromSeconds(5)));

                //send request for job 1
                AutomationMessage m = new AutomationMessage(new SimpleRequest(SimpleRequests.JobRequest));
                m.From = "MockVMName1";
                Job j = msr.WaitForJob(msr.Send(m), DEFAULT_WAIT);
                Assert.That(j, Is.Not.Null);
                Assert.That(j.JobID, Is.EqualTo(j1.JobID));

                //send finished for job 1
                DateTime  finishedSentDateTime = DateTime.Now;
                JobResult jr = new JobResult();
                jr.Completed = true;
                ExecutionResult er = new ExecutionResult();
                er.Success = true;
                jr.ExecutionResults.Add(er);
                msr.ReportJobStatus(new JobCompleted(j1, jr));

                //wait for job completion
                JobCompleted jobCompleted = msr.WaitForJobCompletion(job1msgID, DEFAULT_WAIT);

                sut.EmulateServiceStop();

                VerifyAppLogDoesNotContain(EventLogEntryType.Warning, testStart);
                VerifyAppLogDoesNotContain(EventLogEntryType.Error, testStart);

                Assert.That(jobCompleted, Is.Not.Null);
                Assert.That(jobCompleted.Job, Is.Not.Null);
                Assert.That(jobCompleted.Result, Is.Not.Null);
                Assert.That(jobCompleted.Result.Errors, Is.Empty);
                Assert.That(jobCompleted.Result.Success, Is.True);
                Assert.That(jobCompleted.Result.Completed, Is.True);
            }
            finally
            {
                tempDir.Delete(true);
            }
        }
Esempio n. 3
0
        public void Invoke_AutomationMessage_ToString_With_Get_Status_11()
        {
            var message = new AutomationMessage();

            message.Message = AUTOMATION_11_GET_STATUS;
            message.What    = AutomationWhat.GetStatus;
            message.Where   = Where.GetWhere("11");

            Assert.AreEqual(message.ToString(), AUTOMATION_11_GET_STATUS);
        }
Esempio n. 4
0
        public void Invoke_AutomationMessage_ToString_With_Automation_11_Stop()
        {
            var message = new AutomationMessage();

            message.Message = AUTOMATION_11_STOP;
            message.What    = AutomationWhat.AutomationStop;
            message.Where   = Where.GetWhere("11");

            Assert.AreEqual(message.ToString(), AUTOMATION_11_STOP);
        }
        public void RunDependentJob()
        {
            sut.EmulateServiceStart(null);
            DirectoryInfo tempDir = new DirectoryInfo(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tempdll"));

            tempDir.Create();
            try
            {
                List <ExecutablePackage> packages = new List <ExecutablePackage>();
                ExecutablePackage        package  = new ExecutablePackage("mock.xml", tempDir.FullName, "TestJobManager.dll", "TestJobManager.MockJobRunner", new SerializableDictionary <string, string>(), new SerializableDictionary <string, string>());
                packages.Add(package);
                Job j1 = new Job("nobuildpath", "MockVMConfig1", new SerializableDictionary <string, string>(), packages, new SerializableDictionary <string, string>());
                Job j2 = new Job("nobuildpath", "MockVMConfig2", new SerializableDictionary <string, string>(), packages, new SerializableDictionary <string, string>());
                j2.DependsOnJobIds.Add(j1.JobID);
                MessageSendRecieve msr = new MessageSendRecieve(new DirectoryInfo(inboxPath), new DirectoryInfo(outboxPath));
                DateTime           queuedJobsDateTime = DateTime.Now;
                string             job2msgID          = msr.QueueJob(j2);
                string             job1msgID          = msr.QueueJob(j1);
                //wait for job 1 to start
                Assert.True(WaitForVMAction(vmHash["MockVMName1"], VMActionType.Start, queuedJobsDateTime, TimeSpan.FromSeconds(10)));
                //send request for job 1
                AutomationMessage m = new AutomationMessage(new SimpleRequest(SimpleRequests.JobRequest));
                m.From = "MockVMName1";
                Job j = msr.WaitForJob(msr.Send(m), DEFAULT_WAIT);
                Assert.That(j, Is.Not.Null);
                //send finished for job 1
                DateTime finishedSentDateTime = DateTime.Now;
                msr.ReportJobStatus(new JobCompleted(j1, new JobResult()));
                //wait for job 2 to start
                Assert.True(WaitForVMAction(vmHash["MockVMName2"], VMActionType.Start, finishedSentDateTime, TimeSpan.FromSeconds(10)));
                //send request for job 2
                m      = new AutomationMessage(new SimpleRequest(SimpleRequests.JobRequest));
                m.From = "MockVMName2";
                j      = msr.WaitForJob(msr.Send(m), DEFAULT_WAIT);
                Assert.That(j, Is.Not.Null);
                //send finished for job2
                msr.ReportJobStatus(new JobCompleted(j2, new JobResult()));

                Assert.That(msr.WaitForJobCompletion(job1msgID, DEFAULT_WAIT), Is.Not.Null);
                Assert.That(msr.WaitForJobCompletion(job2msgID, DEFAULT_WAIT), Is.Not.Null);

                sut.EmulateServiceStop();

                VerifyAppLogDoesNotContain(EventLogEntryType.Warning, testStart);
                VerifyAppLogDoesNotContain(EventLogEntryType.Error, testStart);

                VerifyMockVMActionInvoked(vmHash["MockVMName1"], VMActionType.RevertToNamedSnapshot, "Snapshot1");
                VerifyMockVMActionInvoked(vmHash["MockVMName2"], VMActionType.RevertToNamedSnapshot, "Snapshot2");
            }
            finally
            {
                tempDir.Delete(true);
            }
        }
 private void SendToHost(AutomationMessage m)
 {
     //m.To = hostName;
     using (TextWriter tw = new StreamWriter(Path.Combine(AppConfig.Outbox.FullName, m.Id + ".xml")))
     {
         tw.Write(m.ToXML());
     }
     //using (MessageQueue mq = MessageSendRecieve.GetSpecificMessageQueue(@"FormatName:DIRECT=OS:" + hostName + @"\Private$\" + MessageSendRecieve.LocalQueueName))
     //{
     //    mq.Send(m, MessageQueueTransactionType.Single);
     //}
 }
Esempio n. 7
0
 private void buttonResendResult_Click(object sender, EventArgs e)
 {
     if (listViewStatus.SelectedItems.Count > 0)
     {
         ListViewItem item = listViewStatus.SelectedItems[0];
         if (item.Tag != null && item.Tag is FullJob)
         {
             FullJob job = (FullJob)item.Tag;
             AutomationMessage m = new AutomationMessage(job.Job.OriginalHost, job.Job.OriginalMessageID,  new JobCompleted(job.Job, job.Status.Result));
             msr.SendToHost(m);
         }
     }
 }
Esempio n. 8
0
        public void Set_AutomatioMessage_Properties()
        {
            var message = new AutomationMessage();

            message.Message = AUTOMATION_11_UP;
            message.What    = AutomationWhat.AutomationUp;
            message.Where   = Where.GetWhere("11");

            Assert.AreEqual(message.MessageType, MessageType.Command);
            Assert.AreEqual(message.Who, WHO.Automation);
            Assert.AreEqual(message.What, AutomationWhat.AutomationUp);
            Assert.AreEqual(message.Message, AUTOMATION_11_UP);
            Assert.AreEqual(message.Where.A, 1);
            Assert.AreEqual(message.Where.PL, 1);
        }
Esempio n. 9
0
        public static AutomationMessage GetMessage(string data)
        {
            AutomationMessage message = null;

            string[] content;

            try
            {
                content       = data.Split(new char[] { '*' }, StringSplitOptions.RemoveEmptyEntries);
                message       = new AutomationMessage();
                message.What  = GetWhat(content[1]);
                message.Where = Where.GetWhere(content[2]);
            }
            catch (Exception ex)
            {
                return(null);
            }

            return(message);
        }
        private void ReceiveLoop()
        {
            try
            {
                while (true)
                {
                    //check to see if we need to stop
                    if (mainLoopRunning == false)
                    {
                        break;
                    }

                    List <AutomationMessage> newMessages = new List <AutomationMessage>();
                    lock (IncomingQueueLock)
                    {
                        while (incoming.Count > 0)
                        {
                            newMessages.Add(incoming.Dequeue());
                        }
                    }

                    bool updateJobs = false;

                    foreach (AutomationMessage m in newMessages)
                    {
                        try
                        {
                            if (m.Content is JobCreate)
                            {
                                //add job
                                Job       j  = ((JobCreate)m.Content).j;
                                JobStatus js = new JobStatus();

                                if (j.ISOs == null)
                                {
                                    j.ISOs = new SerializableDictionary <string, string>();
                                }

                                if (j.Properties == null)
                                {
                                    j.Properties = new SerializableDictionary <string, string>();
                                }

                                lock (JobsDictLock)
                                {
                                    jobs[j] = js;
                                    //check required fields
                                    if (String.IsNullOrEmpty(j.Configuration))
                                    {
                                        js.ErrorOut("Configuration cannot be null or empty", null, null);
                                    }
                                    else if (vmMap.ContainsKey(j.Configuration))
                                    {
                                        //set the VM path for visibility to jobmanagerconsole
                                        jobs[j].VMPath = vmMap[j.Configuration].Identifier;
                                    }
                                }

                                updateJobs = true;
                            }
                            else if (m.Content is SimpleRequest)
                            {
                                SimpleRequest srm = (SimpleRequest)m.Content;
                                if (srm.Request == SimpleRequests.JobRequest)
                                {
                                    lock (JobsDictLock)
                                    {
                                        bool found = false;
                                        foreach (Job j in jobs.Keys)
                                        {
                                            JobStatus js = jobs[j];
                                            if (js.State == JobStates.VMStarted)
                                            {
                                                VirtualMachine vm = vmMap[j.Configuration];
                                                if (vm.IsSameHost(m.From))
                                                {
                                                    AutomationMessage toSend = new AutomationMessage(vm.HostName, m.Id, new JobReturn(j));
                                                    SendToHost(toSend);
                                                    found    = true;
                                                    js.State = JobStates.AutoStarted;
                                                    break;
                                                }
                                            }
                                        }
                                        if (!found)
                                        {
                                            AutomationMessage toSend = new AutomationMessage(m.From, m.Id, new ErrorMessage(ErrorMessage.ERROR_NO_JOB_FOUND));
                                            SendToHost(toSend);
                                        }
                                    }
                                }
                                else if (srm.Request == SimpleRequests.AllVMRequest)
                                {
                                    try
                                    {
                                        AutomationMessage toSend = new AutomationMessage(m.From, m.Id, new VMRequestReturn(vmHost.AllVMIdentifiers.ToArray(), lockedVMs.ToArray()));
                                        SendToHost(toSend);
                                    }
                                    catch (Exception ex)
                                    {
                                        EventLog.WriteEntry(ex.ToString(), EventLogEntryType.Error);
                                        AutomationMessage toSend = new AutomationMessage(m.From, m.Id, new ErrorMessage(ex.ToString()));
                                        SendToHost(toSend);
                                    }
                                }
                                else if (srm.Request == SimpleRequests.JobReport)
                                {
                                    lock (JobsDictLock)
                                    {
                                        AutomationMessage toSend = new AutomationMessage(m.From, m.Id, new JobReportReturn(jobs));
                                        SendToHost(toSend);
                                    }
                                }
                            }
                            else if (m.Content is JobCompleted)
                            {
                                JobCompleted jcm = (JobCompleted)m.Content;
                                lock (JobsDictLock)
                                {
                                    foreach (Job j in jobs.Keys)
                                    {
                                        if (j.JobID == jcm.Job.JobID)
                                        {
                                            JobStatus js = jobs[j];
                                            js.Result  = jcm.Result;
                                            js.State   = JobStates.AutoFinished;
                                            updateJobs = true;
                                        }
                                    }
                                }
                            }
                            else if (m.Content is JobCancelCommand)
                            {
                                JobCancelCommand jcm = (JobCancelCommand)m.Content;
                                lock (JobsDictLock)
                                {
                                    foreach (Job j in jobs.Keys)
                                    {
                                        if (j.JobID == jcm.JobID)
                                        {
                                            JobStatus js = jobs[j];
                                            js.ErrorOut("Job was canceled by user", null, null);
                                            updateJobs = true;
                                            break;
                                        }
                                    }
                                }
                            }
                            else if (m.Content is JobDeleteCommand)
                            {
                                JobDeleteCommand jdm = (JobDeleteCommand)m.Content;
                                lock (JobsDictLock)
                                {
                                    Job toDelete = null;
                                    foreach (Job j in jobs.Keys)
                                    {
                                        if (j.JobID == jdm.JobID)
                                        {
                                            toDelete = j;
                                            break;
                                        }
                                    }
                                    if (toDelete != null)
                                    {
                                        jobs.Remove(toDelete);
                                    }
                                }
                            }
                            else if (m.Content is LockVMCommand)
                            {
                                LockVMCommand cmd = (LockVMCommand)m.Content;
                                if (!lockedVMs.Contains(cmd.VMPath))
                                {
                                    lockedVMs.Add(cmd.VMPath);
                                }
                            }
                            else if (m.Content is UnLockVMCommand)
                            {
                                UnLockVMCommand cmd = (UnLockVMCommand)m.Content;
                                if (lockedVMs.Contains(cmd.VMPath))
                                {
                                    lockedVMs.Remove(cmd.VMPath);
                                }
                            }
                            else
                            {
                                EventLog.WriteEntry("Unknown message type " + m.Content.GetType().ToString(), EventLogEntryType.Error);
                                string badFilePath = Path.Combine(AppConfig.Inbox.FullName, m.Id + ".xml.bad");
                                File.WriteAllText(badFilePath, m.ToXML(true));
                            }
                        }
                        catch (Exception ex)
                        {
                            string badFilePath = Path.Combine(AppConfig.Inbox.FullName, m.Id + ".xml.bad");
                            EventLog.WriteEntry(string.Format("Exception while processing message. Message saved to \"{0}\". Exception: {1}", badFilePath, ex.ToString()), EventLogEntryType.Error);
                            File.WriteAllText(badFilePath, m.ToXML(true));
                        }
                    }
                    if (updateJobs)
                    {
                        CheckJobs();
                    }

                    lock (ExecuteLock)
                    {
                        //check for more items
                        lock (IncomingQueueLock)
                        {
                            if (incoming.Count > 0)
                            {
                                continue;
                            }
                        }
                        //check to see if we should still be running
                        if (mainLoopRunning == false)
                        {
                            break;
                        }
                        Monitor.Wait(ExecuteLock);
                    }
                }
            }
            catch (ThreadAbortException)
            {
                //eat it
            }
            catch (Exception ex)
            {
                EventLog.WriteEntry("Exception in receive loop: " + ex.ToString());
                throw;
            }
        }
        private void JobWorkLoop()
        {
            try
            {
                while (true)
                {
                    //check to see if we should still be running
                    if (jobLoopRunning == false)
                    {
                        break;
                    }
                    jobsToCheck = false;
                    lock (JobsDictLock)
                    {
                        List <Job> jobsToDelete = new List <Job>();
                        foreach (Job j in jobs.Keys)
                        {
                            try
                            {
                                JobStatus js = jobs[j];
                                if (js.State == JobStates.Received)
                                {
                                    if (vmMap.ContainsKey(j.Configuration))
                                    {
                                        VirtualMachine vm = vmMap[j.Configuration];
                                        try
                                        {
                                            //check other jobs to see if any are using this VM, if so don't start this job yet
                                            //also count the number of running VMs so we can check against the max
                                            bool vmInUse = false;
                                            Dictionary <string, int> runningVMsCount = new Dictionary <string, int>();//per host
                                            foreach (Job otherJob in jobs.Keys)
                                            {
                                                JobStatus otherJobStatus = jobs[otherJob];
                                                if (otherJobStatus.IsRunning)
                                                {
                                                    VirtualMachine otherVM         = vmMap[otherJob.Configuration];
                                                    string         computeResource = otherVM.ComputeResourceName;
                                                    if (!runningVMsCount.ContainsKey(computeResource))
                                                    {
                                                        runningVMsCount[computeResource] = 0;
                                                    }
                                                    runningVMsCount[computeResource]++;
                                                    if (vm.Identifier == otherVM.Identifier)
                                                    {
                                                        vmInUse = true;
                                                    }
                                                }
                                            }

                                            //check to see if the vm is locked
                                            foreach (string lockedVMPath in lockedVMs)
                                            {
                                                if (lockedVMPath == vm.Identifier)
                                                {
                                                    vmInUse = true;
                                                    break;
                                                }
                                            }

                                            //if this job relies on another job, make sure that job has finished before starting this one
                                            bool waitingForAnotherJob = false;
                                            if (j.DependsOnJobIds != null && j.DependsOnJobIds.Count > 0)
                                            {
                                                foreach (string jobId in j.DependsOnJobIds)
                                                {
                                                    bool jobFinished = false;
                                                    foreach (Job baseJob in jobs.Keys)
                                                    {
                                                        JobStatus baseJobStatus = jobs[baseJob];
                                                        if (baseJob.JobID == jobId)
                                                        {
                                                            if (baseJobStatus.State == JobStates.WaitingForChildJobs)
                                                            {
                                                                jobFinished = true;
                                                            }
                                                        }
                                                    }
                                                    if (!jobFinished)
                                                    {
                                                        waitingForAnotherJob = true;
                                                    }
                                                }
                                            }
                                            int    vmsRunningOnThisResource = 0;
                                            string thisComputeResource      = vm.ComputeResourceName;
                                            if (runningVMsCount.ContainsKey(thisComputeResource))
                                            {
                                                vmsRunningOnThisResource = runningVMsCount[thisComputeResource];
                                            }
                                            if (!vmInUse && !waitingForAnotherJob && vmsRunningOnThisResource < AppConfig.MaxVMsAtOnce)
                                            {
                                                //copy ISO to drop directory
                                                List <string> keys = new List <string>(j.ISOs.Keys);
                                                foreach (string isoName in keys)
                                                {
                                                    string isoPath = j.ISOs[isoName];
                                                    if (File.Exists(isoPath))
                                                    {
                                                        string dropFile = dropManager.GetDropFilePath(isoPath, isoPath);
                                                        j.ISOs[isoName] = dropFile;
                                                    }
                                                    else
                                                    {
                                                        //TODO error?
                                                    }
                                                }
                                                if (j.Packages == null || j.Packages.Count == 0)
                                                {
                                                    js.ErrorOut("Job does not have any packages defined", null, null);
                                                }
                                                else
                                                {
                                                    //copy Test Files to readable dir
                                                    foreach (ExecutablePackage ep in j.Packages)
                                                    {
                                                        DirectoryInfo sourceDir = new DirectoryInfo(ep.ContentDirectory);
                                                        DirectoryData testFiles = DirectoryData.FromDirectory(sourceDir);
                                                        DirectoryInfo destDir   = new DirectoryInfo(AppConfig.FileDrop.FullName + "\\" + Guid.NewGuid().ToString());
                                                        destDir.Create();
                                                        testFiles.DumpContentsToDir(destDir);

                                                        foreach (string subDirName in ep.SubContentDirectories.Keys)
                                                        {
                                                            DirectoryInfo subDirSource = new DirectoryInfo(ep.SubContentDirectories[subDirName]);
                                                            DirectoryData subDirFiles  = DirectoryData.FromDirectory(subDirSource);
                                                            DirectoryInfo subDirDest   = new DirectoryInfo(Path.Combine(destDir.FullName, subDirName));
                                                            subDirDest.Create();
                                                            subDirFiles.DumpContentsToDir(subDirDest);
                                                        }

                                                        ep.ContentDirectory = destDir.FullName;
                                                    }

                                                    vm.RevertToNamedSnapshot();
                                                    //vm.RevertToCurrentSnapshot();

                                                    vm.Start();

                                                    js.State = JobStates.VMStarted;
                                                }
                                            }
                                        }
                                        catch (Exception ex)
                                        {
                                            List <FileData> attachements        = new List <FileData>();
                                            FileData        exceptionDetails    = new FileData();
                                            string          exceptionDetailsStr = ex.ToString();
                                            if (ex is System.Web.Services.Protocols.SoapException)
                                            {
                                                System.Web.Services.Protocols.SoapException soapEx = (System.Web.Services.Protocols.SoapException)ex;
                                                if (soapEx.Detail != null)
                                                {
                                                    exceptionDetailsStr += Environment.NewLine + soapEx.Detail.OuterXml;
                                                }
                                            }
                                            exceptionDetails.Data = Encoding.ASCII.GetBytes(exceptionDetailsStr);
                                            exceptionDetails.Name = "exception.txt";
                                            attachements.Add(exceptionDetails);
                                            js.ErrorOut("Exception: " + ex.Message, null, attachements);
                                        }
                                    }
                                    else
                                    {
                                        js.ErrorOut("Could not find a VM suitable for this configuration(" + j.Configuration.ToString() + ")", null, null);
                                    }
                                }
                                else if (js.State == JobStates.VMStarted || js.State == JobStates.AutoStarted)
                                {
                                    if (DateTime.Now.Subtract(js.LastStateChange) > JOB_RUN_TIMEOUT)
                                    {
                                        js.ErrorOut("Job timed out. No response from VM after " + JOB_RUN_TIMEOUT.TotalHours + " hours", null, null);
                                    }
                                }
                                else if (js.State == JobStates.AutoFinished)
                                {
                                    //check to see if any jobs rely on this job
                                    bool hasChildJobs = false;
                                    foreach (Job other in jobs.Keys)
                                    {
                                        if (other.DependsOnJobIds != null && other.DependsOnJobIds.Contains(j.JobID))//if the other job relies on this job
                                        {
                                            hasChildJobs = true;
                                            break;
                                        }
                                    }
                                    if (hasChildJobs)
                                    {
                                        js.State    = JobStates.WaitingForChildJobs;
                                        jobsToCheck = true;
                                    }
                                    else if (js.Result.SnapshotOnShutdown)
                                    {
                                        js.State = JobStates.TakingSnapshot;
                                    }
                                    else
                                    {
                                        js.State = JobStates.JobFinishedNotSent;
                                    }
                                }
                                else if (js.State == JobStates.TakingSnapshot)
                                {
                                    VirtualMachine vm = vmMap[j.Configuration];
                                    if (vm.IsStarted)
                                    {
                                        //VM is still shuting down, do nothing, this will get checked again on the next go around.
                                    }
                                    else
                                    {
                                        string snapshotName = vm.SnapshotName;
                                        if (!String.IsNullOrEmpty(js.Result.SnapshotName))
                                        {
                                            snapshotName = js.Result.SnapshotName;
                                        }
                                        string snapshotDesc = String.Empty;
                                        if (js.Result.SnapshotDesc != null)
                                        {
                                            snapshotDesc = js.Result.SnapshotDesc;
                                        }
                                        vm.TakeSnapshot(snapshotName, snapshotDesc);
                                        if (js.Result.CloneOnShutdown)
                                        {
                                            try
                                            {
                                                vm.CreateLinkedClone(snapshotName, vm.VMName + "_" + snapshotName);
                                            }
                                            catch (Exception ex)
                                            {
                                                js.ErrorOut("Exception: " + ex.Message, null, null);
                                            }
                                        }
                                        js.State = JobStates.JobFinishedNotSent;
                                    }
                                }
                                else if (js.State == JobStates.WaitingForChildJobs)
                                {
                                    bool inUse = false;
                                    foreach (Job other in jobs.Keys)
                                    {
                                        if (other.DependsOnJobIds != null && other.DependsOnJobIds.Contains(j.JobID))//if the other job relies on this job
                                        {
                                            JobStatus otherStatus = jobs[other];
                                            if (!otherStatus.IsFinished)
                                            {
                                                inUse = true;
                                                break;
                                            }
                                        }
                                    }
                                    if (!inUse)
                                    {
                                        if (js.Result.SnapshotOnShutdown)
                                        {
                                            js.State = JobStates.TakingSnapshot;
                                        }
                                        else
                                        {
                                            js.State = JobStates.JobFinishedNotSent;
                                        }
                                    }
                                }
                            }
                            catch (Exception ex)
                            {
                                //TODO
                                throw;
                            }
                        }

                        foreach (Job j in jobs.Keys)
                        {
                            JobStatus js = jobs[j];
                            if (js.State == JobStates.JobFinishedNotSent)
                            {
                                AutomationMessage m = new AutomationMessage(j.OriginalHost, j.OriginalMessageID, new JobCompleted(j, js.Result));
                                SendToHost(m);

                                js.State = JobStates.JobFinishedSent;

                                //if the iso has been copied to the temp directory, delete it
                                if (j.ISOs != null)
                                {
                                    foreach (string isoPath in j.ISOs.Values)
                                    {
                                        if (isoPath.Contains(AppConfig.FileDrop.FullName))
                                        {
                                            try
                                            {
                                                //release the iso copy
                                                dropManager.ReleaseDropFile(isoPath);
                                            }
                                            catch (Exception ex)
                                            {
                                                EventLog.WriteEntry("Could not release ISO \"" + isoPath + "\" : " + ex.ToString());
                                            }
                                        }
                                    }
                                }

                                //if the test package directory has been copied to the temp directory, delete it
                                if (j.Packages != null)
                                {
                                    foreach (ExecutablePackage ep in j.Packages)
                                    {
                                        string packageDir = ep.ContentDirectory;
                                        if (packageDir != null)
                                        {
                                            if (packageDir.Contains(AppConfig.FileDrop.FullName))
                                            {
                                                try
                                                {
                                                    //delete the test files
                                                    System.IO.Directory.Delete(packageDir, true);
                                                }
                                                catch (Exception ex)
                                                {
                                                    EventLog.WriteEntry("Could not delete directory \"" + packageDir + "\" : " + ex.ToString());
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                            else if (js.State == JobStates.JobFinishedSent)
                            {
                                if (DateTime.Now.Subtract(js.LastStateChange) > DURATION_TO_KEEP_JOBS)
                                {
                                    jobsToDelete.Add(j);
                                }
                            }
                        }

                        foreach (Job j in jobsToDelete)
                        {
                            jobs.Remove(j);
                        }
                    }

                    lock (ExecuteJobsLock)
                    {
                        if (jobsToCheck)
                        {
                            continue;
                        }

                        //check to see if we should still be running
                        if (jobLoopRunning == false)
                        {
                            break;
                        }

                        Monitor.Wait(ExecuteJobsLock);
                    }
                }
            }
            catch (ThreadAbortException)
            {
                //eat it
            }
            catch (Exception ex)
            {
                EventLog.WriteEntry("Exception in work loop: " + ex.ToString(), EventLogEntryType.Error);
                throw;
            }
        }
        private void ReceiveLoop()
        {
            try
            {
                while (true)
                {
                    //check to see if we need to stop
                    if (running == false)
                    {
                        break;
                    }

                    List <AutomationMessage> newMessages = new List <AutomationMessage>();
                    lock (IncomingQueueLock)
                    {
                        while (incoming.Count > 0)
                        {
                            newMessages.Add(incoming.Dequeue());
                        }
                    }

                    bool updateJobs = false;

                    foreach (AutomationMessage m in newMessages)
                    {
                        //if (m.Content is JobCreate)
                        //{
                        //    //add job
                        //    Job j = ((JobCreate)m.Content).j;
                        //    lock (JobsDictLock)
                        //    {
                        //        jobs[j] = new JobStatus();
                        //        //set the VM path for visibility to jobmanagerconsole
                        //        if(vmMap.ContainsKey(j.Configuration))
                        //        {
                        //            jobs[j].VMPath = vmMap[j.Configuration].Path;
                        //        }
                        //    }
                        //    updateJobs = true;
                        //}
                        //else
                        if (m.Content is SimpleRequest)
                        {
                            SimpleRequest srm = (SimpleRequest)m.Content;
                            if (srm.Request == SimpleRequests.JobRequest)
                            {
                                lock (JobsDictLock)
                                {
                                    bool found = false;
                                    foreach (Job j in jobs.Keys)
                                    {
                                        JobStatus js = jobs[j];
                                        if (js.State == JobStates.VMStarted)
                                        {
                                            AutomationMessage toSend = new AutomationMessage(m.From, m.Id, new JobReturn(j));
                                            SendToHost(toSend);
                                            found    = true;
                                            js.State = JobStates.AutoStarted;
                                            break;
                                        }
                                    }
                                    if (!found)
                                    {
                                        AutomationMessage toSend = new AutomationMessage(m.From, m.Id, new ErrorMessage(ErrorMessage.ERROR_NO_JOB_FOUND));
                                        SendToHost(toSend);
                                    }
                                }
                            }
                            //else if (srm.Request == SimpleRequests.AllVMRequest)
                            //{
                            //    try
                            //    {
                            //        AutomationMessage toSend = new AutomationMessage(m.From, m.Id, new VMRequestReturn(VimHelper.AllVMPaths.ToArray(), lockedVMs.ToArray()));
                            //        SendToHost(toSend);
                            //    }
                            //    catch (Exception ex)
                            //    {
                            //        EventLog.WriteEntry(ex.ToString(), EventLogEntryType.Error);
                            //        AutomationMessage toSend = new AutomationMessage(m.From, m.Id, new ErrorMessage(ex.ToString()));
                            //        SendToHost(toSend);
                            //    }
                            //}
                            //else if (srm.Request == SimpleRequests.JobReport)
                            //{
                            //    lock (JobsDictLock)
                            //    {
                            //        AutomationMessage toSend = new AutomationMessage(m.From, m.Id, new JobReportReturn(jobs));
                            //        SendToHost(toSend);
                            //    }
                            //}
                        }
                        else if (m.Content is JobCompleted)
                        {
                            JobCompleted jcm = (JobCompleted)m.Content;
                            lock (JobsDictLock)
                            {
                                foreach (Job j in jobs.Keys)
                                {
                                    if (j.JobID == jcm.Job.JobID)
                                    {
                                        JobStatus js = jobs[j];
                                        js.Result  = jcm.Result;
                                        js.State   = JobStates.AutoFinished;
                                        updateJobs = true;
                                    }
                                }
                            }
                        }
                        //else if (m.Content is JobCancelCommand)
                        //{
                        //    JobCancelCommand jcm = (JobCancelCommand)m.Content;
                        //    lock (JobsDictLock)
                        //    {
                        //        foreach (Job j in jobs.Keys)
                        //        {
                        //            if (j.JobID == jcm.JobID)
                        //            {
                        //                JobStatus js = jobs[j];
                        //                js.ErrorOut("Job was canceled by user", null, null);
                        //                updateJobs = true;
                        //                break;
                        //            }
                        //        }
                        //    }
                        //}
                        //else if (m.Content is JobDeleteCommand)
                        //{
                        //    JobDeleteCommand jdm = (JobDeleteCommand)m.Content;
                        //    lock (JobsDictLock)
                        //    {
                        //        Job toDelete = null;
                        //        foreach (Job j in jobs.Keys)
                        //        {
                        //            if (j.JobID == jdm.JobID)
                        //            {
                        //                toDelete = j;
                        //                break;
                        //            }
                        //        }
                        //        if (toDelete != null)
                        //        {
                        //            jobs.Remove(toDelete);
                        //        }
                        //    }
                        //}
                        //else if (m.Content is LockVMCommand)
                        //{
                        //    LockVMCommand cmd = (LockVMCommand)m.Content;
                        //    if (!lockedVMs.Contains(cmd.VMPath))
                        //    {
                        //        lockedVMs.Add(cmd.VMPath);
                        //    }
                        //}
                        //else if (m.Content is UnLockVMCommand)
                        //{
                        //    UnLockVMCommand cmd = (UnLockVMCommand)m.Content;
                        //    if (lockedVMs.Contains(cmd.VMPath))
                        //    {
                        //        lockedVMs.Remove(cmd.VMPath);
                        //    }
                        //}
                    }
                    if (updateJobs)
                    {
                        CheckJobs();
                    }

                    lock (ExecuteLock)
                    {
                        //check for more items
                        lock (IncomingQueueLock)
                        {
                            if (incoming.Count > 0)
                            {
                                continue;
                            }
                        }
                        //check to see if we should still be running
                        if (running == false)
                        {
                            break;
                        }
                        Monitor.Wait(ExecuteLock);
                    }
                }
            }
            catch (ThreadAbortException)
            {
                //eat it
            }
            catch (Exception ex)
            {
                Console.WriteLine("Exception in receive loop: " + ex.ToString());
                throw ex;
            }
        }