private void JobWorkLoop()
{
try
{
while (true)
{
//check to see if we should still be running
if (jobLoopRunning == false)
{
break;
}
jobsToCheck = false;
lock (JobsDictLock)
{
List <Job> jobsToDelete = new List <Job>();
foreach (Job j in jobs.Keys)
{
try
{
JobStatus js = jobs[j];
if (js.State == JobStates.Received)
{
if (vmMap.ContainsKey(j.Configuration))
{
VirtualMachine vm = vmMap[j.Configuration];
try
{
//check other jobs to see if any are using this VM, if so don't start this job yet
//also count the number of running VMs so we can check against the max
bool vmInUse = false;
Dictionary <string, int> runningVMsCount = new Dictionary <string, int>();//per host
foreach (Job otherJob in jobs.Keys)
{
JobStatus otherJobStatus = jobs[otherJob];
if (otherJobStatus.IsRunning)
{
VirtualMachine otherVM = vmMap[otherJob.Configuration];
string computeResource = otherVM.ComputeResourceName;
if (!runningVMsCount.ContainsKey(computeResource))
{
runningVMsCount[computeResource] = 0;
}
runningVMsCount[computeResource]++;
if (vm.Identifier == otherVM.Identifier)
{
vmInUse = true;
}
}
}
//check to see if the vm is locked
foreach (string lockedVMPath in lockedVMs)
{
if (lockedVMPath == vm.Identifier)
{
vmInUse = true;
break;
}
}
//if this job relies on another job, make sure that job has finished before starting this one
bool waitingForAnotherJob = false;
if (j.DependsOnJobIds != null && j.DependsOnJobIds.Count > 0)
{
foreach (string jobId in j.DependsOnJobIds)
{
bool jobFinished = false;
foreach (Job baseJob in jobs.Keys)
{
JobStatus baseJobStatus = jobs[baseJob];
if (baseJob.JobID == jobId)
{
if (baseJobStatus.State == JobStates.WaitingForChildJobs)
{
jobFinished = true;
}
}
}
if (!jobFinished)
{
waitingForAnotherJob = true;
}
}
}
int vmsRunningOnThisResource = 0;
string thisComputeResource = vm.ComputeResourceName;
if (runningVMsCount.ContainsKey(thisComputeResource))
{
vmsRunningOnThisResource = runningVMsCount[thisComputeResource];
}
if (!vmInUse && !waitingForAnotherJob && vmsRunningOnThisResource < AppConfig.MaxVMsAtOnce)
{
//copy ISO to drop directory
List <string> keys = new List <string>(j.ISOs.Keys);
foreach (string isoName in keys)
{
string isoPath = j.ISOs[isoName];
if (File.Exists(isoPath))
{
string dropFile = dropManager.GetDropFilePath(isoPath, isoPath);
j.ISOs[isoName] = dropFile;
}
else
{
//TODO error?
}
}
if (j.Packages == null || j.Packages.Count == 0)
{
js.ErrorOut("Job does not have any packages defined", null, null);
}
else
{
//copy Test Files to readable dir
foreach (ExecutablePackage ep in j.Packages)
{
DirectoryInfo sourceDir = new DirectoryInfo(ep.ContentDirectory);
DirectoryData testFiles = DirectoryData.FromDirectory(sourceDir);
DirectoryInfo destDir = new DirectoryInfo(AppConfig.FileDrop.FullName + "\\" + Guid.NewGuid().ToString());
destDir.Create();
testFiles.DumpContentsToDir(destDir);
foreach (string subDirName in ep.SubContentDirectories.Keys)
{
DirectoryInfo subDirSource = new DirectoryInfo(ep.SubContentDirectories[subDirName]);
DirectoryData subDirFiles = DirectoryData.FromDirectory(subDirSource);
DirectoryInfo subDirDest = new DirectoryInfo(Path.Combine(destDir.FullName, subDirName));
subDirDest.Create();
subDirFiles.DumpContentsToDir(subDirDest);
}
ep.ContentDirectory = destDir.FullName;
}
vm.RevertToNamedSnapshot();
//vm.RevertToCurrentSnapshot();
vm.Start();
js.State = JobStates.VMStarted;
}
}
}
catch (Exception ex)
{
List <FileData> attachements = new List <FileData>();
FileData exceptionDetails = new FileData();
string exceptionDetailsStr = ex.ToString();
if (ex is System.Web.Services.Protocols.SoapException)
{
System.Web.Services.Protocols.SoapException soapEx = (System.Web.Services.Protocols.SoapException)ex;
if (soapEx.Detail != null)
{
exceptionDetailsStr += Environment.NewLine + soapEx.Detail.OuterXml;
}
}
exceptionDetails.Data = Encoding.ASCII.GetBytes(exceptionDetailsStr);
exceptionDetails.Name = "exception.txt";
attachements.Add(exceptionDetails);
js.ErrorOut("Exception: " + ex.Message, null, attachements);
}
}
else
{
js.ErrorOut("Could not find a VM suitable for this configuration(" + j.Configuration.ToString() + ")", null, null);
}
}
else if (js.State == JobStates.VMStarted || js.State == JobStates.AutoStarted)
{
if (DateTime.Now.Subtract(js.LastStateChange) > JOB_RUN_TIMEOUT)
{
js.ErrorOut("Job timed out. No response from VM after " + JOB_RUN_TIMEOUT.TotalHours + " hours", null, null);
}
}
else if (js.State == JobStates.AutoFinished)
{
//check to see if any jobs rely on this job
bool hasChildJobs = false;
foreach (Job other in jobs.Keys)
{
if (other.DependsOnJobIds != null && other.DependsOnJobIds.Contains(j.JobID))//if the other job relies on this job
{
hasChildJobs = true;
break;
}
}
if (hasChildJobs)
{
js.State = JobStates.WaitingForChildJobs;
jobsToCheck = true;
}
else if (js.Result.SnapshotOnShutdown)
{
js.State = JobStates.TakingSnapshot;
}
else
{
js.State = JobStates.JobFinishedNotSent;
}
}
else if (js.State == JobStates.TakingSnapshot)
{
VirtualMachine vm = vmMap[j.Configuration];
if (vm.IsStarted)
{
//VM is still shuting down, do nothing, this will get checked again on the next go around.
}
else
{
string snapshotName = vm.SnapshotName;
if (!String.IsNullOrEmpty(js.Result.SnapshotName))
{
snapshotName = js.Result.SnapshotName;
}
string snapshotDesc = String.Empty;
if (js.Result.SnapshotDesc != null)
{
snapshotDesc = js.Result.SnapshotDesc;
}
vm.TakeSnapshot(snapshotName, snapshotDesc);
if (js.Result.CloneOnShutdown)
{
try
{
vm.CreateLinkedClone(snapshotName, vm.VMName + "_" + snapshotName);
}
catch (Exception ex)
{
js.ErrorOut("Exception: " + ex.Message, null, null);
}
}
js.State = JobStates.JobFinishedNotSent;
}
}
else if (js.State == JobStates.WaitingForChildJobs)
{
bool inUse = false;
foreach (Job other in jobs.Keys)
{
if (other.DependsOnJobIds != null && other.DependsOnJobIds.Contains(j.JobID))//if the other job relies on this job
{
JobStatus otherStatus = jobs[other];
if (!otherStatus.IsFinished)
{
inUse = true;
break;
}
}
}
if (!inUse)
{
if (js.Result.SnapshotOnShutdown)
{
js.State = JobStates.TakingSnapshot;
}
else
{
js.State = JobStates.JobFinishedNotSent;
}
}
}
}
catch (Exception ex)
{
//TODO
throw;
}
}
foreach (Job j in jobs.Keys)
{
JobStatus js = jobs[j];
if (js.State == JobStates.JobFinishedNotSent)
{
AutomationMessage m = new AutomationMessage(j.OriginalHost, j.OriginalMessageID, new JobCompleted(j, js.Result));
SendToHost(m);
js.State = JobStates.JobFinishedSent;
//if the iso has been copied to the temp directory, delete it
if (j.ISOs != null)
{
foreach (string isoPath in j.ISOs.Values)
{
if (isoPath.Contains(AppConfig.FileDrop.FullName))
{
try
{
//release the iso copy
dropManager.ReleaseDropFile(isoPath);
}
catch (Exception ex)
{
EventLog.WriteEntry("Could not release ISO \"" + isoPath + "\" : " + ex.ToString());
}
}
}
}
//if the test package directory has been copied to the temp directory, delete it
if (j.Packages != null)
{
foreach (ExecutablePackage ep in j.Packages)
{
string packageDir = ep.ContentDirectory;
if (packageDir != null)
{
if (packageDir.Contains(AppConfig.FileDrop.FullName))
{
try
{
//delete the test files
System.IO.Directory.Delete(packageDir, true);
}
catch (Exception ex)
{
EventLog.WriteEntry("Could not delete directory \"" + packageDir + "\" : " + ex.ToString());
}
}
}
}
}
}
else if (js.State == JobStates.JobFinishedSent)
{
if (DateTime.Now.Subtract(js.LastStateChange) > DURATION_TO_KEEP_JOBS)
{
jobsToDelete.Add(j);
}
}
}
foreach (Job j in jobsToDelete)
{
jobs.Remove(j);
}
}
lock (ExecuteJobsLock)
{
if (jobsToCheck)
{
continue;
}
//check to see if we should still be running
if (jobLoopRunning == false)
{
break;
}
Monitor.Wait(ExecuteJobsLock);
}
}
}
catch (ThreadAbortException)
{
//eat it
}
catch (Exception ex)
{
EventLog.WriteEntry("Exception in work loop: " + ex.ToString(), EventLogEntryType.Error);
throw;
}
}