private void JobWorkLoop()
{
try
{
while (true)
{
//check to see if we should still be running
if (jobLoopRunning == false)
{
break;
}
jobsToCheck = false;
lock (JobsDictLock)
{
List <Job> jobsToDelete = new List <Job>();
foreach (Job j in jobs.Keys)
{
try
{
JobStatus js = jobs[j];
if (js.State == JobStates.Received)
{
if (vmMap.ContainsKey(j.Configuration))
{
VirtualMachine vm = vmMap[j.Configuration];
try
{
//check other jobs to see if any are using this VM, if so don't start this job yet
//also count the number of running VMs so we can check against the max
bool vmInUse = false;
Dictionary <string, int> runningVMsCount = new Dictionary <string, int>();//per host
foreach (Job otherJob in jobs.Keys)
{
JobStatus otherJobStatus = jobs[otherJob];
if (otherJobStatus.IsRunning)
{
VirtualMachine otherVM = vmMap[otherJob.Configuration];
string computeResource = otherVM.ComputeResourceName;
if (!runningVMsCount.ContainsKey(computeResource))
{
runningVMsCount[computeResource] = 0;
}
runningVMsCount[computeResource]++;
if (vm.Identifier == otherVM.Identifier)
{
vmInUse = true;
}
}
}
//check to see if the vm is locked
foreach (string lockedVMPath in lockedVMs)
{
if (lockedVMPath == vm.Identifier)
{
vmInUse = true;
break;
}
}
//if this job relies on another job, make sure that job has finished before starting this one
bool waitingForAnotherJob = false;
if (j.DependsOnJobIds != null && j.DependsOnJobIds.Count > 0)
{
foreach (string jobId in j.DependsOnJobIds)
{
bool jobFinished = false;
foreach (Job baseJob in jobs.Keys)
{
JobStatus baseJobStatus = jobs[baseJob];
if (baseJob.JobID == jobId)
{
if (baseJobStatus.State == JobStates.WaitingForChildJobs)
{
jobFinished = true;
}
}
}
if (!jobFinished)
{
waitingForAnotherJob = true;
}
}
}
int vmsRunningOnThisResource = 0;
string thisComputeResource = vm.ComputeResourceName;
if (runningVMsCount.ContainsKey(thisComputeResource))
{
vmsRunningOnThisResource = runningVMsCount[thisComputeResource];
}
if (!vmInUse && !waitingForAnotherJob && vmsRunningOnThisResource < AppConfig.MaxVMsAtOnce)
{
//copy ISO to drop directory
List <string> keys = new List <string>(j.ISOs.Keys);
foreach (string isoName in keys)
{
string isoPath = j.ISOs[isoName];
if (File.Exists(isoPath))
{
string dropFile = dropManager.GetDropFilePath(isoPath, isoPath);
j.ISOs[isoName] = dropFile;
}
else
{
//TODO error?
}
}
if (j.Packages == null || j.Packages.Count == 0)
{
js.ErrorOut("Job does not have any packages defined", null, null);
}
else
{
//copy Test Files to readable dir
foreach (ExecutablePackage ep in j.Packages)
{
DirectoryInfo sourceDir = new DirectoryInfo(ep.ContentDirectory);
DirectoryData testFiles = DirectoryData.FromDirectory(sourceDir);
DirectoryInfo destDir = new DirectoryInfo(AppConfig.FileDrop.FullName + "\\" + Guid.NewGuid().ToString());
destDir.Create();
testFiles.DumpContentsToDir(destDir);
foreach (string subDirName in ep.SubContentDirectories.Keys)
{
DirectoryInfo subDirSource = new DirectoryInfo(ep.SubContentDirectories[subDirName]);
DirectoryData subDirFiles = DirectoryData.FromDirectory(subDirSource);
DirectoryInfo subDirDest = new DirectoryInfo(Path.Combine(destDir.FullName, subDirName));
subDirDest.Create();
subDirFiles.DumpContentsToDir(subDirDest);
}
ep.ContentDirectory = destDir.FullName;
}
vm.RevertToNamedSnapshot();
//vm.RevertToCurrentSnapshot();
vm.Start();
js.State = JobStates.VMStarted;
}
}
}
catch (Exception ex)
{
List <FileData> attachements = new List <FileData>();
FileData exceptionDetails = new FileData();
string exceptionDetailsStr = ex.ToString();
if (ex is System.Web.Services.Protocols.SoapException)
{
System.Web.Services.Protocols.SoapException soapEx = (System.Web.Services.Protocols.SoapException)ex;
if (soapEx.Detail != null)
{
exceptionDetailsStr += Environment.NewLine + soapEx.Detail.OuterXml;
}
}
exceptionDetails.Data = Encoding.ASCII.GetBytes(exceptionDetailsStr);
exceptionDetails.Name = "exception.txt";
attachements.Add(exceptionDetails);
js.ErrorOut("Exception: " + ex.Message, null, attachements);
}
}
else
{
js.ErrorOut("Could not find a VM suitable for this configuration(" + j.Configuration.ToString() + ")", null, null);
}
}
else if (js.State == JobStates.VMStarted || js.State == JobStates.AutoStarted)
{
if (DateTime.Now.Subtract(js.LastStateChange) > JOB_RUN_TIMEOUT)
{
js.ErrorOut("Job timed out. No response from VM after " + JOB_RUN_TIMEOUT.TotalHours + " hours", null, null);
}
}
else if (js.State == JobStates.AutoFinished)
{
//check to see if any jobs rely on this job
bool hasChildJobs = false;
foreach (Job other in jobs.Keys)
{
if (other.DependsOnJobIds != null && other.DependsOnJobIds.Contains(j.JobID))//if the other job relies on this job
{
hasChildJobs = true;
break;
}
}
if (hasChildJobs)
{
js.State = JobStates.WaitingForChildJobs;
jobsToCheck = true;
}
else if (js.Result.SnapshotOnShutdown)
{
js.State = JobStates.TakingSnapshot;
}
else
{
js.State = JobStates.JobFinishedNotSent;
}
}
else if (js.State == JobStates.TakingSnapshot)
{
VirtualMachine vm = vmMap[j.Configuration];
if (vm.IsStarted)
{
//VM is still shuting down, do nothing, this will get checked again on the next go around.
}
else
{
string snapshotName = vm.SnapshotName;
if (!String.IsNullOrEmpty(js.Result.SnapshotName))
{
snapshotName = js.Result.SnapshotName;
}
string snapshotDesc = String.Empty;
if (js.Result.SnapshotDesc != null)
{
snapshotDesc = js.Result.SnapshotDesc;
}
vm.TakeSnapshot(snapshotName, snapshotDesc);
if (js.Result.CloneOnShutdown)
{
try
{
vm.CreateLinkedClone(snapshotName, vm.VMName + "_" + snapshotName);
}
catch (Exception ex)
{
js.ErrorOut("Exception: " + ex.Message, null, null);
}
}
js.State = JobStates.JobFinishedNotSent;
}
}
else if (js.State == JobStates.WaitingForChildJobs)
{
bool inUse = false;
foreach (Job other in jobs.Keys)
{
if (other.DependsOnJobIds != null && other.DependsOnJobIds.Contains(j.JobID))//if the other job relies on this job
{
JobStatus otherStatus = jobs[other];
if (!otherStatus.IsFinished)
{
inUse = true;
break;
}
}
}
if (!inUse)
{
if (js.Result.SnapshotOnShutdown)
{
js.State = JobStates.TakingSnapshot;
}
else
{
js.State = JobStates.JobFinishedNotSent;
}
}
}
}
catch (Exception ex)
{
//TODO
throw;
}
}
foreach (Job j in jobs.Keys)
{
JobStatus js = jobs[j];
if (js.State == JobStates.JobFinishedNotSent)
{
AutomationMessage m = new AutomationMessage(j.OriginalHost, j.OriginalMessageID, new JobCompleted(j, js.Result));
SendToHost(m);
js.State = JobStates.JobFinishedSent;
//if the iso has been copied to the temp directory, delete it
if (j.ISOs != null)
{
foreach (string isoPath in j.ISOs.Values)
{
if (isoPath.Contains(AppConfig.FileDrop.FullName))
{
try
{
//release the iso copy
dropManager.ReleaseDropFile(isoPath);
}
catch (Exception ex)
{
EventLog.WriteEntry("Could not release ISO \"" + isoPath + "\" : " + ex.ToString());
}
}
}
}
//if the test package directory has been copied to the temp directory, delete it
if (j.Packages != null)
{
foreach (ExecutablePackage ep in j.Packages)
{
string packageDir = ep.ContentDirectory;
if (packageDir != null)
{
if (packageDir.Contains(AppConfig.FileDrop.FullName))
{
try
{
//delete the test files
System.IO.Directory.Delete(packageDir, true);
}
catch (Exception ex)
{
EventLog.WriteEntry("Could not delete directory \"" + packageDir + "\" : " + ex.ToString());
}
}
}
}
}
}
else if (js.State == JobStates.JobFinishedSent)
{
if (DateTime.Now.Subtract(js.LastStateChange) > DURATION_TO_KEEP_JOBS)
{
jobsToDelete.Add(j);
}
}
}
foreach (Job j in jobsToDelete)
{
jobs.Remove(j);
}
}
lock (ExecuteJobsLock)
{
if (jobsToCheck)
{
continue;
}
//check to see if we should still be running
if (jobLoopRunning == false)
{
break;
}
Monitor.Wait(ExecuteJobsLock);
}
}
}
catch (ThreadAbortException)
{
//eat it
}
catch (Exception ex)
{
EventLog.WriteEntry("Exception in work loop: " + ex.ToString(), EventLogEntryType.Error);
throw;
}
}
public void Run()
{
try
{
FileInfo mapNetBatchFile = new FileInfo(Path.Combine(Utilities.ExecutingAssembly.Directory.FullName, "map.bat"));
//if the network drive is disconected, then we will be unable to get to the server inbox, in which case we should try to remap
if (!AppConfig.ServerInbox.Exists && mapNetBatchFile.Exists)
{
using (System.Diagnostics.Process p = new System.Diagnostics.Process())
{
p.StartInfo.WorkingDirectory = mapNetBatchFile.Directory.FullName;
p.StartInfo.FileName = mapNetBatchFile.Name;
p.StartInfo.CreateNoWindow = true;
p.StartInfo.UseShellExecute = true;
p.Start();
p.WaitForExit();
}
}
int packageToRun = 0;
JobResult result = new JobResult();
result.Completed = true;
//string sendQueueName = @"FormatName:DIRECT=OS:hammerbuildvm\Private$\jobmanager";
//string sendQueueName = @"FormatName:DIRECT=OS:ryanadams2\Private$\test2";
//jci.LogString("Connecting to Job Manager receive queue (" + sendQueueName + ")");
MessageSendRecieve msr = new MessageSendRecieve(AppConfig.ServerInbox, AppConfig.ServerOutbox);
//jci.LogString("Permission = " + msr.RemoteMessageQueue.AccessMode.ToString());
//look for an existing job to run/continue before getting a new job from the server
FileInfo jobXML = new FileInfo(Path.Combine(Utilities.ExecutingAssembly.Directory.FullName, "job.xml"));
if (jobXML.Exists)
{
using (TextReader tr = new StreamReader(jobXML.FullName))
{
j = XMLSerializable.FromXML <Job>(tr.ReadToEnd());
if (j.Properties.ContainsKey("PackageToRun"))
{
packageToRun = Int32.Parse(j.Properties["PackageToRun"]);
}
}
try
{
//rename the job file so the next run doesn't automatically use it. The job.xml file will be put back
//as part of jci.StartupOnNextRun if it is meant to be continued after a restart
string lastFile = jobXML.FullName + ".old";
if (File.Exists(lastFile))
{
File.Delete(lastFile);
}
File.Move(jobXML.FullName, lastFile);
}
catch (Exception ex)
{
//if the delete fails lets log it, but it isn't critical so let's eat the exception
LogString("Could not delete existing job.xml file: " + ex.ToString());
}
//look for an existing JobResult to pull in
FileInfo jobResultXML = new FileInfo(Path.Combine(Utilities.ExecutingAssembly.Directory.FullName, "jobresult.xml"));
if (jobResultXML.Exists)
{
try
{
using (TextReader tr = new StreamReader(jobResultXML.FullName))
{
result = XMLSerializable.FromXML <JobResult>(tr.ReadToEnd());
}
}
catch (Exception ex)
{
//log, but eat it
LogString(ex.ToString());
}
}
}
else
{
LogString("Requesting Jobs from Job Manager");
string messageID = msr.RequestJob();
LogString("Sent request with message id: " + messageID);
LogString("Waiting for Job response from Job Manager");
j = msr.WaitForJob(messageID, DEFAULT_JOB_WAIT);
if (j == null)
{
LogString("No Jobs Available");
return;
}
try
{
LogString("Found Job: " + j.JobID);
if (baseDir.Exists)
{
baseDir.Delete(true);
//TODO wait for files to be deleted?
}
baseDir.Create();
List <string> keys = new List <string>(j.ISOs.Keys);
foreach (string isoName in keys)
{
FileInfo isoPath = new FileInfo(j.ISOs[isoName]);
string destPath = Path.Combine(Utilities.ExecutingAssembly.Directory.FullName, isoPath.Name);
LogString("Copying ISO from \"" + isoPath.Directory.FullName + "\" to \"" + destPath + "\"");
isoPath.CopyTo(destPath);
j.ISOs[isoName] = destPath;
}
if (j.Properties == null)
{
j.Properties = new SerializableDictionary <string, string>();
}
}
catch (Exception ex)
{
LogString(ex.ToString());
result.Completed = false;
ExecutionResult er = new ExecutionResult(ex.ToString(), null);
result.ExecutionResults.Add(er);
Logger.Instance.Pause();
result.Logs.Add(FileData.FromFile(new FileInfo(Logger.Instance.FileName)));
Logger.Instance.Resume();
LogString("Sending Job Result");
msr.ReportJobStatus(new JobCompleted(j, result));
LogString("Job Result Sent");
return;
}
}
if (j.Packages.Count == 0)
{
Logger.Instance.Pause();
result.Logs.Add(FileData.FromFile(new FileInfo(Logger.Instance.FileName)));
Logger.Instance.Resume();
}
while (packageToRun < j.Packages.Count)
{
runningPackageDir = new DirectoryInfo(Path.Combine(baseDir.FullName, packageToRun.ToString()));
ExecutablePackage ep = j.Packages[packageToRun];
runningPackage = ep;
ExecutionResult er = new ExecutionResult();
try
{
if (!ep.ContentDirectory.ToLower().Equals(runningPackageDir.FullName.ToLower()))
{
if (runningPackageDir.Exists)
{
runningPackageDir.Delete(true);
}
runningPackageDir.Create();
LogString("Copying data from \"" + ep.ContentDirectory + "\" to \"" + runningPackageDir.FullName + "\"");
DirectoryData.FromDirectory(new DirectoryInfo(ep.ContentDirectory)).DumpContentsToDir(runningPackageDir);
ep.ContentDirectory = runningPackageDir.FullName;
}
LogString("Loading external test DLL: " + ep.JobRunnerDLLName + " , " + ep.JobRunnerClassName);
JobRunner jr = LoadJobRunner(ep.JobRunnerClassName, Path.Combine(runningPackageDir.FullName, ep.JobRunnerDLLName));
LogString("Executing Execute() method on external DLL");
er = jr.Execute(this);
}
catch (Exception ex)
{
LogString(ex.ToString());
result.Completed = false;
er = new ExecutionResult(ex.ToString(), null);
}
Logger.Instance.Pause();
result.Logs.Add(FileData.FromFile(new FileInfo(Logger.Instance.FileName)));
Logger.Instance.Resume();
if (er != null)
{
result.ExecutionResults.Add(er);
}
//lets save the current job result
using (TextWriter tw = new StreamWriter(Path.Combine(Utilities.ExecutingAssembly.Directory.FullName, "jobresult.xml"), false))
{
tw.Write(result.ToXML());
}
if (er == null)
{
//The automation is likely not finished, the computer is likely going to reboot and
//we want this execution to continue after reboot so we should exit now instead of going to the next package.
//the executable package should have already called startuponnextrun
return;
}
if (!er.Success)
{
//stop on first error
break;
}
packageToRun++;
j.Properties["PackageToRun"] = packageToRun.ToString();
if (er.Success && er.RestartAfter)
{
StartupOnNextRun();
LogString("Restarting ...");
system.Shutdown(true);
return;
}
}
LogString("Sending Job Result");
msr.ReportJobStatus(new JobCompleted(j, result));
LogString("Job Result Sent");
//cleanup
if (File.Exists(Path.Combine(Utilities.ExecutingAssembly.Directory.FullName, "jobresult.xml")))
{
File.Delete(Path.Combine(Utilities.ExecutingAssembly.Directory.FullName, "jobresult.xml"));
}
if (ShutdownOnCompletion)
{
LogString("Shuting Down ...");
system.Shutdown(false);
//so, lets exit the program
System.Windows.Forms.Application.Exit();
}
}
catch (ThreadAbortException)
{
//eat it, get out right away. Program is exiting or user has stopped automation
return;
}
catch (Exception e)
{
LogString("Exception in thread: " + e.ToString());
return;
}
}
