/// <summary>
/// Get open jobs.
/// </summary>
/// <param name="machineName">Calling machine name.</param>
/// <returns></returns>
public OpenJobs GetOpenJobs(string machineName)
{
var caller = machineName;
var openJobs = new OpenJobs {
Jobs = new List <OpenJob>()
};
try
{
if (string.IsNullOrEmpty(machineName))
{
caller = UnknownMachine;
}
foreach (var openJob in GetJobs(caller).Select(jobInfo => new OpenJob {
PipelineId = jobInfo.PipelineId, Command = jobInfo.Command
}))
{
openJobs.Jobs.Add(openJob);
}
}
catch
{
Tracer.Error("Unable to send open jobs to worker manager running in machine: {0}", caller);
}
return(openJobs);
}
public void ProcessInfoFromDirector(OpenJobs openJobs)
{
RemoveWorkerRunnersForNotPresentWorkers();
ProcessDirectorsOpenJobsList(openJobs);
ProcessWorkerManagerRunningJobsList(openJobs);
}
private void ProcessDirectorsOpenJobsList(OpenJobs openJobs)
{
foreach (OpenJob openJob in openJobs.Jobs)
{
try
{
RunningJob runningJob = FindRunningJob(openJob);
if (runningJob == null)
{
// We don't have this job
if (openJob.Command == Command.Run)
{
// And it is in the running state
JobInfo jobInfo = directorCoreServicesClient.GetJobInfo(openJob.PipelineId, Environment.MachineName);
if (jobInfo == null)
{
// This should not happen, but still was observed at least once in P3, so let's add safety blanket here.
Tracer.Warning("Received null JobInfo when requested details of the JobRunId {0} from Director.", openJob.PipelineId);
continue;
}
CreateRunningJob(jobInfo);
}
}
else
{
UpdateRunningJob(runningJob, openJob);
}
}
catch (Exception ex)
{
MessageQueueException messageQueueException = ex as MessageQueueException;
if (messageQueueException != null && (uint)messageQueueException.ErrorCode == 0x80004005)
{
Tracer.Debug("Cannot find pipeline with ID = {0}, so skip processing it.", openJob.PipelineId);
}
else
{
ex.Trace().Swallow();
Tracer.Warning("Previous exception in the job handling causes WorkerManager to Cancel job with PipelineId = {0}", openJob.PipelineId);
}
openJob.Command = Command.Cancel;
}
}
}
/// <summary>
/// Get open jobs.
/// </summary>
/// <param name="machineName">Calling machine name.</param>
/// <returns></returns>
public OpenJobs GetOpenJobs(string machineName)
{
var caller = machineName;
var openJobs = new OpenJobs { Jobs = new List<OpenJob>() };
try
{
if (string.IsNullOrEmpty(machineName))
caller = UnknownMachine;
foreach (var openJob in GetJobs(caller).Select(jobInfo => new OpenJob { PipelineId = jobInfo.PipelineId, Command = jobInfo.Command }))
{
openJobs.Jobs.Add(openJob);
}
}
catch
{
Tracer.Error("Unable to send open jobs to worker manager running in machine: {0}", caller);
}
return openJobs;
}
private void ProcessWorkerManagerRunningJobsList(OpenJobs openJobs)
{
List <RunningJob> updatedListOfRunningJobs = new List <RunningJob>();
foreach (RunningJob runningJob in RunningJobs)
{
OpenJob openJob = FindJobInfo(openJobs, runningJob.PipelineId);
if (openJob == null || openJob.Command == Command.Cancel)
{
if (StopWorkersForJob(runningJob))
{
updatedListOfRunningJobs.Add(runningJob);
}
}
else
{
updatedListOfRunningJobs.Add(runningJob);
}
}
RunningJobs = updatedListOfRunningJobs;
}
private OpenJob FindJobInfo(OpenJobs openJobs, string pipelineId)
{
return openJobs.Jobs.FirstOrDefault(openJob => pipelineId == openJob.PipelineId);
}
private void ProcessWorkerManagerRunningJobsList(OpenJobs openJobs)
{
List<RunningJob> updatedListOfRunningJobs = new List<RunningJob>();
foreach (RunningJob runningJob in RunningJobs)
{
OpenJob openJob = FindJobInfo(openJobs, runningJob.PipelineId);
if (openJob == null || openJob.Command == Command.Cancel)
{
if (StopWorkersForJob(runningJob))
{
updatedListOfRunningJobs.Add(runningJob);
}
}
else
{
updatedListOfRunningJobs.Add(runningJob);
}
}
RunningJobs = updatedListOfRunningJobs;
}
private void ProcessDirectorsOpenJobsList(OpenJobs openJobs)
{
foreach (OpenJob openJob in openJobs.Jobs)
{
try
{
RunningJob runningJob = FindRunningJob(openJob);
if (runningJob == null)
{
// We don't have this job
if (openJob.Command == Command.Run)
{
// And it is in the running state
JobInfo jobInfo = directorCoreServicesClient.GetJobInfo(openJob.PipelineId, Environment.MachineName);
if (jobInfo == null)
{
// This should not happen, but still was observed at least once in P3, so let's add safety blanket here.
Tracer.Warning("Received null JobInfo when requested details of the JobRunId {0} from Director.", openJob.PipelineId);
continue;
}
CreateRunningJob(jobInfo);
}
}
else
{
UpdateRunningJob(runningJob, openJob);
}
}
catch (Exception ex)
{
MessageQueueException messageQueueException = ex as MessageQueueException;
if (messageQueueException != null && (uint)messageQueueException.ErrorCode == 0x80004005)
{
Tracer.Debug("Cannot find pipeline with ID = {0}, so skip processing it.", openJob.PipelineId);
}
else
{
ex.Trace().Swallow();
Tracer.Warning("Previous exception in the job handling causes WorkerManager to Cancel job with PipelineId = {0}", openJob.PipelineId);
}
openJob.Command = Command.Cancel;
}
}
}
public void ProcessInfoFromDirector(OpenJobs openJobs)
{
RemoveWorkerRunnersForNotPresentWorkers();
ProcessDirectorsOpenJobsList(openJobs);
ProcessWorkerManagerRunningJobsList(openJobs);
}
public void LogicalEntryPoint()
{
//Tracer.Trace("WorkerManager BeforeInit");
Init();
//Tracer.Trace("WorkerManager AfterInit");
var stopWatch = new Stopwatch();
while (true)
{
stopWatch.Restart();
bool directorServiceFailure = false;
try
{
if (directorCoreServicesClient == null)
{
directorCoreServicesClient = new DirectorCoreServicesClient("NetTcpBinding_IDirectorCoreServices");
}
//Tracer.Info("Worker Manager polls Director for jobs");
OpenJobs openJobs = directorCoreServicesClient.GetOpenJobs(Environment.MachineName);
declaredStateOfEmergency = false;
if (null != openJobs)
{
//Tracer.Trace("openJobs bin size is {0}", Utils.BinSizeOf(openJobs));
//Tracer.Trace("WorkerManager received {0} open jobs from Director", openJobs.Jobs.Count);
ProcessInfoFromDirector(openJobs);
}
}
catch (TimeoutException)
{
directorServiceFailure = true;
}
catch (CommunicationException) // EndpointNotFoundException also comes here
{
directorServiceFailure = true;
}
catch (Exception ex)
{
Tracer.Error("WorkerManager: Unhandled exception: " + ex.ToDebugString());
// If we want the service to stop we need to throw here and let exception be unhandled
Tracer.Info("WorkerManager: trying to continue after unhandled exception");
continue;
}
if (directorServiceFailure)
{
Debug.Assert(directorCoreServicesClient != null, "_directorCoreServicesClient != null");
directorCoreServicesClient.Abort();
((IDisposable)directorCoreServicesClient).Dispose();
directorCoreServicesClient = null;
//EmergencyFreeze();
EmergencyEvacuation();
}
stopWatch.Stop();
TimeSpan getSomeRest = TimeSpan.Zero;
if (stopWatch.Elapsed < directorPollingInterval)
{
getSomeRest = directorPollingInterval - stopWatch.Elapsed;
}
if (ServiceStopCommand == null)
{
Thread.Sleep(getSomeRest);
}
else
{
if (ServiceStopCommand.WaitOne(getSomeRest))
{
break;
}
}
}
}
private OpenJob FindJobInfo(OpenJobs openJobs, string pipelineId)
{
return(openJobs.Jobs.FirstOrDefault(openJob => pipelineId == openJob.PipelineId));
}
