/// <summary> /// Get open jobs. /// </summary> /// <param name="machineName">Calling machine name.</param> /// <returns></returns> public OpenJobs GetOpenJobs(string machineName) { var caller = machineName; var openJobs = new OpenJobs { Jobs = new List <OpenJob>() }; try { if (string.IsNullOrEmpty(machineName)) { caller = UnknownMachine; } foreach (var openJob in GetJobs(caller).Select(jobInfo => new OpenJob { PipelineId = jobInfo.PipelineId, Command = jobInfo.Command })) { openJobs.Jobs.Add(openJob); } } catch { Tracer.Error("Unable to send open jobs to worker manager running in machine: {0}", caller); } return(openJobs); }
public void ProcessInfoFromDirector(OpenJobs openJobs) { RemoveWorkerRunnersForNotPresentWorkers(); ProcessDirectorsOpenJobsList(openJobs); ProcessWorkerManagerRunningJobsList(openJobs); }
private void ProcessDirectorsOpenJobsList(OpenJobs openJobs) { foreach (OpenJob openJob in openJobs.Jobs) { try { RunningJob runningJob = FindRunningJob(openJob); if (runningJob == null) { // We don't have this job if (openJob.Command == Command.Run) { // And it is in the running state JobInfo jobInfo = directorCoreServicesClient.GetJobInfo(openJob.PipelineId, Environment.MachineName); if (jobInfo == null) { // This should not happen, but still was observed at least once in P3, so let's add safety blanket here. Tracer.Warning("Received null JobInfo when requested details of the JobRunId {0} from Director.", openJob.PipelineId); continue; } CreateRunningJob(jobInfo); } } else { UpdateRunningJob(runningJob, openJob); } } catch (Exception ex) { MessageQueueException messageQueueException = ex as MessageQueueException; if (messageQueueException != null && (uint)messageQueueException.ErrorCode == 0x80004005) { Tracer.Debug("Cannot find pipeline with ID = {0}, so skip processing it.", openJob.PipelineId); } else { ex.Trace().Swallow(); Tracer.Warning("Previous exception in the job handling causes WorkerManager to Cancel job with PipelineId = {0}", openJob.PipelineId); } openJob.Command = Command.Cancel; } } }
/// <summary> /// Get open jobs. /// </summary> /// <param name="machineName">Calling machine name.</param> /// <returns></returns> public OpenJobs GetOpenJobs(string machineName) { var caller = machineName; var openJobs = new OpenJobs { Jobs = new List<OpenJob>() }; try { if (string.IsNullOrEmpty(machineName)) caller = UnknownMachine; foreach (var openJob in GetJobs(caller).Select(jobInfo => new OpenJob { PipelineId = jobInfo.PipelineId, Command = jobInfo.Command })) { openJobs.Jobs.Add(openJob); } } catch { Tracer.Error("Unable to send open jobs to worker manager running in machine: {0}", caller); } return openJobs; }
private void ProcessWorkerManagerRunningJobsList(OpenJobs openJobs) { List <RunningJob> updatedListOfRunningJobs = new List <RunningJob>(); foreach (RunningJob runningJob in RunningJobs) { OpenJob openJob = FindJobInfo(openJobs, runningJob.PipelineId); if (openJob == null || openJob.Command == Command.Cancel) { if (StopWorkersForJob(runningJob)) { updatedListOfRunningJobs.Add(runningJob); } } else { updatedListOfRunningJobs.Add(runningJob); } } RunningJobs = updatedListOfRunningJobs; }
private OpenJob FindJobInfo(OpenJobs openJobs, string pipelineId) { return openJobs.Jobs.FirstOrDefault(openJob => pipelineId == openJob.PipelineId); }
private void ProcessWorkerManagerRunningJobsList(OpenJobs openJobs) { List<RunningJob> updatedListOfRunningJobs = new List<RunningJob>(); foreach (RunningJob runningJob in RunningJobs) { OpenJob openJob = FindJobInfo(openJobs, runningJob.PipelineId); if (openJob == null || openJob.Command == Command.Cancel) { if (StopWorkersForJob(runningJob)) { updatedListOfRunningJobs.Add(runningJob); } } else { updatedListOfRunningJobs.Add(runningJob); } } RunningJobs = updatedListOfRunningJobs; }
public void LogicalEntryPoint() { //Tracer.Trace("WorkerManager BeforeInit"); Init(); //Tracer.Trace("WorkerManager AfterInit"); var stopWatch = new Stopwatch(); while (true) { stopWatch.Restart(); bool directorServiceFailure = false; try { if (directorCoreServicesClient == null) { directorCoreServicesClient = new DirectorCoreServicesClient("NetTcpBinding_IDirectorCoreServices"); } //Tracer.Info("Worker Manager polls Director for jobs"); OpenJobs openJobs = directorCoreServicesClient.GetOpenJobs(Environment.MachineName); declaredStateOfEmergency = false; if (null != openJobs) { //Tracer.Trace("openJobs bin size is {0}", Utils.BinSizeOf(openJobs)); //Tracer.Trace("WorkerManager received {0} open jobs from Director", openJobs.Jobs.Count); ProcessInfoFromDirector(openJobs); } } catch (TimeoutException) { directorServiceFailure = true; } catch (CommunicationException) // EndpointNotFoundException also comes here { directorServiceFailure = true; } catch (Exception ex) { Tracer.Error("WorkerManager: Unhandled exception: " + ex.ToDebugString()); // If we want the service to stop we need to throw here and let exception be unhandled Tracer.Info("WorkerManager: trying to continue after unhandled exception"); continue; } if (directorServiceFailure) { Debug.Assert(directorCoreServicesClient != null, "_directorCoreServicesClient != null"); directorCoreServicesClient.Abort(); ((IDisposable)directorCoreServicesClient).Dispose(); directorCoreServicesClient = null; //EmergencyFreeze(); EmergencyEvacuation(); } stopWatch.Stop(); TimeSpan getSomeRest = TimeSpan.Zero; if (stopWatch.Elapsed < directorPollingInterval) { getSomeRest = directorPollingInterval - stopWatch.Elapsed; } if (ServiceStopCommand == null) { Thread.Sleep(getSomeRest); } else { if (ServiceStopCommand.WaitOne(getSomeRest)) { break; } } } }
private OpenJob FindJobInfo(OpenJobs openJobs, string pipelineId) { return(openJobs.Jobs.FirstOrDefault(openJob => pipelineId == openJob.PipelineId)); }