private void CreateRunningJob(JobInfo jobInfo) { if (jobInfo == null) { return; } Tracer.Info("Starting workers for a new job. PipelineId = {0}, JobId = {1}, JobTypeId = {2}", jobInfo.PipelineId, jobInfo.JobId, jobInfo.JobTypeId); RunningJob runningJob = new RunningJob(jobInfo); foreach (WorkRequest workRequest in jobInfo.WorkRequests) { workRequest.BootParameters = jobInfo.BootParameters; workRequest.JobParameters = jobInfo.JobParameters; WorkerCard workerCard = WorkersInventory.Instance.FindWorkerForRole(workRequest.RoleType); if (null == workerCard) { Tracer.Info("This worker manager does not have workers suitable for the role {0}", workRequest.RoleType); continue; } JobSection jobSection = new JobSection(workRequest, workerCard); runningJob.Sections.Add(jobSection); } RunningJobs.Add(runningJob); if (jobInfo.Command == Command.Run) { EmployWorkersForJob(runningJob); } }
private void EmployWorkersForJob(RunningJob runningJob) { foreach (JobSection jobSection in runningJob.Sections) { if (null == jobSection.WorkerCard) { continue; // We don't have workers for this role } uint availableWorkers = jobSection.WorkerCard.MaxNumOfInstances - GetWorkingWorkersNumber(jobSection.RoleType); if (availableWorkers == 0) { continue; // All available workers for that role are busy } // Try to hire ONE worker for the role. // We should not hire more than one worker for the role in one iteration to give other WorkerManagers a chance to take some workload. RoleSlotToken roleSlotToken = null; var transaction = new CommittableTransaction(TransactionManager.MaximumTimeout); Transaction oldAmbient = Transaction.Current; Transaction.Current = transaction; try { roleSlotToken = jobSection.GetFreeSlot(); } finally { Transaction.Current = oldAmbient; if (null == roleSlotToken) { transaction.Rollback(); transaction.Dispose(); } } if (null == roleSlotToken) { continue; } WorkAssignment workAssignment = new WorkAssignment(jobSection.WorkRequest, roleSlotToken, runningJob.JobId, runningJob.ScheduleRunDuration, runningJob.ScheduleCreatedBy, runningJob.NotificationId, runningJob.Frequency); EmployWorker(jobSection.WorkerCard, workAssignment, transaction); jobSection.WorkerIDs.Add(workAssignment.WorkerId); } }
private void ProcessDirectorsOpenJobsList(OpenJobs openJobs) { foreach (OpenJob openJob in openJobs.Jobs) { try { RunningJob runningJob = FindRunningJob(openJob); if (runningJob == null) { // We don't have this job if (openJob.Command == Command.Run) { // And it is in the running state JobInfo jobInfo = directorCoreServicesClient.GetJobInfo(openJob.PipelineId, Environment.MachineName); if (jobInfo == null) { // This should not happen, but still was observed at least once in P3, so let's add safety blanket here. Tracer.Warning("Received null JobInfo when requested details of the JobRunId {0} from Director.", openJob.PipelineId); continue; } CreateRunningJob(jobInfo); } } else { UpdateRunningJob(runningJob, openJob); } } catch (Exception ex) { MessageQueueException messageQueueException = ex as MessageQueueException; if (messageQueueException != null && (uint)messageQueueException.ErrorCode == 0x80004005) { Tracer.Debug("Cannot find pipeline with ID = {0}, so skip processing it.", openJob.PipelineId); } else { ex.Trace().Swallow(); Tracer.Warning("Previous exception in the job handling causes WorkerManager to Cancel job with PipelineId = {0}", openJob.PipelineId); } openJob.Command = Command.Cancel; } } }
// Returns false when runningJob is ready to be removed from the list of running jobs and disposed private bool StopWorkersForJob(RunningJob runningJob) { Tracer.Info("Stopping workers for a job. PipelineId = {0}", runningJob.PipelineId); runningJob.State = RunningJob.States.ReadyForDispose; // Just an unconfirmed attempt so far if (runningJob.Sections != null) { foreach (JobSection jobSection in runningJob.Sections) { foreach (string workerID in jobSection.WorkerIDs) { WorkerRunner workerRunner = FindWorkerRunner(workerID); if (null == workerRunner) { continue; } if (workerRunner.IsPresent) { //Tracer.Trace("StopWorkersForJob: Worker {0} is still active. Commanding it to quit.", workerRunner.WorkAssignment.WorkerId); workerRunner.Command = Command.Cancel; runningJob.State = RunningJob.States.WaitingForWorkersToQuit; } else { Tracer.Info("StopWorkersForJob: Worker {0} quit. Removing worker runner object.", workerRunner.WorkAssignment.WorkerId); AllWorkerRunners.Remove(workerRunner.WorkAssignment.WorkerId); workerRunner.CommittableTransaction.Commit(); workerRunner.CommittableTransaction.Dispose(); } } } } //Tracer.Trace("runningJob.PipelineId = {0}, runningJob.State = {1}", runningJob.PipelineId, runningJob.State.ToString()); if (runningJob.State == RunningJob.States.ReadyForDispose) { // TODO: Properly dispose JobSections here Tracer.Info("Disposed its resources pertained to job with PipelineId = {0}", runningJob.PipelineId); return(false); } return(true); }
private void UpdateRunningJob(RunningJob runningJob, OpenJob openJob) { if (runningJob != null) { if (runningJob.Sections != null) { foreach (JobSection jobSection in runningJob.Sections) { foreach (string workerID in jobSection.WorkerIDs) { WorkerRunner workerRunner = FindWorkerRunner(workerID); if (null != workerRunner) { workerRunner.Command = openJob.Command; } } } } if (openJob.Command == Command.Run) { EmployWorkersForJob(runningJob); } } }
// Returns false when runningJob is ready to be removed from the list of running jobs and disposed private bool StopWorkersForJob(RunningJob runningJob) { Tracer.Info("Stopping workers for a job. PipelineId = {0}", runningJob.PipelineId); runningJob.State = RunningJob.States.ReadyForDispose; // Just an unconfirmed attempt so far if (runningJob.Sections != null) { foreach (JobSection jobSection in runningJob.Sections) { foreach (string workerID in jobSection.WorkerIDs) { WorkerRunner workerRunner = FindWorkerRunner(workerID); if (null == workerRunner) continue; if (workerRunner.IsPresent) { //Tracer.Trace("StopWorkersForJob: Worker {0} is still active. Commanding it to quit.", workerRunner.WorkAssignment.WorkerId); workerRunner.Command = Command.Cancel; runningJob.State = RunningJob.States.WaitingForWorkersToQuit; } else { Tracer.Info("StopWorkersForJob: Worker {0} quit. Removing worker runner object.", workerRunner.WorkAssignment.WorkerId); AllWorkerRunners.Remove(workerRunner.WorkAssignment.WorkerId); workerRunner.CommittableTransaction.Commit(); workerRunner.CommittableTransaction.Dispose(); } } } } //Tracer.Trace("runningJob.PipelineId = {0}, runningJob.State = {1}", runningJob.PipelineId, runningJob.State.ToString()); if (runningJob.State == RunningJob.States.ReadyForDispose) { // TODO: Properly dispose JobSections here Tracer.Info("Disposed its resources pertained to job with PipelineId = {0}", runningJob.PipelineId); return false; } return true; }
private void EmployWorkersForJob(RunningJob runningJob) { foreach (JobSection jobSection in runningJob.Sections) { if (null == jobSection.WorkerCard) continue; // We don't have workers for this role uint availableWorkers = jobSection.WorkerCard.MaxNumOfInstances - GetWorkingWorkersNumber(jobSection.RoleType); if (availableWorkers == 0) continue; // All available workers for that role are busy // Try to hire ONE worker for the role. // We should not hire more than one worker for the role in one iteration to give other WorkerManagers a chance to take some workload. RoleSlotToken roleSlotToken = null; var transaction = new CommittableTransaction(TransactionManager.MaximumTimeout); Transaction oldAmbient = Transaction.Current; Transaction.Current = transaction; try { roleSlotToken = jobSection.GetFreeSlot(); } finally { Transaction.Current = oldAmbient; if (null == roleSlotToken) { transaction.Rollback(); transaction.Dispose(); } } if (null == roleSlotToken) { continue; } WorkAssignment workAssignment = new WorkAssignment(jobSection.WorkRequest, roleSlotToken, runningJob.JobId, runningJob.ScheduleRunDuration, runningJob.ScheduleCreatedBy, runningJob.NotificationId, runningJob.Frequency); EmployWorker(jobSection.WorkerCard, workAssignment, transaction); jobSection.WorkerIDs.Add(workAssignment.WorkerId); } }