Example #1
0
        private void CreateRunningJob(JobInfo jobInfo)
        {
            if (jobInfo == null)
            {
                return;
            }

            Tracer.Info("Starting workers for a new job. PipelineId = {0}, JobId = {1}, JobTypeId = {2}",
                        jobInfo.PipelineId, jobInfo.JobId, jobInfo.JobTypeId);

            RunningJob runningJob = new RunningJob(jobInfo);

            foreach (WorkRequest workRequest in jobInfo.WorkRequests)
            {
                workRequest.BootParameters = jobInfo.BootParameters;
                workRequest.JobParameters  = jobInfo.JobParameters;
                WorkerCard workerCard = WorkersInventory.Instance.FindWorkerForRole(workRequest.RoleType);
                if (null == workerCard)
                {
                    Tracer.Info("This worker manager does not have workers suitable for the role {0}", workRequest.RoleType);
                    continue;
                }
                JobSection jobSection = new JobSection(workRequest, workerCard);
                runningJob.Sections.Add(jobSection);
            }
            RunningJobs.Add(runningJob);

            if (jobInfo.Command == Command.Run)
            {
                EmployWorkersForJob(runningJob);
            }
        }
Example #2
0
        private void EmployWorkersForJob(RunningJob runningJob)
        {
            foreach (JobSection jobSection in runningJob.Sections)
            {
                if (null == jobSection.WorkerCard)
                {
                    continue;                                // We don't have workers for this role
                }
                uint availableWorkers = jobSection.WorkerCard.MaxNumOfInstances - GetWorkingWorkersNumber(jobSection.RoleType);
                if (availableWorkers == 0)
                {
                    continue;                        // All available workers for that role are busy
                }
                // Try to hire ONE worker for the role.
                // We should not hire more than one worker for the role in one iteration to give other WorkerManagers a chance to take some workload.
                RoleSlotToken roleSlotToken = null;

                var         transaction = new CommittableTransaction(TransactionManager.MaximumTimeout);
                Transaction oldAmbient  = Transaction.Current;
                Transaction.Current = transaction;
                try
                {
                    roleSlotToken = jobSection.GetFreeSlot();
                }
                finally
                {
                    Transaction.Current = oldAmbient;
                    if (null == roleSlotToken)
                    {
                        transaction.Rollback();
                        transaction.Dispose();
                    }
                }

                if (null == roleSlotToken)
                {
                    continue;
                }
                WorkAssignment workAssignment = new WorkAssignment(jobSection.WorkRequest, roleSlotToken,
                                                                   runningJob.JobId, runningJob.ScheduleRunDuration, runningJob.ScheduleCreatedBy, runningJob.NotificationId, runningJob.Frequency);
                EmployWorker(jobSection.WorkerCard, workAssignment, transaction);
                jobSection.WorkerIDs.Add(workAssignment.WorkerId);
            }
        }
Example #3
0
 private void ProcessDirectorsOpenJobsList(OpenJobs openJobs)
 {
     foreach (OpenJob openJob in openJobs.Jobs)
     {
         try
         {
             RunningJob runningJob = FindRunningJob(openJob);
             if (runningJob == null)
             {
                 // We don't have this job
                 if (openJob.Command == Command.Run)
                 {
                     // And it is in the running state
                     JobInfo jobInfo = directorCoreServicesClient.GetJobInfo(openJob.PipelineId, Environment.MachineName);
                     if (jobInfo == null)
                     {
                         // This should not happen, but still was observed at least once in P3, so let's add safety blanket here.
                         Tracer.Warning("Received null JobInfo when requested details of the JobRunId {0} from Director.", openJob.PipelineId);
                         continue;
                     }
                     CreateRunningJob(jobInfo);
                 }
             }
             else
             {
                 UpdateRunningJob(runningJob, openJob);
             }
         }
         catch (Exception ex)
         {
             MessageQueueException messageQueueException = ex as MessageQueueException;
             if (messageQueueException != null && (uint)messageQueueException.ErrorCode == 0x80004005)
             {
                 Tracer.Debug("Cannot find pipeline with ID = {0}, so skip processing it.", openJob.PipelineId);
             }
             else
             {
                 ex.Trace().Swallow();
                 Tracer.Warning("Previous exception in the job handling causes WorkerManager to Cancel job with PipelineId = {0}", openJob.PipelineId);
             }
             openJob.Command = Command.Cancel;
         }
     }
 }
Example #4
0
 // Returns false when runningJob is ready to be removed from the list of running jobs and disposed
 private bool StopWorkersForJob(RunningJob runningJob)
 {
     Tracer.Info("Stopping workers for a job. PipelineId = {0}", runningJob.PipelineId);
     runningJob.State = RunningJob.States.ReadyForDispose; // Just an unconfirmed attempt so far
     if (runningJob.Sections != null)
     {
         foreach (JobSection jobSection in runningJob.Sections)
         {
             foreach (string workerID in jobSection.WorkerIDs)
             {
                 WorkerRunner workerRunner = FindWorkerRunner(workerID);
                 if (null == workerRunner)
                 {
                     continue;
                 }
                 if (workerRunner.IsPresent)
                 {
                     //Tracer.Trace("StopWorkersForJob: Worker {0} is still active. Commanding it to quit.", workerRunner.WorkAssignment.WorkerId);
                     workerRunner.Command = Command.Cancel;
                     runningJob.State     = RunningJob.States.WaitingForWorkersToQuit;
                 }
                 else
                 {
                     Tracer.Info("StopWorkersForJob: Worker {0} quit. Removing worker runner object.", workerRunner.WorkAssignment.WorkerId);
                     AllWorkerRunners.Remove(workerRunner.WorkAssignment.WorkerId);
                     workerRunner.CommittableTransaction.Commit();
                     workerRunner.CommittableTransaction.Dispose();
                 }
             }
         }
     }
     //Tracer.Trace("runningJob.PipelineId = {0}, runningJob.State = {1}", runningJob.PipelineId, runningJob.State.ToString());
     if (runningJob.State == RunningJob.States.ReadyForDispose)
     {
         // TODO: Properly dispose JobSections here
         Tracer.Info("Disposed its resources pertained to job with PipelineId = {0}", runningJob.PipelineId);
         return(false);
     }
     return(true);
 }
Example #5
0
 private void UpdateRunningJob(RunningJob runningJob, OpenJob openJob)
 {
     if (runningJob != null)
     {
         if (runningJob.Sections != null)
         {
             foreach (JobSection jobSection in runningJob.Sections)
             {
                 foreach (string workerID in jobSection.WorkerIDs)
                 {
                     WorkerRunner workerRunner = FindWorkerRunner(workerID);
                     if (null != workerRunner)
                     {
                         workerRunner.Command = openJob.Command;
                     }
                 }
             }
         }
         if (openJob.Command == Command.Run)
         {
             EmployWorkersForJob(runningJob);
         }
     }
 }
 // Returns false when runningJob is ready to be removed from the list of running jobs and disposed 
 private bool StopWorkersForJob(RunningJob runningJob)
 {
     Tracer.Info("Stopping workers for a job. PipelineId = {0}", runningJob.PipelineId);
     runningJob.State = RunningJob.States.ReadyForDispose; // Just an unconfirmed attempt so far
     if (runningJob.Sections != null)
     {
         foreach (JobSection jobSection in runningJob.Sections)
         {
             foreach (string workerID in jobSection.WorkerIDs)
             {
                 WorkerRunner workerRunner = FindWorkerRunner(workerID);
                 if (null == workerRunner) continue;
                 if (workerRunner.IsPresent)
                 {
                     //Tracer.Trace("StopWorkersForJob: Worker {0} is still active. Commanding it to quit.", workerRunner.WorkAssignment.WorkerId);
                     workerRunner.Command = Command.Cancel;
                     runningJob.State = RunningJob.States.WaitingForWorkersToQuit;
                 }
                 else
                 {
                     Tracer.Info("StopWorkersForJob: Worker {0} quit. Removing worker runner object.", workerRunner.WorkAssignment.WorkerId);
                     AllWorkerRunners.Remove(workerRunner.WorkAssignment.WorkerId);
                     workerRunner.CommittableTransaction.Commit();
                     workerRunner.CommittableTransaction.Dispose();
                 }
             }
         }
     }
     //Tracer.Trace("runningJob.PipelineId = {0}, runningJob.State = {1}", runningJob.PipelineId, runningJob.State.ToString());
     if (runningJob.State == RunningJob.States.ReadyForDispose)
     {
         // TODO: Properly dispose JobSections here
         Tracer.Info("Disposed its resources pertained to job with PipelineId = {0}", runningJob.PipelineId);
         return false;
     }
     return true;
 }
 private void UpdateRunningJob(RunningJob runningJob, OpenJob openJob)
 {
     if (runningJob != null)
     {
         if (runningJob.Sections != null)
         {
             foreach (JobSection jobSection in runningJob.Sections)
             {
                 foreach (string workerID in jobSection.WorkerIDs)
                 {
                     WorkerRunner workerRunner = FindWorkerRunner(workerID);
                     if (null != workerRunner)
                     {
                         workerRunner.Command = openJob.Command;
                     }
                 }
             }
         }
         if (openJob.Command == Command.Run)
         {
             EmployWorkersForJob(runningJob);
         }
     }
 }
        private void EmployWorkersForJob(RunningJob runningJob)
        {
            foreach (JobSection jobSection in runningJob.Sections)
            {
                if (null == jobSection.WorkerCard) continue; // We don't have workers for this role
                uint availableWorkers = jobSection.WorkerCard.MaxNumOfInstances - GetWorkingWorkersNumber(jobSection.RoleType);
                if (availableWorkers == 0) continue; // All available workers for that role are busy

                // Try to hire ONE worker for the role.
                // We should not hire more than one worker for the role in one iteration to give other WorkerManagers a chance to take some workload.
                RoleSlotToken roleSlotToken = null;

                var transaction = new CommittableTransaction(TransactionManager.MaximumTimeout);
                Transaction oldAmbient = Transaction.Current;
                Transaction.Current = transaction;
                try
                {
                    roleSlotToken = jobSection.GetFreeSlot();
                }
                finally
                {
                    Transaction.Current = oldAmbient;
                    if (null == roleSlotToken)
                    {
                        transaction.Rollback();
                        transaction.Dispose();
                    }
                }

                if (null == roleSlotToken)
                {
                    continue;
                }
                WorkAssignment workAssignment = new WorkAssignment(jobSection.WorkRequest, roleSlotToken,
                    runningJob.JobId, runningJob.ScheduleRunDuration, runningJob.ScheduleCreatedBy, runningJob.NotificationId, runningJob.Frequency);
                EmployWorker(jobSection.WorkerCard, workAssignment, transaction);
                jobSection.WorkerIDs.Add(workAssignment.WorkerId);
            }
        }
        private void CreateRunningJob(JobInfo jobInfo)
        {
            if (jobInfo == null)
            {
                return;
            }

            Tracer.Info("Starting workers for a new job. PipelineId = {0}, JobId = {1}, JobTypeId = {2}", 
                jobInfo.PipelineId, jobInfo.JobId, jobInfo.JobTypeId);

            RunningJob runningJob = new RunningJob(jobInfo);

            foreach (WorkRequest workRequest in jobInfo.WorkRequests)
            {
                workRequest.BootParameters = jobInfo.BootParameters;
                workRequest.JobParameters = jobInfo.JobParameters;
                WorkerCard workerCard = WorkersInventory.Instance.FindWorkerForRole(workRequest.RoleType);
                if (null == workerCard)
                {
                    Tracer.Info("This worker manager does not have workers suitable for the role {0}", workRequest.RoleType);
                    continue;
                }
                JobSection jobSection = new JobSection(workRequest, workerCard);
                runningJob.Sections.Add(jobSection);
            }
            RunningJobs.Add(runningJob);

            if (jobInfo.Command == Command.Run)
            {
                EmployWorkersForJob(runningJob);
            }
        }