Ejemplo n.º 1
0
 private IEnumerator<IDependency<IModule>> GetDependencyEnumerator(JobExeWrapper jobWrapper, IDependency<IModule> dep, IEnumerator<IDependency<IModule>> iEnumerator)
 {
     switch (jobWrapper.Job.DependencyTraversalMode)
     {
         case "Breadth-first":
             iEnumerator = dep.GetBreadthFirstEnumerator();
             break;
         case "Depth-first":
             iEnumerator = dep.GetDepthFirstEnumerator();
             break;
     }
     return iEnumerator;
 }
Ejemplo n.º 2
0
        private ErrorCode ProcessJobFromRoot(JobExeWrapper jobWrapper, DateTime jobkey, IDependency<IModule> dep)
        {
            ErrorCode retval = ErrorCode.SUCCESS;

            if (killJobs.Contains(jobWrapper.Job.JobName))
                killJobs.Remove(jobWrapper.Job.JobName);

            // pick enumeration type
            IEnumerator<IDependency<IModule>> iEnumerator =
                dep.GetBreadthFirstEnumerator(); //default mode

            //execution
            log.InfoFormat(jobWrapper.Job.KeyCode, "Beginning executing job {0} in {1} mode.", jobWrapper.Job.JobName, jobWrapper.Mode.ToString());

            ArrayList trackWaitState = new ArrayList();

            iEnumerator = GetDependencyEnumerator(jobWrapper, dep, iEnumerator);
            while (iEnumerator.MoveNext())
            {
                IDependency<IModule> d = iEnumerator.Current;
                if (d.Name.Equals(jobWrapper.Job.JobName))
                    continue;
                if (!jobWrapper.States.ContainsKey(d.Name))
                    jobWrapper.States[d.Name] = State.CurrentStateEnum.PendingInitialization;
            }

            iEnumerator = GetDependencyEnumerator(jobWrapper, dep, iEnumerator);
            while (iEnumerator.MoveNext() && retval.Equals(ErrorCode.ABORT_JOB) == false)
            {
                IDependency<IModule> d = iEnumerator.Current;

                if (killJobs.Contains(jobWrapper.Job.JobName))
                {
                    killJobs.Remove(jobWrapper.Job.JobName);
                    ReenableJob(jobWrapper, trackWaitState, d);

                    log.WarnFormat(jobWrapper.Job.KeyCode, "Killing job {0} ", jobWrapper.Job.JobName);

                    Thread.CurrentThread.Abort();
                    break;
                }

                if (d.Name.Equals(jobWrapper.Job.JobName))
                    continue;

                if ((jobWrapper.Mode.Equals(State.ExecutionModeEnum.Initialization) &&
                        !jobWrapper.States[d.Name].Equals(State.CurrentStateEnum.InitializationSuccessful)) ||
                     (jobWrapper.Mode.Equals(State.ExecutionModeEnum.Execution) &&
                        !jobWrapper.States[d.Name].Equals(State.CurrentStateEnum.ExecutionSuccessful) &&
                        !jobWrapper.Job.Limiter.ifWaitingForFirstTrigger))
                {

                    bool ifAllParentsAreReady = true;
                    foreach (IDependency<IModule> dParent in d.Parents)
                    {
                        if (!dParent.Name.Equals(jobWrapper.Job.JobName))
                        {
                            if (jobWrapper.Mode == State.ExecutionModeEnum.Initialization)
                            {
                                if (!jobWrapper.States[dParent.Name].Equals(State.CurrentStateEnum.WaitState) &&
                                    !jobWrapper.States[dParent.Name].Equals(State.CurrentStateEnum.InitializationSuccessful))
                                {
                                    ifAllParentsAreReady = false;
                                    break;
                                }
                            }
                            else if (jobWrapper.Mode == State.ExecutionModeEnum.Execution)
                            {
                                if (!jobWrapper.States[dParent.Name].Equals(State.CurrentStateEnum.WaitState) &&
                                    !jobWrapper.States[dParent.Name].Equals(State.CurrentStateEnum.ExecutionSuccessful))
                                {
                                    ifAllParentsAreReady = false;
                                    break;
                                }
                            }
                        }
                    }

                    if (ifAllParentsAreReady)
                    {
                        int result = ProcessDependency(jobWrapper, jobkey, d);
                        if (result == -99)
                            retval = ErrorCode.ABORT_JOB;
                        else if (result != 0)
                            retval = ErrorCode.MODULE_FAILED;

                        if (retval.Equals(ErrorCode.ABORT_JOB) == false)
                            ReenableJob(jobWrapper, trackWaitState, d);

                        log.InfoFormat("Processed {0}", d.ToString());
                    }
                }
            }

            foreach (string job in trackWaitState)
            {
                if (jobWrapper.States.ContainsKey(job))
                    jobWrapper.States[job] = State.CurrentStateEnum.WaitState;
            }

            return retval;
        }
Ejemplo n.º 3
0
 private void ReenableJob(JobExeWrapper jobWrapper, ArrayList trackWaitState, IDependency<IModule> d)
 {
     if (jobWrapper.Mode.Equals(State.ExecutionModeEnum.Execution))
         foreach (IDependency<IModule> dChild in d.Children)
         {
             if (jobWrapper.States[d.Name].Equals(State.CurrentStateEnum.WaitState))
             {
                 ResetState(jobWrapper, dChild, State.CurrentStateEnum.PendingExecution);
                 if (!trackWaitState.Contains(d.Name))
                     trackWaitState.Add(d.Name);
             }
             else
                 ResetState(jobWrapper, dChild, State.CurrentStateEnum.PendingExecution);
         }
 }
Ejemplo n.º 4
0
 private void ResetState(JobExeWrapper jobWrapper, IDependency<IModule> dep, State.CurrentStateEnum newState)
 {
     jobWrapper.States[dep.Name] = newState;
     return;
 }
Ejemplo n.º 5
0
        private void ResetDependentOutputs(JobExeWrapper jobWrapper, IDependency<IModule> dep)
        {
            #region pick enumeration type

            IEnumerator<IDependency<IModule>> iEnumerator =
                dep.GetBreadthFirstEnumerator(); //default mode

            iEnumerator = GetDependencyEnumerator(jobWrapper, dep, iEnumerator);

            #endregion

            //execution
            log.InfoFormat("Resetting dependent outputs on {0}/{1}.", jobWrapper.Job.JobName, dep.Name);

            iEnumerator = GetDependencyEnumerator(jobWrapper, dep, iEnumerator);
            while (iEnumerator.MoveNext())
            {
                IDependency<IModule> d = iEnumerator.Current;

                d.Module.ResetOutput();
            }
        }
Ejemplo n.º 6
0
        private bool QueueJob(JobExeWrapper jobWrapper)
        {
            bool ifQueueJob = false;
            lock (jobWrapper)
            {
                foreach (KeyValuePair<string, StateEnums.ExecutionState> kvp in jobWrapper.DependencyStates)
                    if (kvp.Value.Equals(StateEnums.ExecutionState.Executing))
                    {
                        ifQueueJob = true;
                        break;
                    }
            }

            if (ifQueueJob)
            {
                bool isAlreadyRunningForFile = false;
                lock (queuedJobs)
                {
                    foreach (JobExeWrapper jw in queuedJobs)
                    {
                        if (((State)jw.JobGlobalState).SourceFileName.Equals(((State)jobWrapper.JobGlobalState).SourceFileName))
                        {
                            isAlreadyRunningForFile = true;
                            break;
                        }
                    }

                    if (isAlreadyRunningForFile)
                    {
                        log.WarnFormat("Found another job [{0}] queued and executing for file {1}", jobWrapper.Job.KeyHash, ((State)jobWrapper.JobGlobalState).SourceFileName);
                        log.WarnFormat("Suppressing job [{0}] for file {1}", jobWrapper.Job.KeyHash, ((State)jobWrapper.JobGlobalState).SourceFileName);
                    }
                    else
                    {
                        queuedJobs.Enqueue(jobWrapper);
                        log.WarnFormat("Found another job [{0}] executing for file {1}", jobWrapper.Job.KeyHash, ((State) jobWrapper.JobGlobalState).SourceFileName);
                        log.WarnFormat("Queuing job [{0}] for file {1}", jobWrapper.Job.KeyHash, ((State)jobWrapper.JobGlobalState).SourceFileName);
                    }
                }
            }
            return ifQueueJob;
        }
Ejemplo n.º 7
0
        private ErrorCode ProcessJob(Job job, DateTime jobkey, JobExeWrapper jobWrapper)
        {
            ErrorCode retval = ErrorCode.SUCCESS;

            if (job.Dependencies.ContainsKey(job.JobName))
            {
                IDependency<IModule> dep = job.Dependencies[job.JobName] as IDependency<IModule>;
                retval = ProcessJobFromRoot(jobWrapper, jobkey, dep);

                UpdateJobDetailsForGui(jobWrapper, State.CurrentStateEnum.WaitState);
            }
            else
                log.WarnFormat("Root job dependency for {0} not found", job.JobName);

            return retval;
        }
Ejemplo n.º 8
0
	    private ErrorCode ProcessJobFromRoot(JobExeWrapper jobWrapper, Job job, IDependency<IModule> dep)
		{
			ErrorCode returnCode = ErrorCode.SUCCESS;

			if (killJobs.Contains(job.JobName))
                killJobs.Remove(job.JobName);

			// pick enumeration type
			IEnumerator<IDependency<IModule>> iEnumerator =
				dep.GetBreadthFirstEnumerator(); //default mode

			//execution
            log.InfoFormat(job.KeyHash, "Beginning executing job {0} in {1} mode.", job.JobName, jobWrapper.DependencyState.ToString());

			ArrayList trackWaitState = new ArrayList();

            job.StartStats();
            iEnumerator = GetDependencyEnumerator(jobWrapper, dep, iEnumerator);
			while (iEnumerator.MoveNext())
			{
				IDependency<IModule> d = iEnumerator.Current;
				if (d.Name.Equals(job.JobName))
					continue;
				if (!jobWrapper.DependencyStates.ContainsKey(d.Name))
					jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.PendingInitialization;
			}

			iEnumerator = GetDependencyEnumerator(jobWrapper, dep, iEnumerator);
            while (iEnumerator.MoveNext() && returnCode.Equals(ErrorCode.ABORT_JOB) == false)
			{
				IDependency<IModule> d = iEnumerator.Current;

				if (killJobs.Contains(job.JobName))
				{
                    killJobs.Remove(job.JobName);
					ReenableJob(jobWrapper, trackWaitState, d);

                    log.WarnFormat(job.KeyHash, "Killing job {0} ", job.JobName);

					Thread.CurrentThread.Abort();
					break;
				}

                if (d.Name.Equals(job.JobName))
					continue;

                if ((jobWrapper.DependencyState.Equals(StateEnums.ExecutionMode.Initialization) &&
						!jobWrapper.DependencyStates[d.Name].Equals(StateEnums.ExecutionState.InitializationSuccessful)) ||
                     (jobWrapper.DependencyState.Equals(StateEnums.ExecutionMode.Execution) &&
						!jobWrapper.DependencyStates[d.Name].Equals(StateEnums.ExecutionState.ExecutionSuccessful) &&
						!jobWrapper.Job.Limiter.ifWaitingForFirstTrigger))
				{

					bool ifAllParentsAreReady = true;
					foreach (IDependency<IModule> dParent in d.Parents)
					{
						if (!dParent.Name.Equals(jobWrapper.Job.JobName))
						{
                            if (jobWrapper.DependencyState == StateEnums.ExecutionMode.Initialization)
							{
								if (!jobWrapper.DependencyStates[dParent.Name].Equals(StateEnums.ExecutionState.WaitState) &&
									!jobWrapper.DependencyStates[dParent.Name].Equals(StateEnums.ExecutionState.InitializationSuccessful))
								{
									ifAllParentsAreReady = false;
									break;
								}
							}
                            else if (jobWrapper.DependencyState == StateEnums.ExecutionMode.Execution)
							{
								if (!jobWrapper.DependencyStates[dParent.Name].Equals(StateEnums.ExecutionState.WaitState) &&
									!jobWrapper.DependencyStates[dParent.Name].Equals(StateEnums.ExecutionState.ExecutionSuccessful))
								{
									ifAllParentsAreReady = false;
									break;
								}
							}
						}
					}

					if (ifAllParentsAreReady)
					{
                        #region Status Update
                        if (jobWrapper.DependencyState.Equals(StateEnums.ExecutionMode.Initialization))
                            job.JobState.CurrentStatusCollection[d.Name].Status = StateEnums.Status.Initializing;
                        else
                        {
                            job.JobState.CurrentStatusCollection[d.Name].Status = StateEnums.Status.Running;
                            //Required for UpdateStatus (Dashboard)
                            job.JobState.CurrentStatusCollection[d.Name].StartTime = DateTime.Now;
                        }

                        OnStatusEvent(job);
                        #endregion

                        //Begin dependency/module processing...
						StateEnums.Status result = ProcessDependency(jobWrapper, job, d);

                        #region Status Update
                        switch (result)
                        {
                            #region Normal Cases
                            case StateEnums.Status.Success:
                                if (jobWrapper.DependencyState.Equals(StateEnums.ExecutionMode.Initialization))
                                    jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.InitializationSuccessful;
                                else
                                {
                                    jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.ExecutionSuccessful;
                                    job.JobState.CurrentStatusCollection[d.Name].EndTime = DateTime.Now;
                                }
                                job.JobState.CurrentStatusCollection[d.Name].Status = StateEnums.Status.Success;
                                break;
                            case StateEnums.Status.WaitingForSqlNotification:
                                jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.WaitState;
                                job.JobState.CurrentStatusCollection[d.Name].Status = StateEnums.Status.WaitingForSqlNotification;
                                break;
                            case StateEnums.Status.WaitingForFile:
                                jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.WaitState;
                                job.JobState.CurrentStatusCollection[d.Name].Status = StateEnums.Status.WaitingForFile;
                                break;
                            #endregion

                            #region Abnormal Cases
                            case StateEnums.Status.Warnings:
                                if (jobWrapper.DependencyState.Equals(StateEnums.ExecutionMode.Initialization))
                                    jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.InitializationSuccessful;
                                else
                                {
                                    jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.ExecutionSuccessful;
                                    job.JobState.CurrentStatusCollection[d.Name].EndTime = DateTime.Now;
                                }
                                job.JobState.CurrentStatusCollection[d.Name].Status = StateEnums.Status.Warnings;
                                break;
                            case StateEnums.Status.Error:
                                if (jobWrapper.DependencyState.Equals(StateEnums.ExecutionMode.Initialization))
                                    jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.InitializationSuccessful;
                                else
                                {
                                    //jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.ExecutionSuccessful;
                                    jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.ExecutionFailed;
                                    job.JobState.CurrentStatusCollection[d.Name].EndTime = DateTime.Now;
                                }
                                job.JobState.CurrentStatusCollection[d.Name].Status = StateEnums.Status.Error;
                                returnCode = ErrorCode.ABORT_JOB;
                                break;
                            default:
                                if (jobWrapper.DependencyState.Equals(StateEnums.ExecutionMode.Initialization))
                                    jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.InitializationSuccessful;
                                else
                                {
                                    jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.ExecutionSuccessful;
                                    job.JobState.CurrentStatusCollection[d.Name].EndTime = DateTime.Now;
                                }
                                job.JobState.CurrentStatusCollection[d.Name].Status = StateEnums.Status.Error;
                                returnCode = ErrorCode.MODULE_FAILED;
                                break;
                            #endregion
                        }
                        OnStatusEvent(job);
                        #endregion

                        if ( !returnCode.Equals(ErrorCode.ABORT_JOB))
							ReenableJob(jobWrapper, trackWaitState, d);

						log.DebugFormat(job.KeyHash,"Processed {0}", d.ToString());
					}
				}
			}
            job.StopStats();

			foreach (string jobName in trackWaitState)
			{
                if (jobWrapper.DependencyStates.ContainsKey(jobName))
                    jobWrapper.DependencyStates[jobName] = StateEnums.ExecutionState.WaitState;
			}

            return returnCode;
		}
Ejemplo n.º 9
0
        private bool QueueJob(JobExeWrapper jobWrapper)
        {
            bool ifQueueJob = false;
            foreach (KeyValuePair<string, State.CurrentStateEnum> kvp in jobWrapper.States)
                if (kvp.Value.Equals(State.CurrentStateEnum.Executing))
                {
                    lock (queuedJobs)
                    {
                        bool isAlreadyRunningForFile = false;
                        foreach (JobExeWrapper jw in queuedJobs)
                            if (jw.SourceFileName.Equals(jobWrapper.SourceFileName))
                            {
                                isAlreadyRunningForFile = true;
                                log.WarnFormat("Found another job [{0}] queued and executing for file {1}", jw.Job.KeyCode, jobWrapper.SourceFileName);
                                log.WarnFormat("Suppressing job [{0}] for file {1}", jobWrapper.Job.KeyCode, jobWrapper.SourceFileName);
                                break;
                            }

                        if (isAlreadyRunningForFile == false)
                            queuedJobs.Enqueue(jobWrapper);
                    }
                    ifQueueJob = true;
                }
            return ifQueueJob;
        }
Ejemplo n.º 10
0
        private void ResetState(JobExeWrapper jobWrapper, IDependency<IModule> dep, StateEnums.ExecutionState newState)
		{
			jobWrapper.DependencyStates[dep.Name] = newState;
			return;
		}
Ejemplo n.º 11
0
 private ErrorCode ProcessJobFromRoot(JobExeWrapper jobWrapper, IDependency<IModule> dep)
 {
     return ProcessJobFromRoot(jobWrapper, jobWrapper.Job, dep);
 }
Ejemplo n.º 12
0
		private ErrorCode ProcessJob(JobExeWrapper jobWrapper)
		{
			ErrorCode retval = ErrorCode.SUCCESS;

            if (jobWrapper.Job.Dependencies.ContainsKey(jobWrapper.Job.JobName))
			{
                IDependency<IModule> dep = jobWrapper.Job.Dependencies[jobWrapper.Job.JobName] as IDependency<IModule>;
				retval = ProcessJobFromRoot(jobWrapper, dep);
			}
			else
                log.WarnFormat("Root job dependency for {0} not found", jobWrapper.Job.JobName);

			return retval;
		}
Ejemplo n.º 13
0
        /// <summary>
        /// Called internally, except for JobLauncher module
        /// </summary>
        /// <param name="oJobDetail"></param>
        /// <returns></returns>
        public ErrorCode ExecuteJobNew(JobDetails jobDetail, ref IJobWrapper ijobWrapper)
        {
            if (jobDetail.Xml.Equals(String.Empty))
            {
                jobDetail.Xml = globalConfig.Hierarchy.GetValueAtLevel("app\\BITS_JOB_SPEC", jobDetail.Name.Replace("_Job", ""), "");
                if (jobDetail.Xml.Equals(String.Empty))
                {
                    log.ErrorFormat("No job spec found for {0}.  Skipping request to build job.", jobDetail.Name);
                    return ErrorCode.JOB_SPEC_NOT_FOUND;
                }
            }

            if (jobWrappers.ContainsKey(jobDetail.Name) && jobDetail.Name.Contains("ADHOC") == false)
            {
                return ErrorCode.JOB_CODE_NOT_UNIQUE;
            }

            // Set up job spec for "Run Once."  This means that the start time will be set to *IMMEDIATELY*, the 
            // weekly schedule will be set to SMTWTFS. 
            if (jobDetail.RunOnce)
            {
                XmlDocument xdocAdhoc = Job.SetRunOnceAttributes(jobDetail.Xml, true);
                jobDetail.Xml = xdocAdhoc.InnerXml;
            }

        #endregion

            // Obtain a new PropertyBag for the job's metatags (ie the job's Dynamic Parameters).  In this PropertyBag, 
            // the parameters that were identified as Environmental will have been set.  Those that are not Environmental,
            // (ie Runtime) will have a value of null;
            PropertyBag jobMetaTags = MetaTagReplacer.GetJobParametersPB(jobDetail.Name, jobDetail.Xml);
            MetaTagReplacer.SetParametersValuesPB(ref jobMetaTags, ConfigUtils.MetatagMode.Environment, null, null);

            // Use the Dynamic Parameters PropertyBag to define the job spec at Enable-time.  Only Environment parameters
            // will be determined at this time.
            jobDetail.Xml = MetaTagReplacer.InlineMetatagSubstitution(jobDetail.Xml, jobMetaTags);

            // Create the job.  This step instantiates the job's modules and sets up the job's dependencies 
            Job job = (Job)JobImpl.BuildJobFromXml(jobDetail);
            if (job == null)
            {
                log.ErrorFormat("Unable to build job from job spec for {0}.  Skipping request to build job.", jobDetail.Name);
                return ErrorCode.UNABLE_TO_BUILD_JOB_FROM_SPEC;
            }

            job.Config.RunInstance = globalConfig.RunInstance;
            job.Config.SuppressEmail = globalConfig.SuppressEmail;
            job.JobState.CurrentParameters.CurrentParameters = jobMetaTags;

            foreach (JobDetailsData jdd in jobDetail.Data)
            {
                ((IModule)job.Modules[jdd.ModuleName]).OutputData.Add(jdd.DataName, jdd.Data);
            }

            // Create JobWrapper
            JobExeWrapper jobWrapper = null;
            if (jobWrappers.ContainsKey(job.JobName))
            {
                jobWrapper = jobWrappers[job.JobName];
                job.Key = DateTime.Now;
                jobWrapper.Job = job;
                jobWrapper.ResetState();
            }
            else
            {
                job.Key = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd"));
                jobWrapper = new JobExeWrapper(job);
            }
            ijobWrapper = jobWrapper;

            // Job will be enabled after this...
            lock (jobWrappers)
            {
                jobWrappers[job.JobName] = jobWrapper;
            }

            return ProcessJob(jobWrapper);
        }
Ejemplo n.º 14
0
        /// <summary>
        /// Called internally, except for JobLauncher module
        /// </summary>
        /// <param name="oJobDetail"></param>
        /// <returns></returns>
        public ErrorCode ExecuteJob(object oJobDetail)
        {
            ErrorCode retval = ErrorCode.SUCCESS;

            #region execution

            PerfTimer hpt = new PerfTimer();
            hpt.Start();

            JobDetails jobDetail = oJobDetail as JobDetails;

            #region sanity checks and prep

            if (jobDetail == null)
            {
                log.ErrorFormat("Job detail is invalid.  Failed to execute job.");
                return ErrorCode.JOB_DETAIL_INVALID;
            }

            if (jobDetail.Xml.Equals(String.Empty))
            {
                jobDetail.Xml = globalConfig.Hierarchy.GetValueAtLevel("app\\BITS_JOB_SPEC", jobDetail.Name.Replace("_Job", ""), "");

                if (jobDetail.Xml.Equals(String.Empty))
                {
                    log.ErrorFormat("No job spec found for {0}.  Skipping request to build job.", jobDetail.Name);
                    return ErrorCode.JOB_SPEC_NOT_FOUND;
                }
            }

            if (jobWrappers.ContainsKey(jobDetail.Name) && jobDetail.Name.Contains("ADHOC") == false)
            {
                return ErrorCode.JOB_CODE_NOT_UNIQUE;
            }

            // Set up job spec for "Run Once."  This means that the start time will be set to *IMMEDIATELY*, the 
            // weekly schedule will be set to SMTWTFS. 
            if (jobDetail.RunOnce)
            {
                XmlDocument xdocAdhoc = Job.SetRunOnceAttributes(jobDetail.Xml, true);
                jobDetail.Xml = xdocAdhoc.InnerXml;
            }

            #endregion

            // Obtain a new PropertyBag for the job's metatags (ie the job's Dynamic Parameters).  In this PropertyBag, 
            // the parameters that were identified as Environmental will have been set.  Those that are not Environmental,
            // (ie Runtime) will have a value of null;
            PropertyBag jobMetaTags = MetaTagReplacer.GetJobParametersPB(jobDetail.Name, jobDetail.Xml);
            MetaTagReplacer.SetParametersValuesPB(ref jobMetaTags, ConfigUtils.MetatagMode.Environment, null, null);

            // Use the Dynamic Parameters PropertyBag to define the job spec at Enable-time.  Only Environment parameters
            // will be determined at this time.
            jobDetail.Xml = MetaTagReplacer.InlineMetatagSubstitution(jobDetail.Xml, jobMetaTags);

            // Create the job.  This step instantiates the job's modules and sets up the job's dependencies 
            Job job = (Job)JobImpl.BuildJobFromXml(jobDetail);
            if (job == null)
            {
                log.ErrorFormat("Unable to build job from job spec for {0}.  Skipping request to build job.", jobDetail.Name);
                return ErrorCode.UNABLE_TO_BUILD_JOB_FROM_SPEC;
            }
            //job.Key = DateTime.Now;
            job.Config.RunInstance = globalConfig.RunInstance;
            job.Config.SuppressEmail = globalConfig.SuppressEmail;
            job.JobState.CurrentParameters.CurrentParameters = jobMetaTags;
            //job.JobState.CurrentJobCode = job.Key;

            foreach (JobDetailsData jdd in jobDetail.Data)
            {
                ((IModule)job.Modules[jdd.ModuleName]).OutputData.Add(jdd.DataName, jdd.Data);
            }

            // Job will be enabled after this...
            JobExeWrapper jobWrapper = null;
            lock (jobWrappers)
            {
                if (jobWrappers.ContainsKey(job.JobName))
                    jobWrapper = jobWrappers[job.JobName];
                else
                    jobWrapper = new JobExeWrapper(job);

                jobWrapper.Job = job;
                jobWrapper.ResetState();
                jobWrappers[job.JobName] = jobWrapper;
            }

            retval = ProcessJob(jobWrapper);

            hpt.Stop();

            #endregion

            #region Wrap-up

            log.InfoFormat(job.KeyHash, "Completed executing job {0} in {1} mode in {2} secs.", jobWrapper.Job.JobName, jobWrapper.DependencyState.ToString(), hpt.Duration);

            if (log.IsInfoEnabled)
            {
                log.DebugFormat("Publishing job {0}", job.JobName);
                job.PublishDependencies();
            }

            #endregion

            return retval;
        }
Ejemplo n.º 15
0
        private int ProcessDependency(JobExeWrapper jobWrapper, DateTime jobkey, IDependency<IModule> d)
        {
            log.InfoFormat("Considering dependency {0}-[{1}] for {2}.", jobWrapper.Job.JobName, d.Name, jobWrapper.Mode.ToString());

            if (((State)jobWrapper.JobGlobalState).CurrentState.CompareTo(State.CurrentStateEnum.Suspended) == 0)
                return 0;

            int retval = 0;

            switch (jobWrapper.Mode)
            {
                case State.ExecutionModeEnum.Initialization:
                    jobWrapper.States[d.Name] = State.CurrentStateEnum.Initializing;

                    try
                    {
                        log.InfoFormat("Starting {2}: {0}-[{1}].", jobWrapper.Job.JobName, d.Name, jobWrapper.Mode.ToString());

                        retval = d.Module.Initialize(jobWrapper.Job as IJob, flatFileLoader, timeManager);
                    }
                    catch (Exception ex)
                    {
                        log.ErrorFormat("Module Initialization Error: {0}", ex);
                        retval = -1;
                    }
                    finally
                    {
                        log.InfoFormat("Finished {0}: {1}:{2}", jobWrapper.Mode.ToString(), jobWrapper.Job.JobName, d.Name);
                    }

                    if (retval == 0)
                        jobWrapper.States[d.Name] = State.CurrentStateEnum.InitializationSuccessful;
                    else if (retval == 1)
                        jobWrapper.States[d.Name] = State.CurrentStateEnum.WaitState;
                    else
                        jobWrapper.States[d.Name] = State.CurrentStateEnum.InitializationFailed;

                    break;

                case State.ExecutionModeEnum.Execution:

                    jobWrapper.States[d.Name] = State.CurrentStateEnum.Executing;
                    if (d.Name == jobWrapper.Job.JobName) break;

                    Hashtable inputData = new Hashtable();
                    if (d.Module.Inputs != null && d.Parents != null)
                    {
                        foreach (IDependency<IModule> p in d.Parents)
                        {
                            Hashtable ht = ((IModule)jobWrapper.HistoricalJobs[jobkey].Modules[p.Module.Name]).OutputData;
                            foreach (string key in ht.Keys)
                                inputData[key] = ht[key];
                        }
                    }

                    try
                    {
                        log.InfoFormat("Starting {2} on {0}-[{1}].", jobWrapper.Job.JobName, d.Name, jobWrapper.Mode.ToString());

                        ((State)jobWrapper.JobGlobalState).Dependency = d;

                        UpdateProcessStatus(State.Status.Running, (State)jobWrapper.JobGlobalState);

                        //  THIS IS THE MAIN [ONLY!] ENTRY POINT TO MODULE EXECUTION
                        retval = d.Module.Execute(this, jobWrapper.JobGlobalState, inputData);
                    }
                    catch (Exception ex)
                    {
                        UpdateProcessStatus(State.Status.Error, (State)jobWrapper.JobGlobalState);
                        log.ErrorFormat(jobWrapper.Job.KeyCode, "Module Execution Error: {0}", ex);
                    }
                    finally
                    {
                        log.InfoFormat(jobWrapper.Job.KeyCode, "Finished {0}: {1}:{2}", jobWrapper.Mode.ToString(), jobWrapper.Job.JobName, d.Name);

                        ((IModule)jobWrapper.HistoricalJobs[jobkey].Modules[d.Module.Name]).OutputData = d.Module.OutputData;

                        if (jobsWaitingForExecution.ContainsKey(d.Name))
                            if (d.Parents[0].Parents.Count != 0)
                                jobsWaitingForExecution.Remove(d.Name);
                            else
                                jobWrapper.States[d.Name] = State.CurrentStateEnum.WaitState;   //importatn; reset job back to wait state

                        if (jobWrapper.States[d.Name] != State.CurrentStateEnum.WaitState)
                        {
                            switch (retval)
                            {
                                case 0:
                                case -1:
                                    jobWrapper.States[d.Name] = State.CurrentStateEnum.ExecutionSuccessful;
                                    break;
                                case 1:
                                    jobWrapper.States[d.Name] = State.CurrentStateEnum.PendingExecution;
                                    break;
                                //case -1:
                                //    jobWrapper.States[d.Name] = State.CurrentStateEnum.ExecutionFailed;
                                //    break;
                                case -99:
                                default:
                                    jobWrapper.States[d.Name] = State.CurrentStateEnum.ExecutionFailed;
                                    break;
                            }
                        }

                        switch (retval)
                        {
                            case 0:
                                UpdateProcessStatus(State.Status.Success, (State)jobWrapper.JobGlobalState);
                                break;
                            case 1:
                                UpdateProcessStatus(State.Status.Running, (State)jobWrapper.JobGlobalState);
                                break;
                            case -1:
                                UpdateProcessStatus(State.Status.Warnings, (State)jobWrapper.JobGlobalState);
                                break;
                            case -99:
                            default:
                                UpdateProcessStatus(State.Status.Error, (State)jobWrapper.JobGlobalState);
                                break;
                        }

                        //BroadcastStatus("Idle", jobWrapper);
                    }

                    jobWrapper.Job.Modules[d.Name] = d.Module as IModule;

                    break;
            }
            return retval;
        }
Ejemplo n.º 16
0
        /// <summary>
        /// Called internally, except for JobLauncher module
        /// </summary>
        /// <param name="oJobDetail"></param>
        /// <returns></returns>
        public  ErrorCode ExecuteJob(object oJobDetail)
        {
            ErrorCode retval = ErrorCode.SUCCESS;

            #region execution

            PerfTimer hpt = new PerfTimer();
            hpt.Start();

            JobExeWrapper jobWrapper = null;
            JobDetails jobDetail = oJobDetail as JobDetails;

            if (jobDetail == null)
            {
                log.ErrorFormat("Job detail is invalid.  Failed to execute job.");
                return ErrorCode.JOB_DETAIL_INVALID;
            }

            if (jobDetail.Xml.Equals(String.Empty))
            {
                jobDetail.Xml = globalConfig.Hierarchy.GetValueAtLevel("appgroup\\BITS_JOB_SPEC", jobDetail.Name.Replace("_Job", ""), "");

                if (jobDetail.Xml.Equals(String.Empty))
                {
                    log.ErrorFormat("No job spec found for {0}.  Skipping request to build job.", jobDetail.Name);
                    return ErrorCode.JOB_SPEC_NOT_FOUND;
                }
            }

            Job job = (Job)JobImpl.BuildJobFromXml(jobDetail);
            if (job == null)
            {
                log.ErrorFormat("Unable to build job from job spec for {0}.  Skipping request to build job.", jobDetail.Name);
                return ErrorCode.UNABLE_TO_BUILD_JOB_FROM_SPEC;
            }
            else
            {
                job.Key = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd"));
                job.Config.RunInstance = globalConfig.RunInstance;
                job.Config.SuppressEmail = globalConfig.SuppressEmail;
            }

            if (jobWrappers.ContainsKey(job.JobName))
            {
                //replaces job with new definition from xml
                jobWrapper = jobWrappers[job.JobName];
                jobWrapper.Job = job;
                lock (jobWrappers)
                {
                    jobWrappers[job.JobName] = jobWrapper;
                }
            }
            else
            {
                jobWrapper = new JobExeWrapper(job);
                lock (jobWrappers)
                {
                    jobWrappers[job.JobName] = jobWrapper;
                }
            }

            lock (jobWrappers[job.JobName].HistoricalJobs)
            {
                if (!jobWrappers[job.JobName].HistoricalJobs.ContainsKey(job.Key))
                {
                    jobWrappers[job.JobName].HistoricalJobs.Add(job.Key, job);

                    jobWrapper.Job = job;
                    jobWrapper.States.Clear();
                    jobWrapper.ChangeMode(State.ExecutionModeEnum.Initialization);

                    retval = ProcessJob(job, job.Key, jobWrapper);
                }
                else
                    log.WarnFormat("Job already exists, aborting new job execution for {0} {1}", job.JobName, job.Key);
            }

            hpt.Stop();
            jobWrapper.DurationSecs = hpt.Duration;

            #endregion

            #region Wrap-up

            log.InfoFormat("Completed executing job {0} in {1} mode in {2} secs.", jobWrapper.Job.JobName, jobWrapper.Mode.ToString(), hpt.Duration);

            if (log.IsInfoEnabled)
            {
                log.InfoFormat("Publishing job {0}", job.JobName);
                job.PublishDependencies();
            }

            #endregion

            return retval;
        }
Ejemplo n.º 17
0
        private void UpdateJobDetailsForGui(JobExeWrapper jobWrapper, State.CurrentStateEnum state)
        {
            lock (guiJobStatus)
            {
                ((State)jobWrapper.JobGlobalState).CurrentState = state;

                if (guiJobStatus.ContainsKey(jobWrapper.Job.JobName) == false)
                    guiJobStatus.Add(jobWrapper.Job.JobName, "Idle");
                if (guiJobHistories.ContainsKey(jobWrapper.Job.JobName) == false)
                    guiJobHistories.Add(jobWrapper.Job.JobName, new List<string>());

                lock (jobWrappers)
                {
                    foreach (JobExeWrapper jw in jobWrappers.Values)
                    {
                        switch (((Bits.StateNS.State)(jw.JobGlobalState)).CurrentState)
                        {
                            case State.CurrentStateEnum.Executing:
                                guiJobStatus[jw.Job.JobName] = "Busy";
                                break;
                            case State.CurrentStateEnum.Suspended:
                                guiJobStatus[jw.Job.JobName] = "Suspended";
                                break;
                            default:
                                guiJobStatus[jw.Job.JobName] = "Idle";
                                break;
                        }

                        foreach (DateTime dt in jw.HistoricalJobs.Keys)
                        {
                            string sdt = dt.ToString("yyyy-MM-dd HH:mm:ss");
                            if (guiJobHistories.ContainsKey(jw.Job.JobName))
                            {
                                if (guiJobHistories[jw.Job.JobName].Contains(sdt) == false)
                                    guiJobHistories[jw.Job.JobName].Add(sdt);
                            }
                        }
                    }
                }
            }
        }
Ejemplo n.º 18
0
        private StateEnums.Status ProcessDependency(JobExeWrapper jobWrapper, Job job, IDependency<IModule> d)
        {
            log.DebugFormat(job.KeyHash,"Considering dependency {0}-[{1}] for {2}.", job.JobName, d.Name, jobWrapper.DependencyState.ToString());
            StateEnums.Status retval = StateEnums.Status.Unknown;

            if (((State)jobWrapper.JobGlobalState).ExecutionState.CompareTo(StateEnums.ExecutionState.Suspended) == 0)
                return 0;

            job.JobState.CurrentDependency = d; //Required for status updates

            log.DebugFormat(job.KeyHash,"Starting {2}: {0}-[{1}].", job.JobName, d.Name, jobWrapper.DependencyState);
            switch (jobWrapper.DependencyState)
            {
                case StateEnums.ExecutionMode.Initialization:

                    try
                    {
                        jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.Initializing;
                        retval = d.Module.Initialize(job as IJob, flatFileLoader, timeManager, datamonitorManager);
                    }
                    catch (Exception ex)
                    {
                        log.ErrorFormat(job.KeyHash,"Module Initialization Error: {0}", ex.Message);
						log.Error(job.KeyHash, ex);
						retval = StateEnums.Status.Error;
                    }

                    switch (retval)
                    {
                        case StateEnums.Status.Success:
                            jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.InitializationSuccessful;
                            break;
                        case StateEnums.Status.WaitingForSqlNotification:
                        case StateEnums.Status.WaitingForFile:
                            jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.WaitState;
                            break;
                        default:
                            jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.InitializationFailed;
                            break;
                    }
                    break;

                case StateEnums.ExecutionMode.Execution:

                    log.DebugFormat(job.KeyHash,"Starting {2} on {0}-[{1}].", job.JobName, d.Name, jobWrapper.DependencyState.ToString());

                    try
                    {
                        jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.Executing;
                        if (d.Name == job.JobName) break;

                        //job.de

                        // Handle data bus (pass all objects along to next dependency)
                        // and add the outputdata from Initial Bus Data
                        Hashtable inputData = new Hashtable();
                        if (d.Module.OutputData != null && d.Module.OutputData.Count > 0)
                            foreach (string key in d.Module.OutputData.Keys)
                                inputData[key] = d.Module.OutputData[key];

                        Hashtable outputData = new Hashtable();
                        if (d.Module.Inputs != null && d.Parents != null)
                        {
                            foreach (IDependency<IModule> p in d.Parents)
                            {
                                // add the outputdata from parent modules running in this instance
                                outputData = ((IModule)jobWrapper.HistoricalJobs[job.Key].Modules[p.Module.Name]).OutputData;
                                foreach (string key in outputData.Keys)
                                    inputData[key] = outputData[key];
                            }
                        }

                        inputData["JobStatsSummary"] = String.Format("Job: {0} [{1}] <br># Modules: {2}<br>Duration: {3} secs",
                                                       job.JobName,
                                                       job.Key,
                                                       job.JobStats.ModuleCount,
                                                       job.JobStats.Duration.ToString("#.000"));

                        //  THIS IS THE MAIN [ONLY!] ENTRY POINT TO MODULE EXECUTION
                        retval = d.Module.Execute(this, job.JobState, inputData);

                    }
                    catch (Exception ex)
                    {
                        log.ErrorFormat(job.KeyHash, "Module Execution Error: {0}", ex);
                        retval = StateEnums.Status.Error;
                    }
                    finally
                    {
                        //log.InfoFormat(job.KeyHash, "Completed {0}: {1}:{2}", jobWrapper.DependencyState.ToString(), job.JobName, d.Name);

                        ((IModule)jobWrapper.HistoricalJobs[job.Key].Modules[d.Module.Name]).OutputData = d.Module.OutputData;

                        if (jobsWaitingForExecution.ContainsKey(d.Name))
                            if (d.Parents[0].Parents.Count != 0)
                                jobsWaitingForExecution.Remove(d.Name);
                            else
                                jobWrapper.DependencyStates[d.Name] = StateEnums.ExecutionState.WaitState; //importatn; reset job back to wait state

                        jobWrapper.Job.Modules[d.Name] = d.Module as IModule;
                    }

                    break;
            }
            log.InfoFormat(job.KeyHash,"Completed {0}: {1}:{2}", jobWrapper.DependencyState.ToString(), job.JobName, d.Name);
            return retval;
        }