コード例 #1
0
ファイル: JobQueueManager.cs プロジェクト: cburriss/SyncObjX
        public static void QueueJob(JobInstance jobInstance)
        {
            JobInstance existingJobInstance;

            if (runningJobsByIntegrationId.ContainsKey(jobInstance.Integration.Id))
            {
                existingJobInstance = runningJobsByIntegrationId[jobInstance.Integration.Id].Where(d => d.Id == jobInstance.Id).FirstOrDefault();

                if (existingJobInstance != null)
                {
                    throw new Exception(string.Format("A job instance with Id '{0}' is already running.", jobInstance.Id));
                }
            }

            if (waitingJobsByIntegrationId.ContainsKey(jobInstance.Integration.Id))
            {
                existingJobInstance = waitingJobsByIntegrationId[jobInstance.Integration.Id].Where(d => d.Id == jobInstance.Id).FirstOrDefault();

                if (existingJobInstance != null)
                {
                    throw new Exception(string.Format("A job instance with Id '{0}' is already waiting for execution.", jobInstance.Id));
                }

                waitingJobsByIntegrationId[jobInstance.Integration.Id].Add(jobInstance);
            }
            else
            {
                SetMaxConcurrentThreadsForIntegration(jobInstance.Integration);

                waitingJobsByIntegrationId.Add(jobInstance.Integration.Id, new List <JobInstance>()
                {
                    jobInstance
                });
            }

            // cache the job instance in memory if on-demand (temporary to get around OutOfMemoryException)
            if (jobInstance.InvocationSourceType == JobInvocationSourceType.OnDemand)
            {
                jobInstancesById.Add(jobInstance.Id, jobInstance);
                queueRequestsById.Add(jobInstance.QueueRequest.Id, jobInstance.QueueRequest);
            }

            jobInstance.StatusChanged += new EventHandler <JobQueueStatusChangedArgs>(HandleChangedJobStatus);

            foreach (var jobStepInstance in jobInstance.JobStepInstances)
            {
                jobStepInstance.StatusChanged += new EventHandler <JobStepQueueStatusChangedArgs>(HandleChangedJobStepStatus);
            }

            jobInstance.QueueRequest.Status = JobQueueRequestStatus.Waiting;

            jobInstance.Status = JobQueueStatus.Scheduled;

            jobInstance.RunningJobStepInstance = null;

            if (JobQueued != null)
            {
                JobQueued(null, new JobQueuedArgs(jobInstance));
            }
        }
コード例 #2
0
        public void Initialize(Integration integration, JobInstance jobInstance,
                               JobStepInstance previousJobStepInstance, JobStepInstance jobStepInstance, ISyncEngineConfigurator configurator)
        {
            Integration = integration;

            JobInstance = jobInstance;

            PreviousJobStepInstance = previousJobStepInstance;

            JobStepInstance = jobStepInstance;

            Configurator = configurator;
        }
コード例 #3
0
        public void AddToQueueLog(JobInstance jobInstance)
        {
            if (jobInstance == null)
            {
                throw new Exception("Job instance can not be null.");
            }

            using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
            {
                var now = DateTime.Now;

                var queuedJobInstance = new ls.JobQueue();

                queuedJobInstance.JobInstanceId      = jobInstance.Id;
                queuedJobInstance.QueueRequestId     = jobInstance.QueueRequest.Id;
                queuedJobInstance.JobId              = jobInstance.Job.Id;
                queuedJobInstance.CreatedDate        = now;
                queuedJobInstance.UpdatedDate        = now;
                queuedJobInstance.SourceDataSourceId = jobInstance.SourceDataSource.DataSource.Id;
                queuedJobInstance.TargetDataSourceId = jobInstance.TargetDataSource.DataSource.Id;
                queuedJobInstance.Filters            = JobFilterHelper.GetTextForDatabase(jobInstance.Filters);
                queuedJobInstance.InvocationSource   = jobInstance.InvocationSource;
                queuedJobInstance.ScheduledStartTime = jobInstance.ScheduledStartTime;
                queuedJobInstance.ActualStartTime    = jobInstance.ActualStartTime.HasValue ? jobInstance.ActualStartTime.Value : new DateTime?();
                queuedJobInstance.IsOnDemand         = jobInstance.InvocationSourceType == JobInvocationSourceType.OnDemand ? true : false;
                queuedJobInstance.JobQueueStatusId   = (byte)jobInstance.Status;

                dbContext.JobQueues.InsertOnSubmit(queuedJobInstance);

                dbContext.SubmitChanges();

                foreach (var jobStepInstance in jobInstance.JobStepInstances)
                {
                    var queuedJobStepInstance = new ls.JobStepQueue();

                    queuedJobStepInstance.JobStepInstanceId    = jobStepInstance.Id;
                    queuedJobStepInstance.JobInstanceId        = jobInstance.Id;
                    queuedJobStepInstance.JobStepId            = jobStepInstance.JobStep.Id;
                    queuedJobStepInstance.CreatedDate          = now;
                    queuedJobStepInstance.UpdatedDate          = now;
                    queuedJobStepInstance.ActualStartTime      = jobStepInstance.ActualStartTime.HasValue ? jobStepInstance.ActualStartTime.Value : new DateTime?();
                    queuedJobStepInstance.ActualEndTime        = jobStepInstance.ActualEndTime.HasValue ? jobStepInstance.ActualEndTime.Value : new DateTime?();
                    queuedJobStepInstance.OrderIndex           = (byte)jobStepInstance.OrderIndex;
                    queuedJobStepInstance.JobStepQueueStatusId = (byte)jobStepInstance.Status;

                    dbContext.JobStepQueues.InsertOnSubmit(queuedJobStepInstance);
                }

                dbContext.SubmitChanges();
            }
        }
コード例 #4
0
ファイル: JobQueueResult.cs プロジェクト: cburriss/SyncObjX
        public static JobInstanceResult GetResult(JobInstance jobInstance)
        {
            if (jobInstance == null)
            {
                return(null);
            }

            object _lock = new object();

            lock (_lock)
            {
                var jobInstanceResult = new JobInstanceResult();

                jobInstanceResult.IntegrationId        = jobInstance.Integration.Id;
                jobInstanceResult.JobId                = jobInstance.Job.Id;
                jobInstanceResult.QueueRequestId       = jobInstance.QueueRequest.Id;
                jobInstanceResult.JobInstanceId        = jobInstance.Id;
                jobInstanceResult.Filters              = jobInstance.Filters;
                jobInstanceResult.SourceDataSourceId   = jobInstance.SourceDataSource.DataSource.Id;
                jobInstanceResult.TargetDataSourceId   = jobInstance.TargetDataSource.DataSource.Id;
                jobInstanceResult.InvocationSource     = jobInstance.InvocationSource;
                jobInstanceResult.InvocationSourceType = jobInstance.InvocationSourceType;
                jobInstanceResult.Status               = jobInstance.Status;
                jobInstanceResult.ScheduledStartTime   = jobInstance.ScheduledStartTime;
                jobInstanceResult.TimeToStartDelay     = jobInstance.TimeToStartDelay;
                jobInstanceResult.ActualStartTime      = jobInstance.ActualStartTime;
                jobInstanceResult.ActualEndTime        = jobInstance.ActualEndTime;
                jobInstanceResult.ActualDuration       = jobInstance.ActualDuration;
                jobInstanceResult.HasRecordErrors      = jobInstance.HasRecordErrors;
                jobInstanceResult.HasRuntimeErrors     = jobInstance.HasRuntimeErrors;
                jobInstanceResult.Exceptions           = WebServiceException.Convert(jobInstance.Exceptions);

                var jobStepResults = new List <JobStepInstanceResult>();

                foreach (var jobStepInstance in jobInstance.JobStepInstances)
                {
                    jobStepResults.Add(GetResult(jobStepInstance));
                }

                jobInstanceResult.JobStepInstanceResults = jobStepResults;

                return(jobInstanceResult);
            }
        }
コード例 #5
0
        public void UpdateJobStatusInQueueLog(JobInstance jobInstance)
        {
            if (jobInstance == null)
            {
                throw new Exception("Job instance can not be null.");
            }

            using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
            {
                var queuedJobInstance = dbContext.JobQueues.Where(d => d.JobInstanceId == jobInstance.Id).FirstOrDefault();

                if (queuedJobInstance != null)
                {
                    queuedJobInstance.UpdatedDate      = DateTime.Now;
                    queuedJobInstance.ActualStartTime  = jobInstance.ActualStartTime.HasValue ? jobInstance.ActualStartTime.Value : new DateTime?();
                    queuedJobInstance.JobQueueStatusId = (byte)jobInstance.Status;

                    dbContext.SubmitChanges();
                }
            }
        }
コード例 #6
0
        public void UpdateJobDataSourceHistory(JobInstance jobInstance)
        {
            if (jobInstance == null)
            {
                throw new Exception("Job instance can not be null.");
            }

            if (!jobInstance.ActualStartTime.HasValue)
            {
                throw new Exception("Job instance is missing an actual start time.");
            }

            if (!jobInstance.ActualEndTime.HasValue)
            {
                throw new Exception("Job instance is missing an actual end time.");
            }

            using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
            {
                dbContext.UpdateJobDataSourceHistory(jobInstance.Job.Id, jobInstance.SourceDataSource.DataSource.Id, jobInstance.TargetDataSource.DataSource.Id,
                                                     jobInstance.QueueRequest.Id, jobInstance.Id, jobInstance.ActualStartTime.Value, jobInstance.ActualEndTime.Value,
                                                     jobInstance.HasRecordErrors, jobInstance.HasRuntimeErrors);
            }
        }
コード例 #7
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="jobInstanceId"></param>
        /// <returns>Returns null if a "sync object" no longer exists.</returns>
        public JobInstance GetJobInstanceFromQueueLog(Guid jobInstanceId, bool throwExceptionIfSyncObjectsAreDeletedOrDisabled = false)
        {
            if (!JobQueueManager.IsSafeForQueueLogRetrieval)
            {
                throw new Exception("The Job Queue Manager is in an unsafe state as job statuses may change. Queue retrieval from the database is disallowed.");
            }

            var configurator = new SyncEngineDatabaseConfigurator(_connectionString);

            using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
            {
                var jobInstanceInDbQueue = dbContext.JobQueues.Where(d => d.JobInstanceId == jobInstanceId).FirstOrDefault();

                var integration = configurator.GetIntegrationByJobId(jobInstanceInDbQueue.JobId);

                if (integration == null || !integration.IsEnabled)
                {
                    goto SyncObjectsForJobInstanceHaveChanged;
                }

                var job = integration.Jobs.Where(d => d.Id == jobInstanceInDbQueue.JobId).FirstOrDefault();

                if (job == null || !job.IsEnabled)
                {
                    goto SyncObjectsForJobInstanceHaveChanged;
                }

                var sourceDataSource = job.SourceDataSources.Values.Where(d => d.Id == jobInstanceInDbQueue.SourceDataSourceId).FirstOrDefault();

                if (sourceDataSource == null)
                {
                    goto SyncObjectsForJobInstanceHaveChanged;
                }

                if (job.TargetDataSource.Id != jobInstanceInDbQueue.TargetDataSourceId)
                {
                    goto SyncObjectsForJobInstanceHaveChanged;
                }

                var jobStepInstancesInDbQueue = dbContext.JobStepQueues.Where(d => d.JobInstanceId == jobInstanceInDbQueue.JobInstanceId).ToList();

                if (jobStepInstancesInDbQueue.Count == 0)
                {
                    goto SyncObjectsForJobInstanceHaveChanged;
                }

                List <JobStepInstance> jobStepInstanceObjs = new List <JobStepInstance>();

                foreach (var jobStepInstanceInDbQueue in jobStepInstancesInDbQueue)
                {
                    var jobStep = job.Steps.Where(d => d.Id == jobStepInstanceInDbQueue.JobStepId).FirstOrDefault();

                    if (jobStep == null || !jobStep.IsEnabled)
                    {
                        goto SyncObjectsForJobInstanceHaveChanged;
                    }

                    var jobStepInstanceObj = new JobStepInstance(jobStepInstanceInDbQueue.JobStepInstanceId, jobStep, jobStepInstanceInDbQueue.OrderIndex);

                    jobStepInstanceObj.Status = (JobStepQueueStatus)jobStepInstanceInDbQueue.JobStepQueueStatusId;

                    jobStepInstanceObjs.Add(jobStepInstanceObj);
                }

                var filters = JobFilterHelper.ParseFromDatabaseText(jobInstanceInDbQueue.Filters);

                var invocationSourceType = jobInstanceInDbQueue.IsOnDemand == true ? JobInvocationSourceType.OnDemand : JobInvocationSourceType.Scheduled;

                var queueRequest = new JobQueueRequest(jobInstanceInDbQueue.QueueRequestId, integration, job, invocationSourceType);

                var jobInstanceObj = new JobInstance(queueRequest, jobInstanceInDbQueue.JobInstanceId, integration, job, sourceDataSource, job.TargetDataSource,
                                                     jobStepInstanceObjs, jobInstanceInDbQueue.ScheduledStartTime,
                                                     jobInstanceInDbQueue.InvocationSource, invocationSourceType, filters.ToList());

                queueRequest.JobInstances.Add(jobInstanceObj);

                jobInstanceObj.Status = (JobQueueStatus)jobInstanceInDbQueue.JobQueueStatusId;

                return(jobInstanceObj);

SyncObjectsForJobInstanceHaveChanged:

                if (throwExceptionIfSyncObjectsAreDeletedOrDisabled)
                {
                    throw new Exception(string.Format("Queued job '{0}' in database logger has changed sync objects.", jobInstanceInDbQueue.JobInstanceId));
                }
                else
                {
                    return(null);
                }
            }
        }
コード例 #8
0
        public void MoveToHistoryLog(JobInstance jobInstance)
        {
            if (jobInstance == null)
            {
                throw new Exception("Job instance can not be null.");
            }

            using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
            {
                var now = DateTime.Now;

                var jobHistory = new ls.JobHistory();

                jobHistory.JobInstanceId      = jobInstance.Id;
                jobHistory.QueueRequestId     = jobInstance.QueueRequest.Id;
                jobHistory.JobId              = jobInstance.Job.Id;
                jobHistory.CreatedDate        = now;
                jobHistory.UpdatedDate        = now;
                jobHistory.SourceDataSourceId = jobInstance.SourceDataSource.DataSource.Id;
                jobHistory.TargetDataSourceId = jobInstance.TargetDataSource.DataSource.Id;
                jobHistory.Filters            = JobFilterHelper.GetTextForDatabase(jobInstance.Filters);
                jobHistory.InvocationSource   = jobInstance.InvocationSource;
                jobHistory.ScheduledStartTime = jobInstance.ScheduledStartTime;
                jobHistory.ActualStartTime    = jobInstance.ActualStartTime.HasValue ? jobInstance.ActualStartTime.Value : new DateTime?();

                if (jobInstance.TimeToStartDelay.HasValue && jobInstance.TimeToStartDelay.Value.Hours > 23)
                {
                    jobHistory.TimeToStartDelay = new TimeSpan(23, 59, 59);
                }
                else if (jobInstance.TimeToStartDelay.HasValue)
                {
                    jobHistory.TimeToStartDelay = jobInstance.TimeToStartDelay.Value;
                }
                else
                {
                    jobHistory.TimeToStartDelay = new TimeSpan(0, 0, 0);
                }

                jobHistory.ActualEndTime = jobInstance.ActualEndTime.HasValue ? jobInstance.ActualEndTime.Value : new DateTime?();

                if (jobInstance.ActualDuration.HasValue && jobInstance.ActualDuration.Value.Hours > 23)
                {
                    jobHistory.ActualDuration = new TimeSpan(23, 59, 59);
                }
                else if (jobInstance.ActualDuration.HasValue)
                {
                    jobHistory.ActualDuration = jobInstance.ActualDuration.Value;
                }
                else
                {
                    jobHistory.ActualDuration = new TimeSpan(0, 0, 0);
                }

                jobHistory.IsOnDemand       = jobInstance.InvocationSourceType == JobInvocationSourceType.OnDemand ? true : false;
                jobHistory.JobQueueStatusId = (byte)jobInstance.Status;
                jobHistory.HasRecordErrors  = jobInstance.HasRecordErrors;
                jobHistory.HasRuntimeErrors = jobInstance.HasRuntimeErrors;

                dbContext.JobHistories.InsertOnSubmit(jobHistory);

                dbContext.SubmitChanges();

                foreach (var jobStepInstance in jobInstance.JobStepInstances)
                {
                    var jobStepHistory = new ls.JobStepHistory();

                    jobStepHistory.JobHistoryId      = jobHistory.JobHistoryId;
                    jobStepHistory.JobStepInstanceId = jobStepInstance.Id;
                    jobStepHistory.JobInstanceId     = jobInstance.Id;
                    jobStepHistory.JobStepId         = jobStepInstance.JobStep.Id;
                    jobStepHistory.CreatedDate       = now;
                    jobStepHistory.UpdatedDate       = now;
                    jobStepHistory.ActualStartTime   = jobStepInstance.ActualStartTime.HasValue ? jobStepInstance.ActualStartTime.Value : new DateTime?();
                    jobStepHistory.ActualEndTime     = jobStepInstance.ActualEndTime.HasValue ? jobStepInstance.ActualEndTime.Value : new DateTime?();

                    if (jobStepInstance.ActualDuration.HasValue && jobStepInstance.ActualDuration.Value.Hours > 23)
                    {
                        jobStepHistory.ActualDuration = new TimeSpan(23, 59, 59);
                    }
                    else if (jobStepInstance.ActualDuration.HasValue)
                    {
                        jobStepHistory.ActualDuration = jobStepInstance.ActualDuration.Value;
                    }
                    else
                    {
                        jobStepHistory.ActualDuration = new TimeSpan(0, 0, 0);
                    }

                    jobStepHistory.OrderIndex           = (byte)jobStepInstance.OrderIndex;
                    jobStepHistory.JobStepQueueStatusId = (byte)jobStepInstance.Status;
                    jobStepHistory.HasRecordErrors      = jobStepInstance.HasRecordErrors;
                    jobStepHistory.HasRuntimeErrors     = jobStepInstance.HasRuntimeErrors;

                    dbContext.JobStepHistories.InsertOnSubmit(jobStepHistory);
                }

                dbContext.SubmitChanges();

                DeleteFromQueueLog(jobInstance.Id);
            }
        }
コード例 #9
0
        public static void ExecuteStep(JobInstance jobInstance, JobStepInstance previousJobStepInstance,
                                       JobStepInstance currentJobStepInstance, ISyncEngineConfigurator configurator)
        {
            Type jobStepType = jobInstance.Integration.PackageAssembly.GetType(currentJobStepInstance.JobStep.FullyQualifiedName);

            if (jobStepType == null)
            {
                throw new Exception(string.Format("Job step with fully qualified name '{0}' was not found in assembly '{1}'.", currentJobStepInstance.JobStep.FullyQualifiedName, jobInstance.Integration.PackageAssembly.Location));
            }

            // ensure the step class inherits from the proper base class to ensure the Initialize method is available
            bool hasCorrectBaseType = false;

            var baseType = jobStepType.BaseType;

            while (baseType != typeof(Object))
            {
                if (baseType == typeof(JobStepInvocation))
                {
                    hasCorrectBaseType = true;
                    break;
                }

                baseType = baseType.BaseType;
            }

            if (!hasCorrectBaseType)
            {
                throw new Exception(string.Format("Job step class '{0}' must derive from '{1}'.", jobStepType.Name, typeof(JobStepInvocation).FullName));
            }

            var jobStepInvocation = Activator.CreateInstance(jobStepType);

            if (jobStepInvocation is CustomActionStep)
            {
                var jobStepObj = (CustomActionStep)jobStepInvocation;

                jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);

                jobStepObj.Process();

                if (previousJobStepInstance != null &&
                    ((previousJobStepInstance.HasDeferredExecutionUntilNextStep.HasValue &&
                      previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true) ||
                     (previousJobStepInstance.GetType() is CustomActionStep)))
                {
                    currentJobStepInstance.SourceJobBatch = previousJobStepInstance.SourceJobBatch;
                    currentJobStepInstance.TargetJobBatch = previousJobStepInstance.TargetJobBatch;
                    currentJobStepInstance.HasDeferredExecutionUntilNextStep = true;
                }
                else
                {
                    currentJobStepInstance.HasDeferredExecutionUntilNextStep = false;
                }
            }
            else if (jobStepInvocation is DataMapStep)
            {
                var jobStepObj = (DataMapStep)jobStepInvocation;

                jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);

                var dataMapOutput = jobStepObj.Process();

                if (dataMapOutput == null)
                {
                    throw new Exception("Job step must return a value.");
                }

                currentJobStepInstance.HasDeferredExecutionUntilNextStep = dataMapOutput.DeferExecutionUntilNextStep;

                JobBatch       sourceJobBatch    = null;
                JobBatch       targetJobBatch    = null;
                EntityBatch    sourceEntityBatch = null;
                EntityBatch    targetEntityBatch = null;
                EntityBatch    oneWayEntityBatch = null;
                IOneWayDataMap oneWayDataMap     = null;
                TwoWayDataMap  twoWayDataMap     = null;

                if (dataMapOutput.DataMap is IOneWayDataMap)
                {
                    oneWayDataMap = (IOneWayDataMap)dataMapOutput.DataMap;

                    if (dataMapOutput.DataMap is OneWayDataMap)
                    {
                        oneWayEntityBatch = OneToOneDataMapProcessor.Compare((OneWayDataMap)dataMapOutput.DataMap,
                                                                             dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior,
                                                                             dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior,
                                                                             dataMapOutput.RowsToProcess);
                    }
                    else if (dataMapOutput.DataMap is OneToMany_OneWayDataMap)
                    {
                        if (dataMapOutput.RowsToProcess != null)
                        {
                            throw new Exception("Rows to process Func is not supported for one-to-many data maps.");
                        }

                        oneWayEntityBatch = OneToManyDataMapProcessor.Compare((OneToMany_OneWayDataMap)dataMapOutput.DataMap,
                                                                              dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior,
                                                                              dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior);
                    }

                    if (oneWayEntityBatch.EntityDefinition.SyncSide == SyncSide.Source)
                    {
                        oneWayEntityBatch.LoggingBehavior = dataMapOutput.SourceSideLoggingBehavior;
                    }
                    else if (oneWayEntityBatch.EntityDefinition.SyncSide == SyncSide.Target)
                    {
                        oneWayEntityBatch.LoggingBehavior = dataMapOutput.TargetSideLoggingBehavior;
                    }
                }
                else if (dataMapOutput.DataMap is TwoWayDataMap)
                {
                    twoWayDataMap = (TwoWayDataMap)dataMapOutput.DataMap;

                    OneToOneDataMapProcessor.Compare(twoWayDataMap, dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior,
                                                     dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior,
                                                     dataMapOutput.RowsToProcess, out sourceEntityBatch, out targetEntityBatch);

                    sourceEntityBatch.LoggingBehavior = dataMapOutput.SourceSideLoggingBehavior;
                    targetEntityBatch.LoggingBehavior = dataMapOutput.TargetSideLoggingBehavior;
                }
                else
                {
                    throw new DerivedClassNotImplementedException <OneToOneDataMap>(dataMapOutput.DataMap);
                }

                if (previousJobStepInstance != null &&
                    previousJobStepInstance.HasDeferredExecutionUntilNextStep.HasValue &&
                    previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true)
                {
                    sourceJobBatch = previousJobStepInstance.SourceJobBatch;
                    targetJobBatch = previousJobStepInstance.TargetJobBatch;
                }
                else
                {
                    sourceJobBatch = new JobBatch(SyncSide.Source, jobInstance.SourceDataSource);
                    targetJobBatch = new JobBatch(SyncSide.Target, jobInstance.TargetDataSource);
                }

                if (dataMapOutput.DataMap is IOneWayDataMap)
                {
                    if (oneWayDataMap.SyncDirection == SyncDirection.SourceToTarget)
                    {
                        targetJobBatch.EntityBatches.Add(oneWayEntityBatch);
                    }
                    else if (oneWayDataMap.SyncDirection == SyncDirection.TargetToSource)
                    {
                        sourceJobBatch.EntityBatches.Add(oneWayEntityBatch);
                    }
                    else
                    {
                        throw new EnumValueNotImplementedException <SyncDirection>(oneWayDataMap.SyncDirection);
                    }
                }
                else if (dataMapOutput.DataMap is TwoWayDataMap)
                {
                    sourceJobBatch.EntityBatches.Add(sourceEntityBatch);
                    targetJobBatch.EntityBatches.Add(targetEntityBatch);
                }
                else
                {
                    throw new DerivedClassNotImplementedException <OneToOneDataMap>(dataMapOutput.DataMap);
                }

                currentJobStepInstance.SourceJobBatch = sourceJobBatch;
                currentJobStepInstance.TargetJobBatch = targetJobBatch;

                if (!currentJobStepInstance.HasDeferredExecutionUntilNextStep.Value)
                {
                    sourceJobBatch.SubmitToDataSource();

                    if (sourceJobBatch.HasRecordErrors)
                    {
                        currentJobStepInstance.HasRecordErrors = true;
                        currentJobStepInstance.Exceptions      = sourceJobBatch.GetExceptions();
                    }

                    SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, sourceJobBatch);

                    targetJobBatch.SubmitToDataSource();

                    if (targetJobBatch.HasRecordErrors)
                    {
                        currentJobStepInstance.HasRecordErrors = true;
                        currentJobStepInstance.Exceptions      = targetJobBatch.GetExceptions();
                    }

                    SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, targetJobBatch);
                }
            }
            else if (jobStepInvocation is EntityBatchStep)
            {
                var jobStepObj = (EntityBatchStep)jobStepInvocation;

                jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);

                var entityBatchOutput = jobStepObj.Process();

                if (entityBatchOutput == null)
                {
                    throw new Exception("Job step must return a value.");
                }

                currentJobStepInstance.HasDeferredExecutionUntilNextStep = entityBatchOutput.DeferExecutionUntilNextStep;

                JobBatch sourceJobBatch;
                JobBatch targetJobBatch;

                if (previousJobStepInstance != null && previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true)
                {
                    sourceJobBatch = previousJobStepInstance.SourceJobBatch;
                    targetJobBatch = previousJobStepInstance.TargetJobBatch;
                }
                else
                {
                    sourceJobBatch = new JobBatch(SyncSide.Source, jobInstance.SourceDataSource);
                    targetJobBatch = new JobBatch(SyncSide.Target, jobInstance.TargetDataSource);
                }

                if (entityBatchOutput.SourceSideEntityBatches != null)
                {
                    foreach (var sourceSideEntityBatch in entityBatchOutput.SourceSideEntityBatches)
                    {
                        sourceJobBatch.EntityBatches.Add(sourceSideEntityBatch);
                    }
                }

                if (entityBatchOutput.TargetSideEntityBatches != null)
                {
                    foreach (var targetSideEntityBatch in entityBatchOutput.TargetSideEntityBatches)
                    {
                        targetJobBatch.EntityBatches.Add(targetSideEntityBatch);
                    }
                }

                currentJobStepInstance.SourceJobBatch = sourceJobBatch;
                currentJobStepInstance.TargetJobBatch = targetJobBatch;

                if (!currentJobStepInstance.HasDeferredExecutionUntilNextStep.Value)
                {
                    sourceJobBatch.SubmitToDataSource();

                    if (sourceJobBatch.HasRecordErrors)
                    {
                        currentJobStepInstance.HasRecordErrors = true;
                        currentJobStepInstance.Exceptions      = sourceJobBatch.GetExceptions();
                    }

                    SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, sourceJobBatch);

                    targetJobBatch.SubmitToDataSource();

                    if (targetJobBatch.HasRecordErrors)
                    {
                        currentJobStepInstance.HasRecordErrors = true;
                        currentJobStepInstance.Exceptions      = targetJobBatch.GetExceptions();
                    }

                    SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, targetJobBatch);
                }
            }
            else if (jobStepInvocation is JobBatchStep)
            {
                var jobStepObj = (JobBatchStep)jobStepInvocation;

                jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);

                var jobBatchOutput = jobStepObj.Process();

                if (jobBatchOutput == null)
                {
                    throw new Exception("Job step must return a value.");
                }

                if (jobBatchOutput.SourceSideJobBatch != null)
                {
                    jobBatchOutput.SourceSideJobBatch.SubmitToDataSource();

                    if (jobBatchOutput.SourceSideJobBatch.HasRecordErrors)
                    {
                        currentJobStepInstance.HasRecordErrors = true;
                        currentJobStepInstance.Exceptions      = jobBatchOutput.SourceSideJobBatch.GetExceptions();
                    }

                    SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, jobBatchOutput.SourceSideJobBatch);
                }
                else
                {
                    SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstance, currentJobStepInstance, jobInstance.SourceDataSource.DataSource,
                                                "Job step '{0}' for job '{1}' returned a null job batch for source-side data source '{2}'.",
                                                currentJobStepInstance.JobStep.Name, jobInstance.Job.Name, jobInstance.SourceDataSource.DataSource.Name);
                }

                if (jobBatchOutput.TargetSideJobBatch != null)
                {
                    jobBatchOutput.TargetSideJobBatch.SubmitToDataSource();

                    if (jobBatchOutput.TargetSideJobBatch.HasRecordErrors)
                    {
                        currentJobStepInstance.HasRecordErrors = true;
                        currentJobStepInstance.Exceptions      = jobBatchOutput.TargetSideJobBatch.GetExceptions();
                    }

                    SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, jobBatchOutput.TargetSideJobBatch);
                }
                else
                {
                    SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstance, currentJobStepInstance, jobInstance.TargetDataSource.DataSource,
                                                "Job step '{0}' for job '{1}' returned a null job batch for target-side data source '{2}'.",
                                                currentJobStepInstance.JobStep.Name, jobInstance.Job.Name, jobInstance.TargetDataSource.DataSource.Name);
                }
            }
            else
            {
                throw new DerivedClassNotImplementedException <JobStepInvocation>(jobStepInvocation);
            }
        }
コード例 #10
0
        public JobStepQueueStatusChangedArgs(JobInstance associatedJobInstance, JobStepInstance updatedJobStepInstance)
        {
            AssociatedJobInstance = associatedJobInstance;

            UpdatedJobStepInstance = updatedJobStepInstance;
        }
コード例 #11
0
        public static void ExecuteJob(JobInstance jobInstance, ISyncEngineConfigurator configurator)
        {
            jobInstance.ActualStartTime = DateTime.Now;

            if (jobInstance.MaxDelayedStartExceeded)
            {
                jobInstance.Status = JobQueueStatus.InProgress_MaxDelayExceeded;
            }
            else
            {
                jobInstance.Status = JobQueueStatus.InProgress;
            }

            bool terminateNextStepsOnStepFailure;
            bool isSafeJobStopViaException = false; // used to stop job execution without logging an error

            switch (jobInstance.Job.TerminateOnErrorType)
            {
            case JobTerminateOnErrorType.NeverTerminateOnError:
            case JobTerminateOnErrorType.TerminateExecutingStepOnly:
                terminateNextStepsOnStepFailure = false;
                break;

            case JobTerminateOnErrorType.TerminateExecutingDataSourceOnly:
            case JobTerminateOnErrorType.TerminateJob:
                terminateNextStepsOnStepFailure = true;
                break;

            default:
                throw new EnumValueNotImplementedException <JobTerminateOnErrorType>(jobInstance.Job.TerminateOnErrorType);
            }

            bool previousJobStepHasRuntimeErrors = false;

            JobStepInstance previousJobStepInstance = null;

            foreach (var jobStepInstance in jobInstance.JobStepInstances)
            {
                // handle potential job instances that may have been partially executed prior to machine or service restart
                if (jobStepInstance.Status == JobStepQueueStatus.Completed)
                {
                    continue;
                }
                else if (jobStepInstance.Status == JobStepQueueStatus.Completed_WithError ||
                         jobStepInstance.Status == JobStepQueueStatus.DidNotRun_JobError)
                {
                    previousJobStepHasRuntimeErrors = true;
                    jobInstance.HasRuntimeErrors    = true;
                    continue;
                }
                else if (isSafeJobStopViaException)
                {
                    continue;
                }

                if (previousJobStepHasRuntimeErrors && terminateNextStepsOnStepFailure)
                {
                    jobStepInstance.Status = JobStepQueueStatus.DidNotRun_JobError;
                }
                else
                {
                    jobStepInstance.ActualStartTime = DateTime.Now;

                    jobStepInstance.Status = JobStepQueueStatus.InProgress;

                    jobStepInstance.HasRuntimeErrors = false;

                    try
                    {
                        jobInstance.RunningJobStepInstance = jobStepInstance;

                        JobStepInvoker.ExecuteStep(jobInstance, previousJobStepInstance, jobStepInstance, configurator);

                        jobStepInstance.ActualEndTime = DateTime.Now;

                        if (jobStepInstance.HasRecordErrors)
                        {
                            jobInstance.HasRecordErrors = true;
                        }

                        if (jobStepInstance.HasRuntimeErrors)
                        {
                            jobStepInstance.Status = JobStepQueueStatus.Completed_WithError;

                            jobInstance.HasRuntimeErrors = true;

                            previousJobStepHasRuntimeErrors = true;
                        }
                        else
                        {
                            jobStepInstance.Status = JobStepQueueStatus.Completed;
                        }
                    }
                    catch (SafeJobStopException ex)
                    {
                        jobStepInstance.ActualEndTime = DateTime.Now;

                        isSafeJobStopViaException = true;

                        SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Info, () =>
                        {
                            return(string.Format("Job step '{0}' of job '{1}' was terminated safely with message: {2}",
                                                 jobStepInstance.JobStep.Name, jobStepInstance.AssociatedJobInstance.Job.Name, ex.Message));
                        });

                        continue;
                    }
                    catch (Exception ex)
                    {
                        jobStepInstance.ActualEndTime = DateTime.Now;

                        jobStepInstance.HasRuntimeErrors = true;

                        jobStepInstance.Status = JobStepQueueStatus.Completed_WithError;

                        jobInstance.HasRuntimeErrors = true;

                        jobInstance.Exceptions.Add(ex);

                        previousJobStepHasRuntimeErrors = true;

                        SyncEngineLogger.WriteByParallelTaskContext(ex);

                        jobInstance.RunningJobStepInstance = null;

                        continue;
                    }
                }

                previousJobStepInstance = jobStepInstance;
            }

            jobInstance.ActualEndTime = jobInstance.JobStepInstances.Select(d => d.ActualEndTime).Max();

            jobInstance.RunningJobStepInstance = null;

            if (jobInstance.HasRuntimeErrors && terminateNextStepsOnStepFailure)
            {
                jobInstance.Status = JobQueueStatus.Terminated_WithError;
            }
            else if (jobInstance.HasRuntimeErrors)
            {
                jobInstance.Status = JobQueueStatus.Completed_WithError;
            }
            else
            {
                jobInstance.Status = JobQueueStatus.Completed;
            }
        }
コード例 #12
0
ファイル: JobDequeuedArgs.cs プロジェクト: cburriss/SyncObjX
 public JobDequeuedArgs(JobInstance dequeuedJobInstance)
 {
     DequeuedJobInstance = dequeuedJobInstance;
 }
コード例 #13
0
ファイル: JobQueueManager.cs プロジェクト: cburriss/SyncObjX
        public static List <JobInstance> QueueJob(Integration integration, IEnumerable <DataSource> sourceSideDataSourcesToRun, Job jobToRun,
                                                  DateTime scheduledStartTime, string invocationSource, JobInvocationSourceType invocationSourceType, List <JobFilter> filters = null)
        {
            if (integration == null)
            {
                throw new Exception("Integration is missing or empty.");
            }

            if (sourceSideDataSourcesToRun == null || sourceSideDataSourcesToRun.Where(d => d != null).Count() == 0)
            {
                throw new Exception(string.Format("At least one source-side data source is required to queue job '{0}'.", jobToRun.Name));
            }

            foreach (var dataSource in sourceSideDataSourcesToRun)
            {
                if (!jobToRun.SourceDataSources.ContainsKey(dataSource.Name))
                {
                    throw new Exception(string.Format("Source-side data source '{0}' does not exist for job '{1}'. This job will not be queued for execution.", dataSource.Name, jobToRun.Name));
                }
            }

            if (String.IsNullOrWhiteSpace(invocationSource))
            {
                throw new Exception("Invocation source is missing or empty.");
            }

            if (!integration.IsEnabled)
            {
                throw new Exception(string.Format("Job '{0}' could not be queued for execution because integration '{1}' is disabled.", jobToRun.Name, integration.Name));
            }

            if (!jobToRun.IsEnabled)
            {
                throw new Exception(string.Format("Job '{0}' could not be queued for execution because it is disabled.", jobToRun.Name));
            }

            if (jobToRun.Steps.Where(d => d.IsEnabled).Count() == 0)
            {
                throw new Exception(string.Format("Job '{0}' could not be queued for execution because no job steps exist or all are disabled.", jobToRun.Name));
            }

            SetMaxConcurrentThreadsForIntegration(integration);

            List <JobInstance> queuedJobs = new List <JobInstance>();

            JobInstance            jobInstance;
            JobStepInstance        jobStepInstance;
            List <JobStepInstance> jobStepInstances;

            var queueRequest = new JobQueueRequest(integration, jobToRun, invocationSourceType);

            queueRequest.StatusChanged += new EventHandler <JobQueueRequestStatusChangedArgs>(HandleChangedQueueRequestStatus);

            queueRequest.Status = JobQueueRequestStatus.Waiting;

            foreach (var sourceSideDataSource in sourceSideDataSourcesToRun)
            {
                byte jobStepIdx = 0;

                jobStepInstances = new List <JobStepInstance>();

                foreach (var jobStep in jobToRun.Steps)
                {
                    if (jobStep.IsEnabled)
                    {
                        jobStepInstance = new JobStepInstance(jobStep, jobStepIdx);

                        jobStepInstance.StatusChanged += new EventHandler <JobStepQueueStatusChangedArgs>(HandleChangedJobStepStatus);

                        jobStepInstances.Add(jobStepInstance);

                        //jobStepInstance.Status = JobStepQueueStatus.Waiting;

                        jobStepIdx++;
                    }
                }

                jobInstance = new JobInstance(queueRequest, integration, jobToRun, sourceSideDataSource, jobToRun.TargetDataSource, jobStepInstances,
                                              scheduledStartTime, invocationSource, invocationSourceType, filters);

                queueRequest.JobInstances.Add(jobInstance);

                // change the status here to avoid the job step not having an association with the job instance when status changes are logged
                foreach (var jobStepInstanceToFireStatusChangeFor in jobStepInstances)
                {
                    jobStepInstanceToFireStatusChangeFor.Status = JobStepQueueStatus.Waiting;
                }

                jobInstance.StatusChanged += new EventHandler <JobQueueStatusChangedArgs>(HandleChangedJobStatus);

                if (waitingJobsByIntegrationId.ContainsKey(jobInstance.Integration.Id))
                {
                    waitingJobsByIntegrationId[jobInstance.Integration.Id].Add(jobInstance);
                }
                else
                {
                    waitingJobsByIntegrationId.Add(jobInstance.Integration.Id, new List <JobInstance>()
                    {
                        jobInstance
                    });
                }

                // cache the job instance in memory if on-demand (temporary to get around OutOfMemoryException)
                if (jobInstance.InvocationSourceType == JobInvocationSourceType.OnDemand)
                {
                    jobInstancesById.Add(jobInstance.Id, jobInstance);
                    queueRequestsById.Add(queueRequest.Id, queueRequest);
                }

                jobInstance.Status = JobQueueStatus.Scheduled;

                queuedJobs.Add(jobInstance);

                if (JobQueued != null)
                {
                    JobQueued(null, new JobQueuedArgs(jobInstance));
                }
            }

            return(queuedJobs);
        }
コード例 #14
0
ファイル: JobQueuedArgs.cs プロジェクト: cburriss/SyncObjX
 public JobQueuedArgs(JobInstance queuedJobInstance)
 {
     QueuedJobInstance = queuedJobInstance;
 }
コード例 #15
0
 public JobQueueStatusChangedArgs(JobInstance updatedJobInstance)
 {
     UpdatedJobInstance = updatedJobInstance;
 }