public static void QueueJob(JobInstance jobInstance)
{
JobInstance existingJobInstance;
if (runningJobsByIntegrationId.ContainsKey(jobInstance.Integration.Id))
{
existingJobInstance = runningJobsByIntegrationId[jobInstance.Integration.Id].Where(d => d.Id == jobInstance.Id).FirstOrDefault();
if (existingJobInstance != null)
{
throw new Exception(string.Format("A job instance with Id '{0}' is already running.", jobInstance.Id));
}
}
if (waitingJobsByIntegrationId.ContainsKey(jobInstance.Integration.Id))
{
existingJobInstance = waitingJobsByIntegrationId[jobInstance.Integration.Id].Where(d => d.Id == jobInstance.Id).FirstOrDefault();
if (existingJobInstance != null)
{
throw new Exception(string.Format("A job instance with Id '{0}' is already waiting for execution.", jobInstance.Id));
}
waitingJobsByIntegrationId[jobInstance.Integration.Id].Add(jobInstance);
}
else
{
SetMaxConcurrentThreadsForIntegration(jobInstance.Integration);
waitingJobsByIntegrationId.Add(jobInstance.Integration.Id, new List <JobInstance>()
{
jobInstance
});
}
// cache the job instance in memory if on-demand (temporary to get around OutOfMemoryException)
if (jobInstance.InvocationSourceType == JobInvocationSourceType.OnDemand)
{
jobInstancesById.Add(jobInstance.Id, jobInstance);
queueRequestsById.Add(jobInstance.QueueRequest.Id, jobInstance.QueueRequest);
}
jobInstance.StatusChanged += new EventHandler <JobQueueStatusChangedArgs>(HandleChangedJobStatus);
foreach (var jobStepInstance in jobInstance.JobStepInstances)
{
jobStepInstance.StatusChanged += new EventHandler <JobStepQueueStatusChangedArgs>(HandleChangedJobStepStatus);
}
jobInstance.QueueRequest.Status = JobQueueRequestStatus.Waiting;
jobInstance.Status = JobQueueStatus.Scheduled;
jobInstance.RunningJobStepInstance = null;
if (JobQueued != null)
{
JobQueued(null, new JobQueuedArgs(jobInstance));
}
}
public void Initialize(Integration integration, JobInstance jobInstance,
JobStepInstance previousJobStepInstance, JobStepInstance jobStepInstance, ISyncEngineConfigurator configurator)
{
Integration = integration;
JobInstance = jobInstance;
PreviousJobStepInstance = previousJobStepInstance;
JobStepInstance = jobStepInstance;
Configurator = configurator;
}
public void AddToQueueLog(JobInstance jobInstance)
{
if (jobInstance == null)
{
throw new Exception("Job instance can not be null.");
}
using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
{
var now = DateTime.Now;
var queuedJobInstance = new ls.JobQueue();
queuedJobInstance.JobInstanceId = jobInstance.Id;
queuedJobInstance.QueueRequestId = jobInstance.QueueRequest.Id;
queuedJobInstance.JobId = jobInstance.Job.Id;
queuedJobInstance.CreatedDate = now;
queuedJobInstance.UpdatedDate = now;
queuedJobInstance.SourceDataSourceId = jobInstance.SourceDataSource.DataSource.Id;
queuedJobInstance.TargetDataSourceId = jobInstance.TargetDataSource.DataSource.Id;
queuedJobInstance.Filters = JobFilterHelper.GetTextForDatabase(jobInstance.Filters);
queuedJobInstance.InvocationSource = jobInstance.InvocationSource;
queuedJobInstance.ScheduledStartTime = jobInstance.ScheduledStartTime;
queuedJobInstance.ActualStartTime = jobInstance.ActualStartTime.HasValue ? jobInstance.ActualStartTime.Value : new DateTime?();
queuedJobInstance.IsOnDemand = jobInstance.InvocationSourceType == JobInvocationSourceType.OnDemand ? true : false;
queuedJobInstance.JobQueueStatusId = (byte)jobInstance.Status;
dbContext.JobQueues.InsertOnSubmit(queuedJobInstance);
dbContext.SubmitChanges();
foreach (var jobStepInstance in jobInstance.JobStepInstances)
{
var queuedJobStepInstance = new ls.JobStepQueue();
queuedJobStepInstance.JobStepInstanceId = jobStepInstance.Id;
queuedJobStepInstance.JobInstanceId = jobInstance.Id;
queuedJobStepInstance.JobStepId = jobStepInstance.JobStep.Id;
queuedJobStepInstance.CreatedDate = now;
queuedJobStepInstance.UpdatedDate = now;
queuedJobStepInstance.ActualStartTime = jobStepInstance.ActualStartTime.HasValue ? jobStepInstance.ActualStartTime.Value : new DateTime?();
queuedJobStepInstance.ActualEndTime = jobStepInstance.ActualEndTime.HasValue ? jobStepInstance.ActualEndTime.Value : new DateTime?();
queuedJobStepInstance.OrderIndex = (byte)jobStepInstance.OrderIndex;
queuedJobStepInstance.JobStepQueueStatusId = (byte)jobStepInstance.Status;
dbContext.JobStepQueues.InsertOnSubmit(queuedJobStepInstance);
}
dbContext.SubmitChanges();
}
}
public static JobInstanceResult GetResult(JobInstance jobInstance)
{
if (jobInstance == null)
{
return(null);
}
object _lock = new object();
lock (_lock)
{
var jobInstanceResult = new JobInstanceResult();
jobInstanceResult.IntegrationId = jobInstance.Integration.Id;
jobInstanceResult.JobId = jobInstance.Job.Id;
jobInstanceResult.QueueRequestId = jobInstance.QueueRequest.Id;
jobInstanceResult.JobInstanceId = jobInstance.Id;
jobInstanceResult.Filters = jobInstance.Filters;
jobInstanceResult.SourceDataSourceId = jobInstance.SourceDataSource.DataSource.Id;
jobInstanceResult.TargetDataSourceId = jobInstance.TargetDataSource.DataSource.Id;
jobInstanceResult.InvocationSource = jobInstance.InvocationSource;
jobInstanceResult.InvocationSourceType = jobInstance.InvocationSourceType;
jobInstanceResult.Status = jobInstance.Status;
jobInstanceResult.ScheduledStartTime = jobInstance.ScheduledStartTime;
jobInstanceResult.TimeToStartDelay = jobInstance.TimeToStartDelay;
jobInstanceResult.ActualStartTime = jobInstance.ActualStartTime;
jobInstanceResult.ActualEndTime = jobInstance.ActualEndTime;
jobInstanceResult.ActualDuration = jobInstance.ActualDuration;
jobInstanceResult.HasRecordErrors = jobInstance.HasRecordErrors;
jobInstanceResult.HasRuntimeErrors = jobInstance.HasRuntimeErrors;
jobInstanceResult.Exceptions = WebServiceException.Convert(jobInstance.Exceptions);
var jobStepResults = new List <JobStepInstanceResult>();
foreach (var jobStepInstance in jobInstance.JobStepInstances)
{
jobStepResults.Add(GetResult(jobStepInstance));
}
jobInstanceResult.JobStepInstanceResults = jobStepResults;
return(jobInstanceResult);
}
}
public void UpdateJobStatusInQueueLog(JobInstance jobInstance)
{
if (jobInstance == null)
{
throw new Exception("Job instance can not be null.");
}
using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
{
var queuedJobInstance = dbContext.JobQueues.Where(d => d.JobInstanceId == jobInstance.Id).FirstOrDefault();
if (queuedJobInstance != null)
{
queuedJobInstance.UpdatedDate = DateTime.Now;
queuedJobInstance.ActualStartTime = jobInstance.ActualStartTime.HasValue ? jobInstance.ActualStartTime.Value : new DateTime?();
queuedJobInstance.JobQueueStatusId = (byte)jobInstance.Status;
dbContext.SubmitChanges();
}
}
}
public void UpdateJobDataSourceHistory(JobInstance jobInstance)
{
if (jobInstance == null)
{
throw new Exception("Job instance can not be null.");
}
if (!jobInstance.ActualStartTime.HasValue)
{
throw new Exception("Job instance is missing an actual start time.");
}
if (!jobInstance.ActualEndTime.HasValue)
{
throw new Exception("Job instance is missing an actual end time.");
}
using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
{
dbContext.UpdateJobDataSourceHistory(jobInstance.Job.Id, jobInstance.SourceDataSource.DataSource.Id, jobInstance.TargetDataSource.DataSource.Id,
jobInstance.QueueRequest.Id, jobInstance.Id, jobInstance.ActualStartTime.Value, jobInstance.ActualEndTime.Value,
jobInstance.HasRecordErrors, jobInstance.HasRuntimeErrors);
}
}
/// <summary>
///
/// </summary>
/// <param name="jobInstanceId"></param>
/// <returns>Returns null if a "sync object" no longer exists.</returns>
public JobInstance GetJobInstanceFromQueueLog(Guid jobInstanceId, bool throwExceptionIfSyncObjectsAreDeletedOrDisabled = false)
{
if (!JobQueueManager.IsSafeForQueueLogRetrieval)
{
throw new Exception("The Job Queue Manager is in an unsafe state as job statuses may change. Queue retrieval from the database is disallowed.");
}
var configurator = new SyncEngineDatabaseConfigurator(_connectionString);
using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
{
var jobInstanceInDbQueue = dbContext.JobQueues.Where(d => d.JobInstanceId == jobInstanceId).FirstOrDefault();
var integration = configurator.GetIntegrationByJobId(jobInstanceInDbQueue.JobId);
if (integration == null || !integration.IsEnabled)
{
goto SyncObjectsForJobInstanceHaveChanged;
}
var job = integration.Jobs.Where(d => d.Id == jobInstanceInDbQueue.JobId).FirstOrDefault();
if (job == null || !job.IsEnabled)
{
goto SyncObjectsForJobInstanceHaveChanged;
}
var sourceDataSource = job.SourceDataSources.Values.Where(d => d.Id == jobInstanceInDbQueue.SourceDataSourceId).FirstOrDefault();
if (sourceDataSource == null)
{
goto SyncObjectsForJobInstanceHaveChanged;
}
if (job.TargetDataSource.Id != jobInstanceInDbQueue.TargetDataSourceId)
{
goto SyncObjectsForJobInstanceHaveChanged;
}
var jobStepInstancesInDbQueue = dbContext.JobStepQueues.Where(d => d.JobInstanceId == jobInstanceInDbQueue.JobInstanceId).ToList();
if (jobStepInstancesInDbQueue.Count == 0)
{
goto SyncObjectsForJobInstanceHaveChanged;
}
List <JobStepInstance> jobStepInstanceObjs = new List <JobStepInstance>();
foreach (var jobStepInstanceInDbQueue in jobStepInstancesInDbQueue)
{
var jobStep = job.Steps.Where(d => d.Id == jobStepInstanceInDbQueue.JobStepId).FirstOrDefault();
if (jobStep == null || !jobStep.IsEnabled)
{
goto SyncObjectsForJobInstanceHaveChanged;
}
var jobStepInstanceObj = new JobStepInstance(jobStepInstanceInDbQueue.JobStepInstanceId, jobStep, jobStepInstanceInDbQueue.OrderIndex);
jobStepInstanceObj.Status = (JobStepQueueStatus)jobStepInstanceInDbQueue.JobStepQueueStatusId;
jobStepInstanceObjs.Add(jobStepInstanceObj);
}
var filters = JobFilterHelper.ParseFromDatabaseText(jobInstanceInDbQueue.Filters);
var invocationSourceType = jobInstanceInDbQueue.IsOnDemand == true ? JobInvocationSourceType.OnDemand : JobInvocationSourceType.Scheduled;
var queueRequest = new JobQueueRequest(jobInstanceInDbQueue.QueueRequestId, integration, job, invocationSourceType);
var jobInstanceObj = new JobInstance(queueRequest, jobInstanceInDbQueue.JobInstanceId, integration, job, sourceDataSource, job.TargetDataSource,
jobStepInstanceObjs, jobInstanceInDbQueue.ScheduledStartTime,
jobInstanceInDbQueue.InvocationSource, invocationSourceType, filters.ToList());
queueRequest.JobInstances.Add(jobInstanceObj);
jobInstanceObj.Status = (JobQueueStatus)jobInstanceInDbQueue.JobQueueStatusId;
return(jobInstanceObj);
SyncObjectsForJobInstanceHaveChanged:
if (throwExceptionIfSyncObjectsAreDeletedOrDisabled)
{
throw new Exception(string.Format("Queued job '{0}' in database logger has changed sync objects.", jobInstanceInDbQueue.JobInstanceId));
}
else
{
return(null);
}
}
}
public void MoveToHistoryLog(JobInstance jobInstance)
{
if (jobInstance == null)
{
throw new Exception("Job instance can not be null.");
}
using (var dbContext = new ls.QueueManagementDataContext(_connectionString))
{
var now = DateTime.Now;
var jobHistory = new ls.JobHistory();
jobHistory.JobInstanceId = jobInstance.Id;
jobHistory.QueueRequestId = jobInstance.QueueRequest.Id;
jobHistory.JobId = jobInstance.Job.Id;
jobHistory.CreatedDate = now;
jobHistory.UpdatedDate = now;
jobHistory.SourceDataSourceId = jobInstance.SourceDataSource.DataSource.Id;
jobHistory.TargetDataSourceId = jobInstance.TargetDataSource.DataSource.Id;
jobHistory.Filters = JobFilterHelper.GetTextForDatabase(jobInstance.Filters);
jobHistory.InvocationSource = jobInstance.InvocationSource;
jobHistory.ScheduledStartTime = jobInstance.ScheduledStartTime;
jobHistory.ActualStartTime = jobInstance.ActualStartTime.HasValue ? jobInstance.ActualStartTime.Value : new DateTime?();
if (jobInstance.TimeToStartDelay.HasValue && jobInstance.TimeToStartDelay.Value.Hours > 23)
{
jobHistory.TimeToStartDelay = new TimeSpan(23, 59, 59);
}
else if (jobInstance.TimeToStartDelay.HasValue)
{
jobHistory.TimeToStartDelay = jobInstance.TimeToStartDelay.Value;
}
else
{
jobHistory.TimeToStartDelay = new TimeSpan(0, 0, 0);
}
jobHistory.ActualEndTime = jobInstance.ActualEndTime.HasValue ? jobInstance.ActualEndTime.Value : new DateTime?();
if (jobInstance.ActualDuration.HasValue && jobInstance.ActualDuration.Value.Hours > 23)
{
jobHistory.ActualDuration = new TimeSpan(23, 59, 59);
}
else if (jobInstance.ActualDuration.HasValue)
{
jobHistory.ActualDuration = jobInstance.ActualDuration.Value;
}
else
{
jobHistory.ActualDuration = new TimeSpan(0, 0, 0);
}
jobHistory.IsOnDemand = jobInstance.InvocationSourceType == JobInvocationSourceType.OnDemand ? true : false;
jobHistory.JobQueueStatusId = (byte)jobInstance.Status;
jobHistory.HasRecordErrors = jobInstance.HasRecordErrors;
jobHistory.HasRuntimeErrors = jobInstance.HasRuntimeErrors;
dbContext.JobHistories.InsertOnSubmit(jobHistory);
dbContext.SubmitChanges();
foreach (var jobStepInstance in jobInstance.JobStepInstances)
{
var jobStepHistory = new ls.JobStepHistory();
jobStepHistory.JobHistoryId = jobHistory.JobHistoryId;
jobStepHistory.JobStepInstanceId = jobStepInstance.Id;
jobStepHistory.JobInstanceId = jobInstance.Id;
jobStepHistory.JobStepId = jobStepInstance.JobStep.Id;
jobStepHistory.CreatedDate = now;
jobStepHistory.UpdatedDate = now;
jobStepHistory.ActualStartTime = jobStepInstance.ActualStartTime.HasValue ? jobStepInstance.ActualStartTime.Value : new DateTime?();
jobStepHistory.ActualEndTime = jobStepInstance.ActualEndTime.HasValue ? jobStepInstance.ActualEndTime.Value : new DateTime?();
if (jobStepInstance.ActualDuration.HasValue && jobStepInstance.ActualDuration.Value.Hours > 23)
{
jobStepHistory.ActualDuration = new TimeSpan(23, 59, 59);
}
else if (jobStepInstance.ActualDuration.HasValue)
{
jobStepHistory.ActualDuration = jobStepInstance.ActualDuration.Value;
}
else
{
jobStepHistory.ActualDuration = new TimeSpan(0, 0, 0);
}
jobStepHistory.OrderIndex = (byte)jobStepInstance.OrderIndex;
jobStepHistory.JobStepQueueStatusId = (byte)jobStepInstance.Status;
jobStepHistory.HasRecordErrors = jobStepInstance.HasRecordErrors;
jobStepHistory.HasRuntimeErrors = jobStepInstance.HasRuntimeErrors;
dbContext.JobStepHistories.InsertOnSubmit(jobStepHistory);
}
dbContext.SubmitChanges();
DeleteFromQueueLog(jobInstance.Id);
}
}
public static void ExecuteStep(JobInstance jobInstance, JobStepInstance previousJobStepInstance,
JobStepInstance currentJobStepInstance, ISyncEngineConfigurator configurator)
{
Type jobStepType = jobInstance.Integration.PackageAssembly.GetType(currentJobStepInstance.JobStep.FullyQualifiedName);
if (jobStepType == null)
{
throw new Exception(string.Format("Job step with fully qualified name '{0}' was not found in assembly '{1}'.", currentJobStepInstance.JobStep.FullyQualifiedName, jobInstance.Integration.PackageAssembly.Location));
}
// ensure the step class inherits from the proper base class to ensure the Initialize method is available
bool hasCorrectBaseType = false;
var baseType = jobStepType.BaseType;
while (baseType != typeof(Object))
{
if (baseType == typeof(JobStepInvocation))
{
hasCorrectBaseType = true;
break;
}
baseType = baseType.BaseType;
}
if (!hasCorrectBaseType)
{
throw new Exception(string.Format("Job step class '{0}' must derive from '{1}'.", jobStepType.Name, typeof(JobStepInvocation).FullName));
}
var jobStepInvocation = Activator.CreateInstance(jobStepType);
if (jobStepInvocation is CustomActionStep)
{
var jobStepObj = (CustomActionStep)jobStepInvocation;
jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);
jobStepObj.Process();
if (previousJobStepInstance != null &&
((previousJobStepInstance.HasDeferredExecutionUntilNextStep.HasValue &&
previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true) ||
(previousJobStepInstance.GetType() is CustomActionStep)))
{
currentJobStepInstance.SourceJobBatch = previousJobStepInstance.SourceJobBatch;
currentJobStepInstance.TargetJobBatch = previousJobStepInstance.TargetJobBatch;
currentJobStepInstance.HasDeferredExecutionUntilNextStep = true;
}
else
{
currentJobStepInstance.HasDeferredExecutionUntilNextStep = false;
}
}
else if (jobStepInvocation is DataMapStep)
{
var jobStepObj = (DataMapStep)jobStepInvocation;
jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);
var dataMapOutput = jobStepObj.Process();
if (dataMapOutput == null)
{
throw new Exception("Job step must return a value.");
}
currentJobStepInstance.HasDeferredExecutionUntilNextStep = dataMapOutput.DeferExecutionUntilNextStep;
JobBatch sourceJobBatch = null;
JobBatch targetJobBatch = null;
EntityBatch sourceEntityBatch = null;
EntityBatch targetEntityBatch = null;
EntityBatch oneWayEntityBatch = null;
IOneWayDataMap oneWayDataMap = null;
TwoWayDataMap twoWayDataMap = null;
if (dataMapOutput.DataMap is IOneWayDataMap)
{
oneWayDataMap = (IOneWayDataMap)dataMapOutput.DataMap;
if (dataMapOutput.DataMap is OneWayDataMap)
{
oneWayEntityBatch = OneToOneDataMapProcessor.Compare((OneWayDataMap)dataMapOutput.DataMap,
dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior,
dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior,
dataMapOutput.RowsToProcess);
}
else if (dataMapOutput.DataMap is OneToMany_OneWayDataMap)
{
if (dataMapOutput.RowsToProcess != null)
{
throw new Exception("Rows to process Func is not supported for one-to-many data maps.");
}
oneWayEntityBatch = OneToManyDataMapProcessor.Compare((OneToMany_OneWayDataMap)dataMapOutput.DataMap,
dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior,
dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior);
}
if (oneWayEntityBatch.EntityDefinition.SyncSide == SyncSide.Source)
{
oneWayEntityBatch.LoggingBehavior = dataMapOutput.SourceSideLoggingBehavior;
}
else if (oneWayEntityBatch.EntityDefinition.SyncSide == SyncSide.Target)
{
oneWayEntityBatch.LoggingBehavior = dataMapOutput.TargetSideLoggingBehavior;
}
}
else if (dataMapOutput.DataMap is TwoWayDataMap)
{
twoWayDataMap = (TwoWayDataMap)dataMapOutput.DataMap;
OneToOneDataMapProcessor.Compare(twoWayDataMap, dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior,
dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior,
dataMapOutput.RowsToProcess, out sourceEntityBatch, out targetEntityBatch);
sourceEntityBatch.LoggingBehavior = dataMapOutput.SourceSideLoggingBehavior;
targetEntityBatch.LoggingBehavior = dataMapOutput.TargetSideLoggingBehavior;
}
else
{
throw new DerivedClassNotImplementedException <OneToOneDataMap>(dataMapOutput.DataMap);
}
if (previousJobStepInstance != null &&
previousJobStepInstance.HasDeferredExecutionUntilNextStep.HasValue &&
previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true)
{
sourceJobBatch = previousJobStepInstance.SourceJobBatch;
targetJobBatch = previousJobStepInstance.TargetJobBatch;
}
else
{
sourceJobBatch = new JobBatch(SyncSide.Source, jobInstance.SourceDataSource);
targetJobBatch = new JobBatch(SyncSide.Target, jobInstance.TargetDataSource);
}
if (dataMapOutput.DataMap is IOneWayDataMap)
{
if (oneWayDataMap.SyncDirection == SyncDirection.SourceToTarget)
{
targetJobBatch.EntityBatches.Add(oneWayEntityBatch);
}
else if (oneWayDataMap.SyncDirection == SyncDirection.TargetToSource)
{
sourceJobBatch.EntityBatches.Add(oneWayEntityBatch);
}
else
{
throw new EnumValueNotImplementedException <SyncDirection>(oneWayDataMap.SyncDirection);
}
}
else if (dataMapOutput.DataMap is TwoWayDataMap)
{
sourceJobBatch.EntityBatches.Add(sourceEntityBatch);
targetJobBatch.EntityBatches.Add(targetEntityBatch);
}
else
{
throw new DerivedClassNotImplementedException <OneToOneDataMap>(dataMapOutput.DataMap);
}
currentJobStepInstance.SourceJobBatch = sourceJobBatch;
currentJobStepInstance.TargetJobBatch = targetJobBatch;
if (!currentJobStepInstance.HasDeferredExecutionUntilNextStep.Value)
{
sourceJobBatch.SubmitToDataSource();
if (sourceJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = sourceJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, sourceJobBatch);
targetJobBatch.SubmitToDataSource();
if (targetJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = targetJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, targetJobBatch);
}
}
else if (jobStepInvocation is EntityBatchStep)
{
var jobStepObj = (EntityBatchStep)jobStepInvocation;
jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);
var entityBatchOutput = jobStepObj.Process();
if (entityBatchOutput == null)
{
throw new Exception("Job step must return a value.");
}
currentJobStepInstance.HasDeferredExecutionUntilNextStep = entityBatchOutput.DeferExecutionUntilNextStep;
JobBatch sourceJobBatch;
JobBatch targetJobBatch;
if (previousJobStepInstance != null && previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true)
{
sourceJobBatch = previousJobStepInstance.SourceJobBatch;
targetJobBatch = previousJobStepInstance.TargetJobBatch;
}
else
{
sourceJobBatch = new JobBatch(SyncSide.Source, jobInstance.SourceDataSource);
targetJobBatch = new JobBatch(SyncSide.Target, jobInstance.TargetDataSource);
}
if (entityBatchOutput.SourceSideEntityBatches != null)
{
foreach (var sourceSideEntityBatch in entityBatchOutput.SourceSideEntityBatches)
{
sourceJobBatch.EntityBatches.Add(sourceSideEntityBatch);
}
}
if (entityBatchOutput.TargetSideEntityBatches != null)
{
foreach (var targetSideEntityBatch in entityBatchOutput.TargetSideEntityBatches)
{
targetJobBatch.EntityBatches.Add(targetSideEntityBatch);
}
}
currentJobStepInstance.SourceJobBatch = sourceJobBatch;
currentJobStepInstance.TargetJobBatch = targetJobBatch;
if (!currentJobStepInstance.HasDeferredExecutionUntilNextStep.Value)
{
sourceJobBatch.SubmitToDataSource();
if (sourceJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = sourceJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, sourceJobBatch);
targetJobBatch.SubmitToDataSource();
if (targetJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = targetJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, targetJobBatch);
}
}
else if (jobStepInvocation is JobBatchStep)
{
var jobStepObj = (JobBatchStep)jobStepInvocation;
jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);
var jobBatchOutput = jobStepObj.Process();
if (jobBatchOutput == null)
{
throw new Exception("Job step must return a value.");
}
if (jobBatchOutput.SourceSideJobBatch != null)
{
jobBatchOutput.SourceSideJobBatch.SubmitToDataSource();
if (jobBatchOutput.SourceSideJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = jobBatchOutput.SourceSideJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, jobBatchOutput.SourceSideJobBatch);
}
else
{
SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstance, currentJobStepInstance, jobInstance.SourceDataSource.DataSource,
"Job step '{0}' for job '{1}' returned a null job batch for source-side data source '{2}'.",
currentJobStepInstance.JobStep.Name, jobInstance.Job.Name, jobInstance.SourceDataSource.DataSource.Name);
}
if (jobBatchOutput.TargetSideJobBatch != null)
{
jobBatchOutput.TargetSideJobBatch.SubmitToDataSource();
if (jobBatchOutput.TargetSideJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = jobBatchOutput.TargetSideJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, jobBatchOutput.TargetSideJobBatch);
}
else
{
SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstance, currentJobStepInstance, jobInstance.TargetDataSource.DataSource,
"Job step '{0}' for job '{1}' returned a null job batch for target-side data source '{2}'.",
currentJobStepInstance.JobStep.Name, jobInstance.Job.Name, jobInstance.TargetDataSource.DataSource.Name);
}
}
else
{
throw new DerivedClassNotImplementedException <JobStepInvocation>(jobStepInvocation);
}
}
public JobStepQueueStatusChangedArgs(JobInstance associatedJobInstance, JobStepInstance updatedJobStepInstance)
{
AssociatedJobInstance = associatedJobInstance;
UpdatedJobStepInstance = updatedJobStepInstance;
}
public static void ExecuteJob(JobInstance jobInstance, ISyncEngineConfigurator configurator)
{
jobInstance.ActualStartTime = DateTime.Now;
if (jobInstance.MaxDelayedStartExceeded)
{
jobInstance.Status = JobQueueStatus.InProgress_MaxDelayExceeded;
}
else
{
jobInstance.Status = JobQueueStatus.InProgress;
}
bool terminateNextStepsOnStepFailure;
bool isSafeJobStopViaException = false; // used to stop job execution without logging an error
switch (jobInstance.Job.TerminateOnErrorType)
{
case JobTerminateOnErrorType.NeverTerminateOnError:
case JobTerminateOnErrorType.TerminateExecutingStepOnly:
terminateNextStepsOnStepFailure = false;
break;
case JobTerminateOnErrorType.TerminateExecutingDataSourceOnly:
case JobTerminateOnErrorType.TerminateJob:
terminateNextStepsOnStepFailure = true;
break;
default:
throw new EnumValueNotImplementedException <JobTerminateOnErrorType>(jobInstance.Job.TerminateOnErrorType);
}
bool previousJobStepHasRuntimeErrors = false;
JobStepInstance previousJobStepInstance = null;
foreach (var jobStepInstance in jobInstance.JobStepInstances)
{
// handle potential job instances that may have been partially executed prior to machine or service restart
if (jobStepInstance.Status == JobStepQueueStatus.Completed)
{
continue;
}
else if (jobStepInstance.Status == JobStepQueueStatus.Completed_WithError ||
jobStepInstance.Status == JobStepQueueStatus.DidNotRun_JobError)
{
previousJobStepHasRuntimeErrors = true;
jobInstance.HasRuntimeErrors = true;
continue;
}
else if (isSafeJobStopViaException)
{
continue;
}
if (previousJobStepHasRuntimeErrors && terminateNextStepsOnStepFailure)
{
jobStepInstance.Status = JobStepQueueStatus.DidNotRun_JobError;
}
else
{
jobStepInstance.ActualStartTime = DateTime.Now;
jobStepInstance.Status = JobStepQueueStatus.InProgress;
jobStepInstance.HasRuntimeErrors = false;
try
{
jobInstance.RunningJobStepInstance = jobStepInstance;
JobStepInvoker.ExecuteStep(jobInstance, previousJobStepInstance, jobStepInstance, configurator);
jobStepInstance.ActualEndTime = DateTime.Now;
if (jobStepInstance.HasRecordErrors)
{
jobInstance.HasRecordErrors = true;
}
if (jobStepInstance.HasRuntimeErrors)
{
jobStepInstance.Status = JobStepQueueStatus.Completed_WithError;
jobInstance.HasRuntimeErrors = true;
previousJobStepHasRuntimeErrors = true;
}
else
{
jobStepInstance.Status = JobStepQueueStatus.Completed;
}
}
catch (SafeJobStopException ex)
{
jobStepInstance.ActualEndTime = DateTime.Now;
isSafeJobStopViaException = true;
SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Info, () =>
{
return(string.Format("Job step '{0}' of job '{1}' was terminated safely with message: {2}",
jobStepInstance.JobStep.Name, jobStepInstance.AssociatedJobInstance.Job.Name, ex.Message));
});
continue;
}
catch (Exception ex)
{
jobStepInstance.ActualEndTime = DateTime.Now;
jobStepInstance.HasRuntimeErrors = true;
jobStepInstance.Status = JobStepQueueStatus.Completed_WithError;
jobInstance.HasRuntimeErrors = true;
jobInstance.Exceptions.Add(ex);
previousJobStepHasRuntimeErrors = true;
SyncEngineLogger.WriteByParallelTaskContext(ex);
jobInstance.RunningJobStepInstance = null;
continue;
}
}
previousJobStepInstance = jobStepInstance;
}
jobInstance.ActualEndTime = jobInstance.JobStepInstances.Select(d => d.ActualEndTime).Max();
jobInstance.RunningJobStepInstance = null;
if (jobInstance.HasRuntimeErrors && terminateNextStepsOnStepFailure)
{
jobInstance.Status = JobQueueStatus.Terminated_WithError;
}
else if (jobInstance.HasRuntimeErrors)
{
jobInstance.Status = JobQueueStatus.Completed_WithError;
}
else
{
jobInstance.Status = JobQueueStatus.Completed;
}
}
public JobDequeuedArgs(JobInstance dequeuedJobInstance)
{
DequeuedJobInstance = dequeuedJobInstance;
}
public static List <JobInstance> QueueJob(Integration integration, IEnumerable <DataSource> sourceSideDataSourcesToRun, Job jobToRun,
DateTime scheduledStartTime, string invocationSource, JobInvocationSourceType invocationSourceType, List <JobFilter> filters = null)
{
if (integration == null)
{
throw new Exception("Integration is missing or empty.");
}
if (sourceSideDataSourcesToRun == null || sourceSideDataSourcesToRun.Where(d => d != null).Count() == 0)
{
throw new Exception(string.Format("At least one source-side data source is required to queue job '{0}'.", jobToRun.Name));
}
foreach (var dataSource in sourceSideDataSourcesToRun)
{
if (!jobToRun.SourceDataSources.ContainsKey(dataSource.Name))
{
throw new Exception(string.Format("Source-side data source '{0}' does not exist for job '{1}'. This job will not be queued for execution.", dataSource.Name, jobToRun.Name));
}
}
if (String.IsNullOrWhiteSpace(invocationSource))
{
throw new Exception("Invocation source is missing or empty.");
}
if (!integration.IsEnabled)
{
throw new Exception(string.Format("Job '{0}' could not be queued for execution because integration '{1}' is disabled.", jobToRun.Name, integration.Name));
}
if (!jobToRun.IsEnabled)
{
throw new Exception(string.Format("Job '{0}' could not be queued for execution because it is disabled.", jobToRun.Name));
}
if (jobToRun.Steps.Where(d => d.IsEnabled).Count() == 0)
{
throw new Exception(string.Format("Job '{0}' could not be queued for execution because no job steps exist or all are disabled.", jobToRun.Name));
}
SetMaxConcurrentThreadsForIntegration(integration);
List <JobInstance> queuedJobs = new List <JobInstance>();
JobInstance jobInstance;
JobStepInstance jobStepInstance;
List <JobStepInstance> jobStepInstances;
var queueRequest = new JobQueueRequest(integration, jobToRun, invocationSourceType);
queueRequest.StatusChanged += new EventHandler <JobQueueRequestStatusChangedArgs>(HandleChangedQueueRequestStatus);
queueRequest.Status = JobQueueRequestStatus.Waiting;
foreach (var sourceSideDataSource in sourceSideDataSourcesToRun)
{
byte jobStepIdx = 0;
jobStepInstances = new List <JobStepInstance>();
foreach (var jobStep in jobToRun.Steps)
{
if (jobStep.IsEnabled)
{
jobStepInstance = new JobStepInstance(jobStep, jobStepIdx);
jobStepInstance.StatusChanged += new EventHandler <JobStepQueueStatusChangedArgs>(HandleChangedJobStepStatus);
jobStepInstances.Add(jobStepInstance);
//jobStepInstance.Status = JobStepQueueStatus.Waiting;
jobStepIdx++;
}
}
jobInstance = new JobInstance(queueRequest, integration, jobToRun, sourceSideDataSource, jobToRun.TargetDataSource, jobStepInstances,
scheduledStartTime, invocationSource, invocationSourceType, filters);
queueRequest.JobInstances.Add(jobInstance);
// change the status here to avoid the job step not having an association with the job instance when status changes are logged
foreach (var jobStepInstanceToFireStatusChangeFor in jobStepInstances)
{
jobStepInstanceToFireStatusChangeFor.Status = JobStepQueueStatus.Waiting;
}
jobInstance.StatusChanged += new EventHandler <JobQueueStatusChangedArgs>(HandleChangedJobStatus);
if (waitingJobsByIntegrationId.ContainsKey(jobInstance.Integration.Id))
{
waitingJobsByIntegrationId[jobInstance.Integration.Id].Add(jobInstance);
}
else
{
waitingJobsByIntegrationId.Add(jobInstance.Integration.Id, new List <JobInstance>()
{
jobInstance
});
}
// cache the job instance in memory if on-demand (temporary to get around OutOfMemoryException)
if (jobInstance.InvocationSourceType == JobInvocationSourceType.OnDemand)
{
jobInstancesById.Add(jobInstance.Id, jobInstance);
queueRequestsById.Add(queueRequest.Id, queueRequest);
}
jobInstance.Status = JobQueueStatus.Scheduled;
queuedJobs.Add(jobInstance);
if (JobQueued != null)
{
JobQueued(null, new JobQueuedArgs(jobInstance));
}
}
return(queuedJobs);
}
public JobQueuedArgs(JobInstance queuedJobInstance)
{
QueuedJobInstance = queuedJobInstance;
}
public JobQueueStatusChangedArgs(JobInstance updatedJobInstance)
{
UpdatedJobInstance = updatedJobInstance;
}