public static void AddMissingDataColumns(DataTable tbl, IEnumerable <string> columns)
{
List <string> addedColumns = new List <string>();
foreach (var column in columns)
{
if (!tbl.Columns.Contains(column))
{
tbl.Columns.Add(column);
addedColumns.Add(column);
}
}
SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Debug, () =>
{
StringBuilder logMessage = new StringBuilder();
logMessage.AppendLine("Missing data columns added to data:");
foreach (var column in addedColumns)
{
logMessage.AppendLine(column);
}
return(logMessage.ToString());
});
}
public static void RefreshServiceConfig()
{
try
{
_serviceConfig = _configurator.GetServiceConfig();
JobQueueManager.Stop();
ConfigureMailNotifications();
if (!_serviceConfig.LogToDatabase)
{
SyncEngineLogger.DeregisterLogger(_databaseLogger);
}
if (!_serviceConfig.LogToFile)
{
SyncEngineLogger.DeregisterLogger(_textFileLogger);
}
JobQueueManager.Start(_serviceConfig.IntervalInSeconds);
}
catch (Exception ex)
{
EventViewerLogger.WriteToLog(ex);
SyncEngineLogger.WriteExceptionToLog(ex);
SyncEngineLogger.WriteToLog(LogEntryType.Warning, "The service config could not be refreshed and will occur the next time the service starts.");
}
}
public void ProcessBatch(EntityBatch entityBatch)
{
SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Info, this.AssociatedDataSource, () =>
{
if (entityBatch.EntityDefinition.TechnicalEntityName == entityBatch.EntityDefinition.UserFriendlyEntityName)
{
return(string.Format("Processing {0} insert(s), {1} update(s), and {2} deletion(s) for entity batch for entity '{3}'.",
entityBatch.RecordsToAdd.Count, entityBatch.RecordsToUpdate.Count, entityBatch.RecordsToDelete.Count,
entityBatch.EntityDefinition.TechnicalEntityName));
}
else
{
return(string.Format("Processing {0} insert(s), {1} update(s), and {2} deletion(s) for entity batch for entity '{3}' ({4}).",
entityBatch.RecordsToAdd.Count, entityBatch.RecordsToUpdate.Count, entityBatch.RecordsToDelete.Count,
entityBatch.EntityDefinition.TechnicalEntityName, entityBatch.EntityDefinition.UserFriendlyEntityName));
}
});
if (!(entityBatch.EntityDefinition.PrimaryKeyGenerationType == Data.PrimaryKeyGenerationType.AutoGenerate ||
entityBatch.EntityDefinition.PrimaryKeyGenerationType == Data.PrimaryKeyGenerationType.Manual ||
entityBatch.EntityDefinition.PrimaryKeyGenerationType == Data.PrimaryKeyGenerationType.Custom))
{
throw new EnumValueNotImplementedException <PrimaryKeyGenerationType>(entityBatch.EntityDefinition.PrimaryKeyGenerationType);
}
var tableName = entityBatch.EntityDefinition.TechnicalEntityName;
ProcessInserts(entityBatch.RecordsToAdd, entityBatch, tableName);
ProcessUpdates(entityBatch.RecordsToUpdate, entityBatch, tableName);
ProcessDeletions(entityBatch.RecordsToDelete, entityBatch, tableName);
entityBatch.HasBeenProcessed = true;
}
protected override void OnStop()
{
CloseWebServiceHosts();
// stop the queue manager so that no new job instances are started
JobQueueManager.Stop();
if (JobQueueManager.HasRunningJobs)
{
StringBuilder msg = new StringBuilder();
var runningJobs = JobQueueManager.RunningJobsByIntegrationId;
msg.AppendLine("The Windows service is stopping. The following jobs will be completed first:");
foreach (var integrationName in runningJobs.Keys)
{
foreach (var jobInstance in runningJobs[integrationName])
{
msg.AppendFormat("{0} : {1} : {2} ({3})",
jobInstance.Integration.Name, jobInstance.Job.Name, jobInstance.SourceDataSource.DataSource.Name, jobInstance.Id);
}
}
SyncEngineLogger.WriteToLog(LogEntryType.Info, msg.ToString());
}
// continue to pause the service thread so long as jobs are running
while (JobQueueManager.HasRunningJobs)
{
Thread.Sleep(1000);
}
}
private static void ConfigureMailNotifications()
{
SyncEngineLogger.EnableMailNotifications = _serviceConfig.EnableMailNotifications;
if (_serviceConfig.EnableMailNotifications)
{
try
{
var fromAddress = new MailAddress(_serviceConfig.FromEmailAddress);
var toAddresses = new List <MailAddress>();
foreach (var toAddress in _serviceConfig.ToEmailAddresses)
{
toAddresses.Add(new MailAddress(toAddress));
}
SyncEngineLogger.MailConfig = new MailConfig(fromAddress, toAddresses,
_serviceConfig.SmtpHost, _serviceConfig.SmtpPort);
SyncEngineLogger.MailConfig.EnableSsl = _serviceConfig.SmtpRequiresSsl;
if (!(_serviceConfig.SmtpUsername == null || _serviceConfig.SmtpUsername.Trim() == string.Empty))
{
SyncEngineLogger.MailConfig.Credentials = new NetworkCredential(_serviceConfig.SmtpUsername, _serviceConfig.SmtpPassword);
}
}
catch (Exception ex)
{
EventViewerLogger.WriteToLog(ex);
SyncEngineLogger.WriteExceptionToLog(ex);
}
}
}
private static void InvokeJobInstancesUsingParallelTasks(Guid integrationId, IEnumerable <JobInstance> jobInstancesToStartNext)
{
foreach (var jobInstanceToStartNext in jobInstancesToStartNext)
{
waitingJobsByIntegrationId[integrationId].Remove(jobInstanceToStartNext);
if (runningJobsByIntegrationId.ContainsKey(jobInstanceToStartNext.Integration.Id))
{
runningJobsByIntegrationId[jobInstanceToStartNext.Integration.Id].Add(jobInstanceToStartNext);
}
else
{
runningJobsByIntegrationId.Add(jobInstanceToStartNext.Integration.Id, new List <JobInstance>()
{
jobInstanceToStartNext
});
}
// invoke the job using a parallel job
// NOTE: code called downstream but within this block will have a unique Task ID that is used
// to refer back to the associated integration within the sync engine logger
var workerThread = new Thread(() =>
{
try
{
// this is an unlikely scenario but may occur if the Task IDs start over
if (jobInstancesByParallelTaskId.ContainsKey(Thread.CurrentThread.ManagedThreadId))
{
jobInstancesByParallelTaskId[Thread.CurrentThread.ManagedThreadId] = jobInstanceToStartNext;
}
else
{
jobInstancesByParallelTaskId.Add(Thread.CurrentThread.ManagedThreadId, jobInstanceToStartNext);
}
SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstanceToStartNext, "Job instance '{0}' is starting.", jobInstanceToStartNext.Id);
JobInvoker.ExecuteJob(jobInstanceToStartNext, Configurator);
SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstanceToStartNext, "Job instance '{0}' has ended.", jobInstanceToStartNext.Id);
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(jobInstanceToStartNext, ex);
}
});
workerThread.Start();
var threadStateForIntegration = threadsInUseByIntegrationId[jobInstanceToStartNext.Integration.Id];
threadStateForIntegration.ThreadsInUse++;
if (ThreadUsageChanged != null)
{
ThreadUsageChanged(null, new ConcurrentThreadUsageChangedArgs(jobInstanceToStartNext.Integration, threadStateForIntegration));
}
}
}
private static void RegisterIntegrations()
{
var integrations = _configurator.GetIntegrations();
foreach (var integration in integrations)
{
SyncEngineLogger.RegisterIntegration(integration);
}
}
public override void ProcessBatch(JobBatch jobBatch)
{
SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Info, this.AssociatedDataSource,
"Processing {0} entity batch(es) for sync side '{1}'.",
jobBatch.EntityBatches.Count, Enum.GetName(typeof(SyncSide), jobBatch.SyncSide));
foreach (EntityBatch entityBatch in jobBatch.EntityBatches)
{
ProcessBatch(entityBatch);
}
}
public SelfContainedMigrator(string migrationName, Guid migrationId, string sourceName, string targetName)
{
JobQueueManager.Start(1);
_integration = new Integration(migrationId, migrationName, Assembly.GetCallingAssembly().Location, sourceName, targetName);
var loggingPath = Path.GetDirectoryName(Assembly.GetCallingAssembly().Location) + @"\Logs";
var textFileLogger = new TextFileLogger(loggingPath);
SyncEngineLogger.RegisterLogger(textFileLogger, LoggingLevel.ErrorsWarningsInfoDebugAndTrace, -1);
}
public JobInstanceResult GetJobInstanceResult(Guid jobInstanceId)
{
try
{
return(JobQueueManager.GetJobInstanceResult(jobInstanceId));
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public bool?QueueRequestIsComplete(Guid queueRequestId)
{
try
{
return(JobQueueManager.QueueRequestIsComplete(queueRequestId));
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public List <KeyValuePair <Guid, List <JobInstance> > > GetRunningJobsByIntegrationId()
{
try
{
return(JobQueueManager.RunningJobsByIntegrationId.ToList());
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public bool HasRunningJobs()
{
try
{
return(JobQueueManager.HasRunningJobs);
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public List <JobSchedule> GetJobSchedulesByJobId(Guid jobId)
{
try
{
return(configurator.GetJobSchedulesByJobId(jobId));
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public double GetIntervalInSeconds()
{
try
{
return(JobQueueManager.IntervalInSeconds);
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public void Stop()
{
try
{
JobQueueManager.Stop();
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public bool JobScheduleExistsById(Guid jobScheduleId)
{
try
{
return(configurator.JobScheduleExistsById(jobScheduleId));
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public void DeleteAllHistory()
{
try
{
SyncEngineLogger.DeleteAllHistory();
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public void DeleteServiceHistory(int daysToKeep)
{
try
{
SyncEngineLogger.DeleteServiceHistory(daysToKeep);
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public void DeleteIntegrationHistory(Guid integrationId, int daysToKeep)
{
try
{
SyncEngineLogger.DeleteIntegrationHistory(integrationId, daysToKeep);
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public void DeleteJobDataSourceHistory(Guid jobId)
{
try
{
SyncEngineLogger.DeleteJobDataSourceHistory(jobId);
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public List <JobInstance> DequeueScheduledJobs()
{
try
{
return(JobQueueManager.DequeueScheduledJobs());
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public bool DequeueJob(Guid jobInstanceGuid)
{
try
{
return(JobQueueManager.DequeueJob(jobInstanceGuid));
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public TimeSpan GetMaxDelayedStartForJobPriority(JobPriority jobPriority)
{
try
{
return(JobQueueManager.GetMaxDelayedStartForJobPriority(jobPriority));
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public JobQueueManagerStatus GetStatus()
{
try
{
return(JobQueueManager.Status);
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public List <DataSource> GetJobDataSourcesByJobName(string integrationName, string jobName, SyncSide syncSide)
{
try
{
return(configurator.GetJobDataSourcesByJobName(integrationName, jobName, syncSide));
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public List <KeyValuePair <JobPriority, TimeSpan> > GetMaxDelayedStartByJobPriority()
{
try
{
return(JobQueueManager.MaxDelayedStartByJobPriority.ToList());
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public void Start(double intervalInSeconds)
{
try
{
JobQueueManager.Start(intervalInSeconds);
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public ConcurrentThreadState GetThreadUsage(Guid integrationId)
{
try
{
return(JobQueueManager.GetThreadUsage(integrationId));
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
public List <KeyValuePair <Guid, ConcurrentThreadState> > GetThreadUsagesByIntegrationId()
{
try
{
return(JobQueueManager.GetThreadUsagesByIntegrationId().ToList());
}
catch (Exception ex)
{
SyncEngineLogger.WriteExceptionToLog(ex, WEB_SERVICE_EXCEPTION_MESSAGE);
throw;
}
}
