public async Task CreateJobBatchAsync_assigns_new_JobBatchId_when_attempt_to_save_fails(int iterations)
{
var jobBatch = new JobBatch();
int count = 0;
_testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache
.Setup(jobBatchCache => jobBatchCache.TryAddValue(It.IsAny <Guid>(), jobBatch))
.Returns((Guid guid, JobBatch batch) => count++ < iterations ? false : true)
.Verifiable();
var memoryCacheJobBatchRepository = _testMemoryCacheJobBatchRepositoryFactory.CreateMemoryCacheJobBatchRepository();
Assert.Equal(Guid.Empty, jobBatch.JobBatchId);
Assert.Same(jobBatch, await memoryCacheJobBatchRepository.CreateJobBatchAsync(jobBatch));
Assert.NotEqual(Guid.Empty, jobBatch.JobBatchId);
_testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache.Verify(
jobBatchCache => jobBatchCache.TryAddValue(
It.Is <Guid>(guid => guid != jobBatch.JobBatchId),
jobBatch),
Times.Exactly(iterations));
_testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache.Verify(
jobBatchCache => jobBatchCache.TryAddValue(jobBatch.JobBatchId, jobBatch),
Times.Once);
}
public async Task GetJobBatchAsync_return_JobBatch_when_present_in_cache()
{
JobBatch jobBatch = new JobBatch
{
JobBatchId = Guid.NewGuid()
};
_testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache
.Setup(jobBatchCache => jobBatchCache.TryGetValue(
jobBatch.JobBatchId,
out It.Ref <JobBatch> .IsAny))
.Callback(new GetCachedJobBatch((Guid batchId, out JobBatch jobBatchToGet) =>
{
jobBatchToGet = jobBatch;
}))
.Returns(true)
.Verifiable();
var memoryCacheJobBatchRepository = _testMemoryCacheJobBatchRepositoryFactory.CreateMemoryCacheJobBatchRepository();
Assert.Same(jobBatch, await memoryCacheJobBatchRepository.GetJobBatchAsync(jobBatch.JobBatchId));
_testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache
.Verify(
jobBatchCache => jobBatchCache.TryGetValue(
jobBatch.JobBatchId,
out It.Ref <JobBatch> .IsAny),
Times.Once);
}
public override void ProcessBatch(JobBatch jobBatch)
{
SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Info, this.AssociatedDataSource,
"Processing {0} entity batch(es) for sync side '{1}'.",
jobBatch.EntityBatches.Count, Enum.GetName(typeof(SyncSide), jobBatch.SyncSide));
foreach (EntityBatch entityBatch in jobBatch.EntityBatches)
{
ProcessBatch(entityBatch);
}
}
public Task <JobBatch> UpdateJobBatchAsync(JobBatch jobBatch)
{
if (jobBatch == null)
{
throw new ArgumentNullException(nameof(jobBatch));
}
jobBatch = _jobBatchCache.SetValue(
jobBatch.JobBatchId,
jobBatch);
return(Task.FromResult(jobBatch));
}
public void TestScheduleInBatch()
{
JobBatch.Do(() =>
{
for (var i = 0; i < 100000; i++)
{
MockJob.Publish();
}
});
StopWorkers();
Assert.AreEqual(100000, _counter.Value);
}
public async Task <JobBatch> CreateJobBatchAsync(CreateJobBatchRequest createJobBatchRequest)
{
if (createJobBatchRequest == null)
{
throw new ArgumentNullException(nameof(createJobBatchRequest));
}
_jobBatchServiceLogger.LogInformation(
BlueshiftJobsResources.CreatingJobBatch,
createJobBatchRequest.JobBatchRequestor,
createJobBatchRequest.JobBatchDescription,
createJobBatchRequest.BatchingJobFilter.MaximumItems);
try
{
IReadOnlyCollection <Job> jobs = await _jobRepository
.GetJobsAsync(createJobBatchRequest.BatchingJobFilter)
.ConfigureAwait(false);
var jobBatch = new JobBatch
{
JobBatchOwnerId = createJobBatchRequest.JobBatchRequestor,
JobBatchDescription = createJobBatchRequest.JobBatchDescription
}
.WithJobs(jobs);
jobBatch = await _jobBatchRepository
.CreateJobBatchAsync(jobBatch)
.ConfigureAwait(false);
_jobBatchServiceLogger.LogInformation(
BlueshiftJobsResources.CreatedJobBatch,
jobBatch.JobBatchOwnerId,
jobBatch.JobBatchDescription,
jobBatch.Jobs.Count);
return(jobBatch);
}
catch (Exception e)
{
_jobBatchServiceLogger.LogError(
BlueshiftJobsResources.ErrorCreatingJobBatch,
_jobBatchServiceLogger.IsEnabled(LogLevel.Debug)
? e.StackTrace
: e.Message);
throw;
}
}
private JobBatch ProcessJobBatch(SyncSide syncSide, DataSource dataSource, List <EntityBatch> entityBatches)
{
var jobDataSource = new JobDataSource(syncSide, dataSource);
var jobBatch = new JobBatch(syncSide, jobDataSource);
foreach (var entityBatch in entityBatches)
{
jobBatch.EntityBatches.Add(entityBatch);
}
dataSource.ProcessBatch(jobBatch);
return(jobBatch);
}
private void Initalize(Int32 numberOfThreads)
{
if (numberOfThreads <= 0)
{
throw new ArgumentException("Number of threads must be greater than 0");
}
m_NumThreads = numberOfThreads;
m_AllTasks = new Dictionary <Action, Job>();
m_ActiveTasks = new List <Job>(42);
m_JobBatches = new JobBatch[m_NumThreads];
for (int i = 0; i < m_JobBatches.Length; i++)
{
m_JobBatches[i] = new JobBatch();
}
}
public async Task CreateJobBatchAsync_requests_the_expected_number_of_jobs_and_saves_the_batch(int requestedJobCount)
{
_testJobBatchServiceFactory.MockJobRepository
.Setup(jobRepository => jobRepository.GetJobsAsync(It.IsAny <JobSearchCriteria>()))
.ReturnsAsync((JobSearchCriteria jobSearchCriteria) =>
_testJobBatchServiceFactory.AvailableJobs
.Take(jobSearchCriteria.MaximumItems.Value)
.ToList()
.AsReadOnly()
)
.Verifiable();
_testJobBatchServiceFactory.MockJobBatchRepository
.Setup(jobBatchRepository => jobBatchRepository.CreateJobBatchAsync(It.IsAny <JobBatch>()))
.ReturnsAsync((JobBatch batch) => batch)
.Verifiable();
JobBatchService jobBatchService = _testJobBatchServiceFactory.CreateJobBatchService();
var createJobBatchRequest = new CreateJobBatchRequest()
{
BatchingJobFilter =
{
MaximumItems = requestedJobCount
}
};
JobBatch jobBatch = await jobBatchService.CreateJobBatchAsync(createJobBatchRequest);
Assert.NotNull(jobBatch);
Assert.Equal(
_testJobBatchServiceFactory.AvailableJobs.Take(requestedJobCount),
jobBatch.Jobs);
_testJobBatchServiceFactory.MockJobRepository
.Verify(
jobRepository => jobRepository.GetJobsAsync(
It.Is <JobSearchCriteria>(jobSearchCriteria => jobSearchCriteria.MaximumItems == requestedJobCount)),
Times.Once);
_testJobBatchServiceFactory.MockJobBatchRepository
.Verify(
jobBatchRepository => jobBatchRepository.CreateJobBatchAsync(jobBatch),
Times.Once);
}
public Task <JobBatch> CreateJobBatchAsync(JobBatch jobBatch)
{
if (jobBatch == null)
{
throw new ArgumentNullException(nameof(jobBatch));
}
Guid jobBatchId;
do
{
jobBatchId = Guid.NewGuid();
}while (!_jobBatchCache.TryAddValue(jobBatchId, jobBatch));
jobBatch.JobBatchId = jobBatchId;
return(Task.FromResult(jobBatch));
}
public async Task UpdateJobBatchAsync_sets_value_in_cache()
{
var jobBatch = new JobBatch
{
JobBatchId = Guid.NewGuid()
};
_testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache
.Setup(jobBatchCache => jobBatchCache.SetValue(jobBatch.JobBatchId, jobBatch))
.Returns((Guid guid, JobBatch batch) => batch)
.Verifiable();
var memoryCacheJobBatchRepository = _testMemoryCacheJobBatchRepositoryFactory.CreateMemoryCacheJobBatchRepository();
Assert.Same(jobBatch, await memoryCacheJobBatchRepository.UpdateJobBatchAsync(jobBatch));
_testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache.Verify(
jobBatchCache => jobBatchCache.SetValue(jobBatch.JobBatchId, jobBatch),
Times.Once);
}
public async Task CreateJobBatchAsync_assigns_JobBatchId_and_saves_to_cache()
{
var jobBatch = new JobBatch();
_testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache
.Setup(jobBatchCache => jobBatchCache.TryAddValue(It.IsAny <Guid>(), jobBatch))
.Returns(true)
.Verifiable();
var memoryCacheJobBatchRepository = _testMemoryCacheJobBatchRepositoryFactory.CreateMemoryCacheJobBatchRepository();
Assert.Equal(Guid.Empty, jobBatch.JobBatchId);
Assert.Same(jobBatch, await memoryCacheJobBatchRepository.CreateJobBatchAsync(jobBatch));
Assert.NotEqual(Guid.Empty, jobBatch.JobBatchId);
_testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache.Verify(
jobBatchCache => jobBatchCache.TryAddValue(jobBatch.JobBatchId, jobBatch),
Times.Once);
}
public IActionResult Post(DtoChirpPost dto)
{
_db.BeginTransaction();
var chirp = new Chirp
{
UserId = int.Parse(User.Identity.Name ?? "1"),
ChirpTimeUtc = DateTime.UtcNow,
ChirpType = ChirpType.Chirp,
Contents = dto.Contents
};
_db.Chirps.Add(chirp);
_db.SaveChanges();
// Publish chirp processing job, before commit.
// If the publishing fails, the transaction will be rolled back.
//
// We're using the job publish as the last committing resource because
// it does not participate in the DB transaction.
JobBatch.Do(() =>
{
TimelineUpdate.Publish(new TimelineUpdateArgs
{
ChirpId = chirp.Id,
AuthorId = chirp.UserId,
TimeUtc = chirp.ChirpTimeUtc
});
HashTagUpdate.Publish(chirp.Id);
});
_db.CommitTransaction();
return(Ok(chirp.Id));
}
public void ProcessBatch(JobBatch jobBatch)
{
if (jobBatch == null)
{
throw new Exception("Job batch can not be null.");
}
if (jobBatch.EntityBatches == null || jobBatch.EntityBatches.Count == 0)
{
SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Info, () =>
{
return(string.Format("The job batch for sync side '{0}' and data source '{1}' contains no entity batches. No processing has occurred.",
Enum.GetName(typeof(SyncSide), jobBatch.SyncSide), jobBatch.AssociatedDataSource.DataSource.Name));
});
}
else
{
using (var adapter = GetAdapterInstance(Adapter))
{
adapter.ProcessBatch(jobBatch);
}
}
}
public Job(JobBatch batch)
{
this.batch = batch;
}
public Job(JobBatch batch)
{
this.batch = batch;
}
public abstract void ProcessBatch(JobBatch jobBatch);
public JobBatchStepOutput(JobBatch sourceSideJobBatch, JobBatch targetSideJobBatch)
{
SourceSideJobBatch = sourceSideJobBatch;
TargetSideJobBatch = targetSideJobBatch;
}
public static void ExecuteStep(JobInstance jobInstance, JobStepInstance previousJobStepInstance,
JobStepInstance currentJobStepInstance, ISyncEngineConfigurator configurator)
{
Type jobStepType = jobInstance.Integration.PackageAssembly.GetType(currentJobStepInstance.JobStep.FullyQualifiedName);
if (jobStepType == null)
{
throw new Exception(string.Format("Job step with fully qualified name '{0}' was not found in assembly '{1}'.", currentJobStepInstance.JobStep.FullyQualifiedName, jobInstance.Integration.PackageAssembly.Location));
}
// ensure the step class inherits from the proper base class to ensure the Initialize method is available
bool hasCorrectBaseType = false;
var baseType = jobStepType.BaseType;
while (baseType != typeof(Object))
{
if (baseType == typeof(JobStepInvocation))
{
hasCorrectBaseType = true;
break;
}
baseType = baseType.BaseType;
}
if (!hasCorrectBaseType)
{
throw new Exception(string.Format("Job step class '{0}' must derive from '{1}'.", jobStepType.Name, typeof(JobStepInvocation).FullName));
}
var jobStepInvocation = Activator.CreateInstance(jobStepType);
if (jobStepInvocation is CustomActionStep)
{
var jobStepObj = (CustomActionStep)jobStepInvocation;
jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);
jobStepObj.Process();
if (previousJobStepInstance != null &&
((previousJobStepInstance.HasDeferredExecutionUntilNextStep.HasValue &&
previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true) ||
(previousJobStepInstance.GetType() is CustomActionStep)))
{
currentJobStepInstance.SourceJobBatch = previousJobStepInstance.SourceJobBatch;
currentJobStepInstance.TargetJobBatch = previousJobStepInstance.TargetJobBatch;
currentJobStepInstance.HasDeferredExecutionUntilNextStep = true;
}
else
{
currentJobStepInstance.HasDeferredExecutionUntilNextStep = false;
}
}
else if (jobStepInvocation is DataMapStep)
{
var jobStepObj = (DataMapStep)jobStepInvocation;
jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);
var dataMapOutput = jobStepObj.Process();
if (dataMapOutput == null)
{
throw new Exception("Job step must return a value.");
}
currentJobStepInstance.HasDeferredExecutionUntilNextStep = dataMapOutput.DeferExecutionUntilNextStep;
JobBatch sourceJobBatch = null;
JobBatch targetJobBatch = null;
EntityBatch sourceEntityBatch = null;
EntityBatch targetEntityBatch = null;
EntityBatch oneWayEntityBatch = null;
IOneWayDataMap oneWayDataMap = null;
TwoWayDataMap twoWayDataMap = null;
if (dataMapOutput.DataMap is IOneWayDataMap)
{
oneWayDataMap = (IOneWayDataMap)dataMapOutput.DataMap;
if (dataMapOutput.DataMap is OneWayDataMap)
{
oneWayEntityBatch = OneToOneDataMapProcessor.Compare((OneWayDataMap)dataMapOutput.DataMap,
dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior,
dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior,
dataMapOutput.RowsToProcess);
}
else if (dataMapOutput.DataMap is OneToMany_OneWayDataMap)
{
if (dataMapOutput.RowsToProcess != null)
{
throw new Exception("Rows to process Func is not supported for one-to-many data maps.");
}
oneWayEntityBatch = OneToManyDataMapProcessor.Compare((OneToMany_OneWayDataMap)dataMapOutput.DataMap,
dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior,
dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior);
}
if (oneWayEntityBatch.EntityDefinition.SyncSide == SyncSide.Source)
{
oneWayEntityBatch.LoggingBehavior = dataMapOutput.SourceSideLoggingBehavior;
}
else if (oneWayEntityBatch.EntityDefinition.SyncSide == SyncSide.Target)
{
oneWayEntityBatch.LoggingBehavior = dataMapOutput.TargetSideLoggingBehavior;
}
}
else if (dataMapOutput.DataMap is TwoWayDataMap)
{
twoWayDataMap = (TwoWayDataMap)dataMapOutput.DataMap;
OneToOneDataMapProcessor.Compare(twoWayDataMap, dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior,
dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior,
dataMapOutput.RowsToProcess, out sourceEntityBatch, out targetEntityBatch);
sourceEntityBatch.LoggingBehavior = dataMapOutput.SourceSideLoggingBehavior;
targetEntityBatch.LoggingBehavior = dataMapOutput.TargetSideLoggingBehavior;
}
else
{
throw new DerivedClassNotImplementedException <OneToOneDataMap>(dataMapOutput.DataMap);
}
if (previousJobStepInstance != null &&
previousJobStepInstance.HasDeferredExecutionUntilNextStep.HasValue &&
previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true)
{
sourceJobBatch = previousJobStepInstance.SourceJobBatch;
targetJobBatch = previousJobStepInstance.TargetJobBatch;
}
else
{
sourceJobBatch = new JobBatch(SyncSide.Source, jobInstance.SourceDataSource);
targetJobBatch = new JobBatch(SyncSide.Target, jobInstance.TargetDataSource);
}
if (dataMapOutput.DataMap is IOneWayDataMap)
{
if (oneWayDataMap.SyncDirection == SyncDirection.SourceToTarget)
{
targetJobBatch.EntityBatches.Add(oneWayEntityBatch);
}
else if (oneWayDataMap.SyncDirection == SyncDirection.TargetToSource)
{
sourceJobBatch.EntityBatches.Add(oneWayEntityBatch);
}
else
{
throw new EnumValueNotImplementedException <SyncDirection>(oneWayDataMap.SyncDirection);
}
}
else if (dataMapOutput.DataMap is TwoWayDataMap)
{
sourceJobBatch.EntityBatches.Add(sourceEntityBatch);
targetJobBatch.EntityBatches.Add(targetEntityBatch);
}
else
{
throw new DerivedClassNotImplementedException <OneToOneDataMap>(dataMapOutput.DataMap);
}
currentJobStepInstance.SourceJobBatch = sourceJobBatch;
currentJobStepInstance.TargetJobBatch = targetJobBatch;
if (!currentJobStepInstance.HasDeferredExecutionUntilNextStep.Value)
{
sourceJobBatch.SubmitToDataSource();
if (sourceJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = sourceJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, sourceJobBatch);
targetJobBatch.SubmitToDataSource();
if (targetJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = targetJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, targetJobBatch);
}
}
else if (jobStepInvocation is EntityBatchStep)
{
var jobStepObj = (EntityBatchStep)jobStepInvocation;
jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);
var entityBatchOutput = jobStepObj.Process();
if (entityBatchOutput == null)
{
throw new Exception("Job step must return a value.");
}
currentJobStepInstance.HasDeferredExecutionUntilNextStep = entityBatchOutput.DeferExecutionUntilNextStep;
JobBatch sourceJobBatch;
JobBatch targetJobBatch;
if (previousJobStepInstance != null && previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true)
{
sourceJobBatch = previousJobStepInstance.SourceJobBatch;
targetJobBatch = previousJobStepInstance.TargetJobBatch;
}
else
{
sourceJobBatch = new JobBatch(SyncSide.Source, jobInstance.SourceDataSource);
targetJobBatch = new JobBatch(SyncSide.Target, jobInstance.TargetDataSource);
}
if (entityBatchOutput.SourceSideEntityBatches != null)
{
foreach (var sourceSideEntityBatch in entityBatchOutput.SourceSideEntityBatches)
{
sourceJobBatch.EntityBatches.Add(sourceSideEntityBatch);
}
}
if (entityBatchOutput.TargetSideEntityBatches != null)
{
foreach (var targetSideEntityBatch in entityBatchOutput.TargetSideEntityBatches)
{
targetJobBatch.EntityBatches.Add(targetSideEntityBatch);
}
}
currentJobStepInstance.SourceJobBatch = sourceJobBatch;
currentJobStepInstance.TargetJobBatch = targetJobBatch;
if (!currentJobStepInstance.HasDeferredExecutionUntilNextStep.Value)
{
sourceJobBatch.SubmitToDataSource();
if (sourceJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = sourceJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, sourceJobBatch);
targetJobBatch.SubmitToDataSource();
if (targetJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = targetJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, targetJobBatch);
}
}
else if (jobStepInvocation is JobBatchStep)
{
var jobStepObj = (JobBatchStep)jobStepInvocation;
jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator);
var jobBatchOutput = jobStepObj.Process();
if (jobBatchOutput == null)
{
throw new Exception("Job step must return a value.");
}
if (jobBatchOutput.SourceSideJobBatch != null)
{
jobBatchOutput.SourceSideJobBatch.SubmitToDataSource();
if (jobBatchOutput.SourceSideJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = jobBatchOutput.SourceSideJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, jobBatchOutput.SourceSideJobBatch);
}
else
{
SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstance, currentJobStepInstance, jobInstance.SourceDataSource.DataSource,
"Job step '{0}' for job '{1}' returned a null job batch for source-side data source '{2}'.",
currentJobStepInstance.JobStep.Name, jobInstance.Job.Name, jobInstance.SourceDataSource.DataSource.Name);
}
if (jobBatchOutput.TargetSideJobBatch != null)
{
jobBatchOutput.TargetSideJobBatch.SubmitToDataSource();
if (jobBatchOutput.TargetSideJobBatch.HasRecordErrors)
{
currentJobStepInstance.HasRecordErrors = true;
currentJobStepInstance.Exceptions = jobBatchOutput.TargetSideJobBatch.GetExceptions();
}
SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, jobBatchOutput.TargetSideJobBatch);
}
else
{
SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstance, currentJobStepInstance, jobInstance.TargetDataSource.DataSource,
"Job step '{0}' for job '{1}' returned a null job batch for target-side data source '{2}'.",
currentJobStepInstance.JobStep.Name, jobInstance.Job.Name, jobInstance.TargetDataSource.DataSource.Name);
}
}
else
{
throw new DerivedClassNotImplementedException <JobStepInvocation>(jobStepInvocation);
}
}
public void WriteToLog(JobInstance jobInstance, JobStepInstance jobStepInstance, JobBatch jobBatch)
{
using (var dbContext = new ls.LoggingDataContext(ConnectionString))
{
foreach (var entityBatch in jobBatch.EntityBatches)
{
int countOfInsertsWithoutError = 0;
int countOfUpdatesWithoutError = 0;
int countOfDeletionsWithoutError = 0;
if (!entityBatch.HasBeenProcessed)
{
WriteToLog(LogEntryType.Error, jobInstance.Integration, jobBatch.AssociatedDataSource.DataSource, jobInstance, jobStepInstance, "The entity batch has not been processed by a data source. No history can be logged.");
continue;
}
var entityBatchHistoryId = (int)dbContext.AddEntityBatchHistory(jobInstance.Integration.Id, jobInstance.Id, jobStepInstance.Id,
jobBatch.AssociatedDataSource.DataSource.Id, Enum.GetName(typeof(SyncSide), jobBatch.SyncSide),
entityBatch.EntityDefinition.TechnicalEntityName, entityBatch.EntityDefinition.UserFriendlyEntityName,
StringHelper.GetDelimitedString(entityBatch.EntityDefinition.PrimaryKeyColumnNames))
.FirstOrDefault().InsertedID;
string batchHistoryTemplate = @"DECLARE @EntityBatchRecordId int
INSERT INTO EntityBatchRecordHistory VALUES ({0}, GETDATE(), '{1}', '{2}', '{3}', {4})
SELECT @EntityBatchRecordId = SCOPE_IDENTITY()";
string batchHistoryDetailTemplate = "\n\rINSERT INTO EntityBatchRecordHistoryDetail VALUES (@EntityBatchRecordId, GETDATE(), '{0}', NULL, {1}, {2})";
// log inserts
foreach (var insertedRecord in entityBatch.RecordsToAdd)
{
//var entityBatchHistoryRecordId = (int)dbContext.AddEntityBatchRecordHistory(entityBatchHistoryId, 'I', StringHelper.GetDelimitedStringFromList(insertedRecord.PrimaryKeyValues),
// insertedRecord.HasError, insertedRecord.ErrorMessage).FirstOrDefault().InsertedID;
if (entityBatch.LoggingBehavior.MaxNumberOfInsertsWithoutErrorToLog.HasValue && !insertedRecord.HasError)
{
countOfInsertsWithoutError++;
if (countOfInsertsWithoutError > entityBatch.LoggingBehavior.MaxNumberOfInsertsWithoutErrorToLog.Value)
{
continue;
}
}
StringBuilder sqlForInsert = new StringBuilder();
sqlForInsert.AppendFormat(batchHistoryTemplate,
entityBatchHistoryId, "I", StringHelper.GetDelimitedString(insertedRecord.PrimaryKeyValues), insertedRecord.HasError,
insertedRecord.ErrorMessage == null ? "NULL" : "'" + insertedRecord.ErrorMessage.Replace("'", "''") + "'");
foreach (var fieldValuePair in insertedRecord.FieldValuePairs)
{
//dbContext.AddEntityBatchRecordHistoryDetail(entityBatchHistoryRecordId, fieldValuePair.Key, null, null, fieldValuePair.Value);
sqlForInsert.AppendFormat(batchHistoryDetailTemplate,
fieldValuePair.Key, "NULL", fieldValuePair.Value == null ? "NULL" : "'" + fieldValuePair.Value.Replace("'", "''") + "'");
}
dbContext.ExecuteCommand(sqlForInsert.ToString());
}
// log updates
foreach (var updatedRecord in entityBatch.RecordsToUpdate)
{
//var entityBatchHistoryRecordId = (int)dbContext.AddEntityBatchRecordHistory(entityBatchHistoryId, 'U', StringHelper.GetDelimitedStringFromList(updatedRecord.PrimaryKeyValues),
// updatedRecord.HasError, updatedRecord.ErrorMessage).FirstOrDefault().InsertedID;
if (entityBatch.LoggingBehavior.MaxNumberOfUpdatesWithoutErrorToLog.HasValue && !updatedRecord.HasError)
{
countOfUpdatesWithoutError++;
if (countOfUpdatesWithoutError > entityBatch.LoggingBehavior.MaxNumberOfUpdatesWithoutErrorToLog.Value)
{
continue;
}
}
StringBuilder sqlForUpdate = new StringBuilder();
sqlForUpdate.AppendFormat(batchHistoryTemplate,
entityBatchHistoryId, "U", StringHelper.GetDelimitedString(updatedRecord.PrimaryKeyValues), updatedRecord.HasError,
updatedRecord.ErrorMessage == null ? "NULL" : "'" + updatedRecord.ErrorMessage.Replace("'", "''") + "'");
foreach (var fieldValuePair in updatedRecord.FieldValuePairs)
{
//dbContext.AddEntityBatchRecordHistoryDetail(entityBatchHistoryRecordId, fieldValuePair.Key, null, fieldValuePair.Value.OldValue, fieldValuePair.Value.NewValue);
sqlForUpdate.AppendFormat(batchHistoryDetailTemplate,
fieldValuePair.Key,
fieldValuePair.Value.OldValue == null ? "NULL" : "'" + fieldValuePair.Value.OldValue.Replace("'", "''") + "'",
fieldValuePair.Value.NewValue == null ? "NULL" : "'" + fieldValuePair.Value.NewValue.Replace("'", "''") + "'");
}
dbContext.ExecuteCommand(sqlForUpdate.ToString());
}
// log deletions
foreach (var deletedRecord in entityBatch.RecordsToDelete)
{
if (entityBatch.LoggingBehavior.MaxNumberOfDeletionsWithoutErrorToLog.HasValue && !deletedRecord.HasError)
{
countOfDeletionsWithoutError++;
if (countOfDeletionsWithoutError > entityBatch.LoggingBehavior.MaxNumberOfDeletionsWithoutErrorToLog.Value)
{
continue;
}
}
var entityBatchHistoryRecordId = (int)dbContext.AddEntityBatchRecordHistory(entityBatchHistoryId, 'D',
StringHelper.GetDelimitedString(deletedRecord.PrimaryKeyValues),
deletedRecord.HasError,
deletedRecord.ErrorMessage).FirstOrDefault().InsertedID;
}
}
}
}
private static string GetLogLine(DateTime logDate, LogEntryType logEntryType, JobInstance jobInstance, JobStepInstance jobStepInstance, JobBatch jobBatch, string message)
{
StringBuilder fullLine = new StringBuilder();
fullLine.Append(logDate);
fullLine.Append("\t");
fullLine.Append(logEntryType);
fullLine.Append("\t");
fullLine.Append(string.Format("{0} ({1})", jobInstance.Integration.Name, jobInstance.Integration.Id));
fullLine.Append("\t");
fullLine.Append(string.Format("{0} ({1})", jobBatch.AssociatedDataSource.DataSource.Name, jobBatch.AssociatedDataSource.DataSource.Id));
fullLine.Append("\t");
fullLine.Append(string.Format("{0} ({1})", jobInstance.Job.Name, jobInstance.Job.Id));
fullLine.Append("\t");
fullLine.Append(string.Format("{0} ({1})", jobInstance.QueueRequest.InvocationSourceType, jobInstance.QueueRequest.Id));
fullLine.Append("\t");
fullLine.Append(jobInstance.Id);
fullLine.Append("\t");
fullLine.Append(JobFilterHelper.GetTextForDatabase(jobInstance.Filters));
fullLine.Append("\t");
fullLine.Append(string.Format("{0} ({1})", jobStepInstance.JobStep.Name, jobStepInstance.JobStep.Id));
fullLine.Append("\t");
fullLine.Append(jobStepInstance.Id);
fullLine.Append("\t");
fullLine.Append(message);
return(fullLine.ToString());
}
public void WriteToLog(JobInstance jobInstance, JobStepInstance jobStepInstance, JobBatch jobBatch)
{
string path = null;
for (int i = 0; i <= NUMBER_OF_FILEACCESS_RETRIES; i++)
{
try
{
path = GetIntegrationLogPathAndFileName(jobInstance.Integration);
break;
}
catch (IOException)
{
if (i == NUMBER_OF_FILEACCESS_RETRIES)
{
throw;
}
else
{
Thread.Sleep(DELAY_ON_FILEACCES_RETRY);
}
}
}
lock (_lock)
{
for (int i = 0; i <= NUMBER_OF_FILEACCESS_RETRIES; i++)
{
try
{
using (TextWriter log = File.AppendText(path))
{
StringBuilder logLine = new StringBuilder();
var now = DateTime.Now;
var errorLogLineMinusMessage = GetLogLine(now, LogEntryType.Error, jobInstance, jobStepInstance, jobBatch, null);
var warningLogLineMinusMessage = GetLogLine(now, LogEntryType.Warning, jobInstance, jobStepInstance, jobBatch, null);
var infoLogLineMinusMessage = GetLogLine(now, LogEntryType.Info, jobInstance, jobStepInstance, jobBatch, null);
foreach (var entityBatch in jobBatch.EntityBatches)
{
int countOfInsertsWithoutError = 0;
int countOfUpdatesWithoutError = 0;
int countOfDeletionsWithoutError = 0;
if (entityBatch.RecordsToAdd.Count == 0 && entityBatch.RecordsToUpdate.Count == 0 && entityBatch.RecordsToDelete.Count == 0)
{
continue;
}
if (!entityBatch.HasBeenProcessed)
{
logLine.Append(warningLogLineMinusMessage);
logLine.AppendLine("The entity batch has not been processed by a data source. No history can be logged.");
continue;
}
logLine.Append(infoLogLineMinusMessage);
if (entityBatch.EntityDefinition.TechnicalEntityName == entityBatch.EntityDefinition.UserFriendlyEntityName)
{
logLine.AppendLine(string.Format("Entity batch applied on sync side '{0}' for entity '{1}':",
jobBatch.SyncSide, entityBatch.EntityDefinition.TechnicalEntityName));
}
else
{
logLine.AppendLine(string.Format("Entity batch applied on sync side '{0}' for entity '{1}' ({2}):",
jobBatch.SyncSide, entityBatch.EntityDefinition.TechnicalEntityName, entityBatch.EntityDefinition.UserFriendlyEntityName));
}
foreach (var insertedRecord in entityBatch.RecordsToAdd)
{
if (entityBatch.LoggingBehavior.MaxNumberOfInsertsWithoutErrorToLog.HasValue && !insertedRecord.HasError)
{
countOfInsertsWithoutError++;
if (countOfInsertsWithoutError > entityBatch.LoggingBehavior.MaxNumberOfInsertsWithoutErrorToLog.Value)
{
continue;
}
}
if (insertedRecord.HasError)
{
logLine.Append(errorLogLineMinusMessage);
logLine.AppendLine(string.Format("INSERT failed with error: {0}. Command text is: {1}. Failed values are:",
insertedRecord.ErrorMessage, insertedRecord.CommandText));
}
else if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsAndInfo)
{
if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsInfoAndDebug)
{
logLine.AppendLine(string.Format("Command text successfully executed: {0}", insertedRecord.CommandText));
}
logLine.Append(infoLogLineMinusMessage);
logLine.AppendLine(string.Format("INSERT applied for keys \"{0}\" with values:",
LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, insertedRecord.PrimaryKeyValues.ToList())));
}
StringBuilder insertedMessage = new StringBuilder();
foreach (var fieldValuePair in insertedRecord.FieldValuePairs)
{
insertedMessage.AppendLine(string.Format("{0}{1}: {2}",
PRE_MESSAGES_TABS, fieldValuePair.Key, fieldValuePair.Value == null ? "NULL" : "'" + fieldValuePair.Value + "'"));
}
logLine.AppendLine(insertedMessage.ToString());
WriteAndClearLogLineIfExceedsSize(jobInstance.Integration.Id, log, logLine);
}
foreach (var updatedRecord in entityBatch.RecordsToUpdate)
{
if (entityBatch.LoggingBehavior.MaxNumberOfUpdatesWithoutErrorToLog.HasValue && !updatedRecord.HasError)
{
countOfUpdatesWithoutError++;
if (countOfUpdatesWithoutError > entityBatch.LoggingBehavior.MaxNumberOfUpdatesWithoutErrorToLog.Value)
{
continue;
}
}
if (updatedRecord.HasError)
{
logLine.Append(errorLogLineMinusMessage);
logLine.AppendLine(string.Format("UPDATE with keys '{0}' failed with error: {1}. Command text is: {2}. Failed values are:",
LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, updatedRecord.PrimaryKeyValues.ToList()), updatedRecord.ErrorMessage,
updatedRecord.CommandText));
}
else if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsAndInfo)
{
if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsInfoAndDebug)
{
logLine.AppendLine(string.Format("Command text successfully executed: {0}", updatedRecord.CommandText));
}
logLine.Append(infoLogLineMinusMessage);
logLine.AppendLine(string.Format("UPDATE applied for keys \"{0}\" with values:",
LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, updatedRecord.PrimaryKeyValues.ToList())));
}
StringBuilder updatedMessage = new StringBuilder();
foreach (var fieldValuePair in updatedRecord.FieldValuePairs)
{
updatedMessage.AppendLine(string.Format("{0}{1}: {2} -> {3}",
PRE_MESSAGES_TABS, fieldValuePair.Key,
fieldValuePair.Value.OldValue == null ? "NULL" : "'" + fieldValuePair.Value.OldValue + "'",
fieldValuePair.Value.NewValue == null ? "NULL" : "'" + fieldValuePair.Value.NewValue + "'"));
}
logLine.Append(updatedMessage.ToString());
WriteAndClearLogLineIfExceedsSize(jobInstance.Integration.Id, log, logLine);
}
foreach (var deletedRecord in entityBatch.RecordsToDelete)
{
if (entityBatch.LoggingBehavior.MaxNumberOfDeletionsWithoutErrorToLog.HasValue && !deletedRecord.HasError)
{
countOfDeletionsWithoutError++;
if (countOfDeletionsWithoutError > entityBatch.LoggingBehavior.MaxNumberOfDeletionsWithoutErrorToLog.Value)
{
continue;
}
}
if (deletedRecord.HasError)
{
logLine.Append(errorLogLineMinusMessage);
logLine.AppendLine(string.Format("DELETE with keys \"{0}\" failed with error: {1}.",
LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, deletedRecord.PrimaryKeyValues.ToList()), deletedRecord.ErrorMessage));
}
else if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsAndInfo)
{
if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsInfoAndDebug)
{
logLine.AppendLine(string.Format("Command text successfully executed: {0}", deletedRecord.CommandText));
}
logLine.Append(infoLogLineMinusMessage);
logLine.AppendLine(string.Format("DELETE applied for keys '{0}'.",
LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, deletedRecord.PrimaryKeyValues.ToList())));
}
WriteAndClearLogLineIfExceedsSize(jobInstance.Integration.Id, log, logLine);
}
}
log.Write(logLine);
log.Flush();
log.Close();
}
break;
}
catch (IOException)
{
if (i == NUMBER_OF_FILEACCESS_RETRIES)
{
throw;
}
else
{
Thread.Sleep(DELAY_ON_FILEACCES_RETRY);
}
}
}
}
}
public static void WriteToLog(JobInstance jobInstance, JobStepInstance jobStepInstance, JobBatch jobBatch)
{
foreach (var logger in registeredLoggers)
{
try
{
logger.Logger.WriteToLog(jobInstance, jobStepInstance, jobBatch);
}
catch (Exception ex)
{
WriteExceptionToLog(jobInstance.Integration, jobBatch.AssociatedDataSource.DataSource, jobInstance, jobStepInstance, ex);
}
}
}
public Resource Add(JobBatch batch)
{
Cpu.Add(batch.Cpu * batch.Size, batch.BeginTime, batch.Duration);
Mem.Add(batch.Mem * batch.Size, batch.BeginTime, batch.Duration);
return(this);
}
public Resource Subtract(JobBatch batch)
{
Cpu.Subtract(batch.Cpu * batch.Size, batch.BeginTime, batch.Duration);
Mem.Subtract(batch.Mem * batch.Size, batch.BeginTime, batch.Duration);
return(this);
}