public async Task CreateJobBatchAsync_assigns_new_JobBatchId_when_attempt_to_save_fails(int iterations) { var jobBatch = new JobBatch(); int count = 0; _testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache .Setup(jobBatchCache => jobBatchCache.TryAddValue(It.IsAny <Guid>(), jobBatch)) .Returns((Guid guid, JobBatch batch) => count++ < iterations ? false : true) .Verifiable(); var memoryCacheJobBatchRepository = _testMemoryCacheJobBatchRepositoryFactory.CreateMemoryCacheJobBatchRepository(); Assert.Equal(Guid.Empty, jobBatch.JobBatchId); Assert.Same(jobBatch, await memoryCacheJobBatchRepository.CreateJobBatchAsync(jobBatch)); Assert.NotEqual(Guid.Empty, jobBatch.JobBatchId); _testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache.Verify( jobBatchCache => jobBatchCache.TryAddValue( It.Is <Guid>(guid => guid != jobBatch.JobBatchId), jobBatch), Times.Exactly(iterations)); _testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache.Verify( jobBatchCache => jobBatchCache.TryAddValue(jobBatch.JobBatchId, jobBatch), Times.Once); }
public async Task GetJobBatchAsync_return_JobBatch_when_present_in_cache() { JobBatch jobBatch = new JobBatch { JobBatchId = Guid.NewGuid() }; _testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache .Setup(jobBatchCache => jobBatchCache.TryGetValue( jobBatch.JobBatchId, out It.Ref <JobBatch> .IsAny)) .Callback(new GetCachedJobBatch((Guid batchId, out JobBatch jobBatchToGet) => { jobBatchToGet = jobBatch; })) .Returns(true) .Verifiable(); var memoryCacheJobBatchRepository = _testMemoryCacheJobBatchRepositoryFactory.CreateMemoryCacheJobBatchRepository(); Assert.Same(jobBatch, await memoryCacheJobBatchRepository.GetJobBatchAsync(jobBatch.JobBatchId)); _testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache .Verify( jobBatchCache => jobBatchCache.TryGetValue( jobBatch.JobBatchId, out It.Ref <JobBatch> .IsAny), Times.Once); }
public override void ProcessBatch(JobBatch jobBatch) { SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Info, this.AssociatedDataSource, "Processing {0} entity batch(es) for sync side '{1}'.", jobBatch.EntityBatches.Count, Enum.GetName(typeof(SyncSide), jobBatch.SyncSide)); foreach (EntityBatch entityBatch in jobBatch.EntityBatches) { ProcessBatch(entityBatch); } }
public Task <JobBatch> UpdateJobBatchAsync(JobBatch jobBatch) { if (jobBatch == null) { throw new ArgumentNullException(nameof(jobBatch)); } jobBatch = _jobBatchCache.SetValue( jobBatch.JobBatchId, jobBatch); return(Task.FromResult(jobBatch)); }
public void TestScheduleInBatch() { JobBatch.Do(() => { for (var i = 0; i < 100000; i++) { MockJob.Publish(); } }); StopWorkers(); Assert.AreEqual(100000, _counter.Value); }
public async Task <JobBatch> CreateJobBatchAsync(CreateJobBatchRequest createJobBatchRequest) { if (createJobBatchRequest == null) { throw new ArgumentNullException(nameof(createJobBatchRequest)); } _jobBatchServiceLogger.LogInformation( BlueshiftJobsResources.CreatingJobBatch, createJobBatchRequest.JobBatchRequestor, createJobBatchRequest.JobBatchDescription, createJobBatchRequest.BatchingJobFilter.MaximumItems); try { IReadOnlyCollection <Job> jobs = await _jobRepository .GetJobsAsync(createJobBatchRequest.BatchingJobFilter) .ConfigureAwait(false); var jobBatch = new JobBatch { JobBatchOwnerId = createJobBatchRequest.JobBatchRequestor, JobBatchDescription = createJobBatchRequest.JobBatchDescription } .WithJobs(jobs); jobBatch = await _jobBatchRepository .CreateJobBatchAsync(jobBatch) .ConfigureAwait(false); _jobBatchServiceLogger.LogInformation( BlueshiftJobsResources.CreatedJobBatch, jobBatch.JobBatchOwnerId, jobBatch.JobBatchDescription, jobBatch.Jobs.Count); return(jobBatch); } catch (Exception e) { _jobBatchServiceLogger.LogError( BlueshiftJobsResources.ErrorCreatingJobBatch, _jobBatchServiceLogger.IsEnabled(LogLevel.Debug) ? e.StackTrace : e.Message); throw; } }
private JobBatch ProcessJobBatch(SyncSide syncSide, DataSource dataSource, List <EntityBatch> entityBatches) { var jobDataSource = new JobDataSource(syncSide, dataSource); var jobBatch = new JobBatch(syncSide, jobDataSource); foreach (var entityBatch in entityBatches) { jobBatch.EntityBatches.Add(entityBatch); } dataSource.ProcessBatch(jobBatch); return(jobBatch); }
private void Initalize(Int32 numberOfThreads) { if (numberOfThreads <= 0) { throw new ArgumentException("Number of threads must be greater than 0"); } m_NumThreads = numberOfThreads; m_AllTasks = new Dictionary <Action, Job>(); m_ActiveTasks = new List <Job>(42); m_JobBatches = new JobBatch[m_NumThreads]; for (int i = 0; i < m_JobBatches.Length; i++) { m_JobBatches[i] = new JobBatch(); } }
public async Task CreateJobBatchAsync_requests_the_expected_number_of_jobs_and_saves_the_batch(int requestedJobCount) { _testJobBatchServiceFactory.MockJobRepository .Setup(jobRepository => jobRepository.GetJobsAsync(It.IsAny <JobSearchCriteria>())) .ReturnsAsync((JobSearchCriteria jobSearchCriteria) => _testJobBatchServiceFactory.AvailableJobs .Take(jobSearchCriteria.MaximumItems.Value) .ToList() .AsReadOnly() ) .Verifiable(); _testJobBatchServiceFactory.MockJobBatchRepository .Setup(jobBatchRepository => jobBatchRepository.CreateJobBatchAsync(It.IsAny <JobBatch>())) .ReturnsAsync((JobBatch batch) => batch) .Verifiable(); JobBatchService jobBatchService = _testJobBatchServiceFactory.CreateJobBatchService(); var createJobBatchRequest = new CreateJobBatchRequest() { BatchingJobFilter = { MaximumItems = requestedJobCount } }; JobBatch jobBatch = await jobBatchService.CreateJobBatchAsync(createJobBatchRequest); Assert.NotNull(jobBatch); Assert.Equal( _testJobBatchServiceFactory.AvailableJobs.Take(requestedJobCount), jobBatch.Jobs); _testJobBatchServiceFactory.MockJobRepository .Verify( jobRepository => jobRepository.GetJobsAsync( It.Is <JobSearchCriteria>(jobSearchCriteria => jobSearchCriteria.MaximumItems == requestedJobCount)), Times.Once); _testJobBatchServiceFactory.MockJobBatchRepository .Verify( jobBatchRepository => jobBatchRepository.CreateJobBatchAsync(jobBatch), Times.Once); }
public Task <JobBatch> CreateJobBatchAsync(JobBatch jobBatch) { if (jobBatch == null) { throw new ArgumentNullException(nameof(jobBatch)); } Guid jobBatchId; do { jobBatchId = Guid.NewGuid(); }while (!_jobBatchCache.TryAddValue(jobBatchId, jobBatch)); jobBatch.JobBatchId = jobBatchId; return(Task.FromResult(jobBatch)); }
public async Task UpdateJobBatchAsync_sets_value_in_cache() { var jobBatch = new JobBatch { JobBatchId = Guid.NewGuid() }; _testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache .Setup(jobBatchCache => jobBatchCache.SetValue(jobBatch.JobBatchId, jobBatch)) .Returns((Guid guid, JobBatch batch) => batch) .Verifiable(); var memoryCacheJobBatchRepository = _testMemoryCacheJobBatchRepositoryFactory.CreateMemoryCacheJobBatchRepository(); Assert.Same(jobBatch, await memoryCacheJobBatchRepository.UpdateJobBatchAsync(jobBatch)); _testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache.Verify( jobBatchCache => jobBatchCache.SetValue(jobBatch.JobBatchId, jobBatch), Times.Once); }
public async Task CreateJobBatchAsync_assigns_JobBatchId_and_saves_to_cache() { var jobBatch = new JobBatch(); _testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache .Setup(jobBatchCache => jobBatchCache.TryAddValue(It.IsAny <Guid>(), jobBatch)) .Returns(true) .Verifiable(); var memoryCacheJobBatchRepository = _testMemoryCacheJobBatchRepositoryFactory.CreateMemoryCacheJobBatchRepository(); Assert.Equal(Guid.Empty, jobBatch.JobBatchId); Assert.Same(jobBatch, await memoryCacheJobBatchRepository.CreateJobBatchAsync(jobBatch)); Assert.NotEqual(Guid.Empty, jobBatch.JobBatchId); _testMemoryCacheJobBatchRepositoryFactory.MockJobBatchCache.Verify( jobBatchCache => jobBatchCache.TryAddValue(jobBatch.JobBatchId, jobBatch), Times.Once); }
public IActionResult Post(DtoChirpPost dto) { _db.BeginTransaction(); var chirp = new Chirp { UserId = int.Parse(User.Identity.Name ?? "1"), ChirpTimeUtc = DateTime.UtcNow, ChirpType = ChirpType.Chirp, Contents = dto.Contents }; _db.Chirps.Add(chirp); _db.SaveChanges(); // Publish chirp processing job, before commit. // If the publishing fails, the transaction will be rolled back. // // We're using the job publish as the last committing resource because // it does not participate in the DB transaction. JobBatch.Do(() => { TimelineUpdate.Publish(new TimelineUpdateArgs { ChirpId = chirp.Id, AuthorId = chirp.UserId, TimeUtc = chirp.ChirpTimeUtc }); HashTagUpdate.Publish(chirp.Id); }); _db.CommitTransaction(); return(Ok(chirp.Id)); }
public void ProcessBatch(JobBatch jobBatch) { if (jobBatch == null) { throw new Exception("Job batch can not be null."); } if (jobBatch.EntityBatches == null || jobBatch.EntityBatches.Count == 0) { SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Info, () => { return(string.Format("The job batch for sync side '{0}' and data source '{1}' contains no entity batches. No processing has occurred.", Enum.GetName(typeof(SyncSide), jobBatch.SyncSide), jobBatch.AssociatedDataSource.DataSource.Name)); }); } else { using (var adapter = GetAdapterInstance(Adapter)) { adapter.ProcessBatch(jobBatch); } } }
public Job(JobBatch batch) { this.batch = batch; }
public abstract void ProcessBatch(JobBatch jobBatch);
public JobBatchStepOutput(JobBatch sourceSideJobBatch, JobBatch targetSideJobBatch) { SourceSideJobBatch = sourceSideJobBatch; TargetSideJobBatch = targetSideJobBatch; }
public static void ExecuteStep(JobInstance jobInstance, JobStepInstance previousJobStepInstance, JobStepInstance currentJobStepInstance, ISyncEngineConfigurator configurator) { Type jobStepType = jobInstance.Integration.PackageAssembly.GetType(currentJobStepInstance.JobStep.FullyQualifiedName); if (jobStepType == null) { throw new Exception(string.Format("Job step with fully qualified name '{0}' was not found in assembly '{1}'.", currentJobStepInstance.JobStep.FullyQualifiedName, jobInstance.Integration.PackageAssembly.Location)); } // ensure the step class inherits from the proper base class to ensure the Initialize method is available bool hasCorrectBaseType = false; var baseType = jobStepType.BaseType; while (baseType != typeof(Object)) { if (baseType == typeof(JobStepInvocation)) { hasCorrectBaseType = true; break; } baseType = baseType.BaseType; } if (!hasCorrectBaseType) { throw new Exception(string.Format("Job step class '{0}' must derive from '{1}'.", jobStepType.Name, typeof(JobStepInvocation).FullName)); } var jobStepInvocation = Activator.CreateInstance(jobStepType); if (jobStepInvocation is CustomActionStep) { var jobStepObj = (CustomActionStep)jobStepInvocation; jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator); jobStepObj.Process(); if (previousJobStepInstance != null && ((previousJobStepInstance.HasDeferredExecutionUntilNextStep.HasValue && previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true) || (previousJobStepInstance.GetType() is CustomActionStep))) { currentJobStepInstance.SourceJobBatch = previousJobStepInstance.SourceJobBatch; currentJobStepInstance.TargetJobBatch = previousJobStepInstance.TargetJobBatch; currentJobStepInstance.HasDeferredExecutionUntilNextStep = true; } else { currentJobStepInstance.HasDeferredExecutionUntilNextStep = false; } } else if (jobStepInvocation is DataMapStep) { var jobStepObj = (DataMapStep)jobStepInvocation; jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator); var dataMapOutput = jobStepObj.Process(); if (dataMapOutput == null) { throw new Exception("Job step must return a value."); } currentJobStepInstance.HasDeferredExecutionUntilNextStep = dataMapOutput.DeferExecutionUntilNextStep; JobBatch sourceJobBatch = null; JobBatch targetJobBatch = null; EntityBatch sourceEntityBatch = null; EntityBatch targetEntityBatch = null; EntityBatch oneWayEntityBatch = null; IOneWayDataMap oneWayDataMap = null; TwoWayDataMap twoWayDataMap = null; if (dataMapOutput.DataMap is IOneWayDataMap) { oneWayDataMap = (IOneWayDataMap)dataMapOutput.DataMap; if (dataMapOutput.DataMap is OneWayDataMap) { oneWayEntityBatch = OneToOneDataMapProcessor.Compare((OneWayDataMap)dataMapOutput.DataMap, dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior, dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior, dataMapOutput.RowsToProcess); } else if (dataMapOutput.DataMap is OneToMany_OneWayDataMap) { if (dataMapOutput.RowsToProcess != null) { throw new Exception("Rows to process Func is not supported for one-to-many data maps."); } oneWayEntityBatch = OneToManyDataMapProcessor.Compare((OneToMany_OneWayDataMap)dataMapOutput.DataMap, dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior, dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior); } if (oneWayEntityBatch.EntityDefinition.SyncSide == SyncSide.Source) { oneWayEntityBatch.LoggingBehavior = dataMapOutput.SourceSideLoggingBehavior; } else if (oneWayEntityBatch.EntityDefinition.SyncSide == SyncSide.Target) { oneWayEntityBatch.LoggingBehavior = dataMapOutput.TargetSideLoggingBehavior; } } else if (dataMapOutput.DataMap is TwoWayDataMap) { twoWayDataMap = (TwoWayDataMap)dataMapOutput.DataMap; OneToOneDataMapProcessor.Compare(twoWayDataMap, dataMapOutput.SourceData, dataMapOutput.SourceDataDuplicateRowBehavior, dataMapOutput.TargetData, dataMapOutput.TargetDataDuplicateRowBehavior, dataMapOutput.RowsToProcess, out sourceEntityBatch, out targetEntityBatch); sourceEntityBatch.LoggingBehavior = dataMapOutput.SourceSideLoggingBehavior; targetEntityBatch.LoggingBehavior = dataMapOutput.TargetSideLoggingBehavior; } else { throw new DerivedClassNotImplementedException <OneToOneDataMap>(dataMapOutput.DataMap); } if (previousJobStepInstance != null && previousJobStepInstance.HasDeferredExecutionUntilNextStep.HasValue && previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true) { sourceJobBatch = previousJobStepInstance.SourceJobBatch; targetJobBatch = previousJobStepInstance.TargetJobBatch; } else { sourceJobBatch = new JobBatch(SyncSide.Source, jobInstance.SourceDataSource); targetJobBatch = new JobBatch(SyncSide.Target, jobInstance.TargetDataSource); } if (dataMapOutput.DataMap is IOneWayDataMap) { if (oneWayDataMap.SyncDirection == SyncDirection.SourceToTarget) { targetJobBatch.EntityBatches.Add(oneWayEntityBatch); } else if (oneWayDataMap.SyncDirection == SyncDirection.TargetToSource) { sourceJobBatch.EntityBatches.Add(oneWayEntityBatch); } else { throw new EnumValueNotImplementedException <SyncDirection>(oneWayDataMap.SyncDirection); } } else if (dataMapOutput.DataMap is TwoWayDataMap) { sourceJobBatch.EntityBatches.Add(sourceEntityBatch); targetJobBatch.EntityBatches.Add(targetEntityBatch); } else { throw new DerivedClassNotImplementedException <OneToOneDataMap>(dataMapOutput.DataMap); } currentJobStepInstance.SourceJobBatch = sourceJobBatch; currentJobStepInstance.TargetJobBatch = targetJobBatch; if (!currentJobStepInstance.HasDeferredExecutionUntilNextStep.Value) { sourceJobBatch.SubmitToDataSource(); if (sourceJobBatch.HasRecordErrors) { currentJobStepInstance.HasRecordErrors = true; currentJobStepInstance.Exceptions = sourceJobBatch.GetExceptions(); } SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, sourceJobBatch); targetJobBatch.SubmitToDataSource(); if (targetJobBatch.HasRecordErrors) { currentJobStepInstance.HasRecordErrors = true; currentJobStepInstance.Exceptions = targetJobBatch.GetExceptions(); } SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, targetJobBatch); } } else if (jobStepInvocation is EntityBatchStep) { var jobStepObj = (EntityBatchStep)jobStepInvocation; jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator); var entityBatchOutput = jobStepObj.Process(); if (entityBatchOutput == null) { throw new Exception("Job step must return a value."); } currentJobStepInstance.HasDeferredExecutionUntilNextStep = entityBatchOutput.DeferExecutionUntilNextStep; JobBatch sourceJobBatch; JobBatch targetJobBatch; if (previousJobStepInstance != null && previousJobStepInstance.HasDeferredExecutionUntilNextStep.Value == true) { sourceJobBatch = previousJobStepInstance.SourceJobBatch; targetJobBatch = previousJobStepInstance.TargetJobBatch; } else { sourceJobBatch = new JobBatch(SyncSide.Source, jobInstance.SourceDataSource); targetJobBatch = new JobBatch(SyncSide.Target, jobInstance.TargetDataSource); } if (entityBatchOutput.SourceSideEntityBatches != null) { foreach (var sourceSideEntityBatch in entityBatchOutput.SourceSideEntityBatches) { sourceJobBatch.EntityBatches.Add(sourceSideEntityBatch); } } if (entityBatchOutput.TargetSideEntityBatches != null) { foreach (var targetSideEntityBatch in entityBatchOutput.TargetSideEntityBatches) { targetJobBatch.EntityBatches.Add(targetSideEntityBatch); } } currentJobStepInstance.SourceJobBatch = sourceJobBatch; currentJobStepInstance.TargetJobBatch = targetJobBatch; if (!currentJobStepInstance.HasDeferredExecutionUntilNextStep.Value) { sourceJobBatch.SubmitToDataSource(); if (sourceJobBatch.HasRecordErrors) { currentJobStepInstance.HasRecordErrors = true; currentJobStepInstance.Exceptions = sourceJobBatch.GetExceptions(); } SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, sourceJobBatch); targetJobBatch.SubmitToDataSource(); if (targetJobBatch.HasRecordErrors) { currentJobStepInstance.HasRecordErrors = true; currentJobStepInstance.Exceptions = targetJobBatch.GetExceptions(); } SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, targetJobBatch); } } else if (jobStepInvocation is JobBatchStep) { var jobStepObj = (JobBatchStep)jobStepInvocation; jobStepObj.Initialize(jobInstance.Integration, jobInstance, previousJobStepInstance, currentJobStepInstance, configurator); var jobBatchOutput = jobStepObj.Process(); if (jobBatchOutput == null) { throw new Exception("Job step must return a value."); } if (jobBatchOutput.SourceSideJobBatch != null) { jobBatchOutput.SourceSideJobBatch.SubmitToDataSource(); if (jobBatchOutput.SourceSideJobBatch.HasRecordErrors) { currentJobStepInstance.HasRecordErrors = true; currentJobStepInstance.Exceptions = jobBatchOutput.SourceSideJobBatch.GetExceptions(); } SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, jobBatchOutput.SourceSideJobBatch); } else { SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstance, currentJobStepInstance, jobInstance.SourceDataSource.DataSource, "Job step '{0}' for job '{1}' returned a null job batch for source-side data source '{2}'.", currentJobStepInstance.JobStep.Name, jobInstance.Job.Name, jobInstance.SourceDataSource.DataSource.Name); } if (jobBatchOutput.TargetSideJobBatch != null) { jobBatchOutput.TargetSideJobBatch.SubmitToDataSource(); if (jobBatchOutput.TargetSideJobBatch.HasRecordErrors) { currentJobStepInstance.HasRecordErrors = true; currentJobStepInstance.Exceptions = jobBatchOutput.TargetSideJobBatch.GetExceptions(); } SyncEngineLogger.WriteToLog(jobInstance, currentJobStepInstance, jobBatchOutput.TargetSideJobBatch); } else { SyncEngineLogger.WriteToLog(LogEntryType.Info, jobInstance, currentJobStepInstance, jobInstance.TargetDataSource.DataSource, "Job step '{0}' for job '{1}' returned a null job batch for target-side data source '{2}'.", currentJobStepInstance.JobStep.Name, jobInstance.Job.Name, jobInstance.TargetDataSource.DataSource.Name); } } else { throw new DerivedClassNotImplementedException <JobStepInvocation>(jobStepInvocation); } }
public void WriteToLog(JobInstance jobInstance, JobStepInstance jobStepInstance, JobBatch jobBatch) { using (var dbContext = new ls.LoggingDataContext(ConnectionString)) { foreach (var entityBatch in jobBatch.EntityBatches) { int countOfInsertsWithoutError = 0; int countOfUpdatesWithoutError = 0; int countOfDeletionsWithoutError = 0; if (!entityBatch.HasBeenProcessed) { WriteToLog(LogEntryType.Error, jobInstance.Integration, jobBatch.AssociatedDataSource.DataSource, jobInstance, jobStepInstance, "The entity batch has not been processed by a data source. No history can be logged."); continue; } var entityBatchHistoryId = (int)dbContext.AddEntityBatchHistory(jobInstance.Integration.Id, jobInstance.Id, jobStepInstance.Id, jobBatch.AssociatedDataSource.DataSource.Id, Enum.GetName(typeof(SyncSide), jobBatch.SyncSide), entityBatch.EntityDefinition.TechnicalEntityName, entityBatch.EntityDefinition.UserFriendlyEntityName, StringHelper.GetDelimitedString(entityBatch.EntityDefinition.PrimaryKeyColumnNames)) .FirstOrDefault().InsertedID; string batchHistoryTemplate = @"DECLARE @EntityBatchRecordId int INSERT INTO EntityBatchRecordHistory VALUES ({0}, GETDATE(), '{1}', '{2}', '{3}', {4}) SELECT @EntityBatchRecordId = SCOPE_IDENTITY()"; string batchHistoryDetailTemplate = "\n\rINSERT INTO EntityBatchRecordHistoryDetail VALUES (@EntityBatchRecordId, GETDATE(), '{0}', NULL, {1}, {2})"; // log inserts foreach (var insertedRecord in entityBatch.RecordsToAdd) { //var entityBatchHistoryRecordId = (int)dbContext.AddEntityBatchRecordHistory(entityBatchHistoryId, 'I', StringHelper.GetDelimitedStringFromList(insertedRecord.PrimaryKeyValues), // insertedRecord.HasError, insertedRecord.ErrorMessage).FirstOrDefault().InsertedID; if (entityBatch.LoggingBehavior.MaxNumberOfInsertsWithoutErrorToLog.HasValue && !insertedRecord.HasError) { countOfInsertsWithoutError++; if (countOfInsertsWithoutError > entityBatch.LoggingBehavior.MaxNumberOfInsertsWithoutErrorToLog.Value) { continue; } } StringBuilder sqlForInsert = new StringBuilder(); sqlForInsert.AppendFormat(batchHistoryTemplate, entityBatchHistoryId, "I", StringHelper.GetDelimitedString(insertedRecord.PrimaryKeyValues), insertedRecord.HasError, insertedRecord.ErrorMessage == null ? "NULL" : "'" + insertedRecord.ErrorMessage.Replace("'", "''") + "'"); foreach (var fieldValuePair in insertedRecord.FieldValuePairs) { //dbContext.AddEntityBatchRecordHistoryDetail(entityBatchHistoryRecordId, fieldValuePair.Key, null, null, fieldValuePair.Value); sqlForInsert.AppendFormat(batchHistoryDetailTemplate, fieldValuePair.Key, "NULL", fieldValuePair.Value == null ? "NULL" : "'" + fieldValuePair.Value.Replace("'", "''") + "'"); } dbContext.ExecuteCommand(sqlForInsert.ToString()); } // log updates foreach (var updatedRecord in entityBatch.RecordsToUpdate) { //var entityBatchHistoryRecordId = (int)dbContext.AddEntityBatchRecordHistory(entityBatchHistoryId, 'U', StringHelper.GetDelimitedStringFromList(updatedRecord.PrimaryKeyValues), // updatedRecord.HasError, updatedRecord.ErrorMessage).FirstOrDefault().InsertedID; if (entityBatch.LoggingBehavior.MaxNumberOfUpdatesWithoutErrorToLog.HasValue && !updatedRecord.HasError) { countOfUpdatesWithoutError++; if (countOfUpdatesWithoutError > entityBatch.LoggingBehavior.MaxNumberOfUpdatesWithoutErrorToLog.Value) { continue; } } StringBuilder sqlForUpdate = new StringBuilder(); sqlForUpdate.AppendFormat(batchHistoryTemplate, entityBatchHistoryId, "U", StringHelper.GetDelimitedString(updatedRecord.PrimaryKeyValues), updatedRecord.HasError, updatedRecord.ErrorMessage == null ? "NULL" : "'" + updatedRecord.ErrorMessage.Replace("'", "''") + "'"); foreach (var fieldValuePair in updatedRecord.FieldValuePairs) { //dbContext.AddEntityBatchRecordHistoryDetail(entityBatchHistoryRecordId, fieldValuePair.Key, null, fieldValuePair.Value.OldValue, fieldValuePair.Value.NewValue); sqlForUpdate.AppendFormat(batchHistoryDetailTemplate, fieldValuePair.Key, fieldValuePair.Value.OldValue == null ? "NULL" : "'" + fieldValuePair.Value.OldValue.Replace("'", "''") + "'", fieldValuePair.Value.NewValue == null ? "NULL" : "'" + fieldValuePair.Value.NewValue.Replace("'", "''") + "'"); } dbContext.ExecuteCommand(sqlForUpdate.ToString()); } // log deletions foreach (var deletedRecord in entityBatch.RecordsToDelete) { if (entityBatch.LoggingBehavior.MaxNumberOfDeletionsWithoutErrorToLog.HasValue && !deletedRecord.HasError) { countOfDeletionsWithoutError++; if (countOfDeletionsWithoutError > entityBatch.LoggingBehavior.MaxNumberOfDeletionsWithoutErrorToLog.Value) { continue; } } var entityBatchHistoryRecordId = (int)dbContext.AddEntityBatchRecordHistory(entityBatchHistoryId, 'D', StringHelper.GetDelimitedString(deletedRecord.PrimaryKeyValues), deletedRecord.HasError, deletedRecord.ErrorMessage).FirstOrDefault().InsertedID; } } } }
private static string GetLogLine(DateTime logDate, LogEntryType logEntryType, JobInstance jobInstance, JobStepInstance jobStepInstance, JobBatch jobBatch, string message) { StringBuilder fullLine = new StringBuilder(); fullLine.Append(logDate); fullLine.Append("\t"); fullLine.Append(logEntryType); fullLine.Append("\t"); fullLine.Append(string.Format("{0} ({1})", jobInstance.Integration.Name, jobInstance.Integration.Id)); fullLine.Append("\t"); fullLine.Append(string.Format("{0} ({1})", jobBatch.AssociatedDataSource.DataSource.Name, jobBatch.AssociatedDataSource.DataSource.Id)); fullLine.Append("\t"); fullLine.Append(string.Format("{0} ({1})", jobInstance.Job.Name, jobInstance.Job.Id)); fullLine.Append("\t"); fullLine.Append(string.Format("{0} ({1})", jobInstance.QueueRequest.InvocationSourceType, jobInstance.QueueRequest.Id)); fullLine.Append("\t"); fullLine.Append(jobInstance.Id); fullLine.Append("\t"); fullLine.Append(JobFilterHelper.GetTextForDatabase(jobInstance.Filters)); fullLine.Append("\t"); fullLine.Append(string.Format("{0} ({1})", jobStepInstance.JobStep.Name, jobStepInstance.JobStep.Id)); fullLine.Append("\t"); fullLine.Append(jobStepInstance.Id); fullLine.Append("\t"); fullLine.Append(message); return(fullLine.ToString()); }
public void WriteToLog(JobInstance jobInstance, JobStepInstance jobStepInstance, JobBatch jobBatch) { string path = null; for (int i = 0; i <= NUMBER_OF_FILEACCESS_RETRIES; i++) { try { path = GetIntegrationLogPathAndFileName(jobInstance.Integration); break; } catch (IOException) { if (i == NUMBER_OF_FILEACCESS_RETRIES) { throw; } else { Thread.Sleep(DELAY_ON_FILEACCES_RETRY); } } } lock (_lock) { for (int i = 0; i <= NUMBER_OF_FILEACCESS_RETRIES; i++) { try { using (TextWriter log = File.AppendText(path)) { StringBuilder logLine = new StringBuilder(); var now = DateTime.Now; var errorLogLineMinusMessage = GetLogLine(now, LogEntryType.Error, jobInstance, jobStepInstance, jobBatch, null); var warningLogLineMinusMessage = GetLogLine(now, LogEntryType.Warning, jobInstance, jobStepInstance, jobBatch, null); var infoLogLineMinusMessage = GetLogLine(now, LogEntryType.Info, jobInstance, jobStepInstance, jobBatch, null); foreach (var entityBatch in jobBatch.EntityBatches) { int countOfInsertsWithoutError = 0; int countOfUpdatesWithoutError = 0; int countOfDeletionsWithoutError = 0; if (entityBatch.RecordsToAdd.Count == 0 && entityBatch.RecordsToUpdate.Count == 0 && entityBatch.RecordsToDelete.Count == 0) { continue; } if (!entityBatch.HasBeenProcessed) { logLine.Append(warningLogLineMinusMessage); logLine.AppendLine("The entity batch has not been processed by a data source. No history can be logged."); continue; } logLine.Append(infoLogLineMinusMessage); if (entityBatch.EntityDefinition.TechnicalEntityName == entityBatch.EntityDefinition.UserFriendlyEntityName) { logLine.AppendLine(string.Format("Entity batch applied on sync side '{0}' for entity '{1}':", jobBatch.SyncSide, entityBatch.EntityDefinition.TechnicalEntityName)); } else { logLine.AppendLine(string.Format("Entity batch applied on sync side '{0}' for entity '{1}' ({2}):", jobBatch.SyncSide, entityBatch.EntityDefinition.TechnicalEntityName, entityBatch.EntityDefinition.UserFriendlyEntityName)); } foreach (var insertedRecord in entityBatch.RecordsToAdd) { if (entityBatch.LoggingBehavior.MaxNumberOfInsertsWithoutErrorToLog.HasValue && !insertedRecord.HasError) { countOfInsertsWithoutError++; if (countOfInsertsWithoutError > entityBatch.LoggingBehavior.MaxNumberOfInsertsWithoutErrorToLog.Value) { continue; } } if (insertedRecord.HasError) { logLine.Append(errorLogLineMinusMessage); logLine.AppendLine(string.Format("INSERT failed with error: {0}. Command text is: {1}. Failed values are:", insertedRecord.ErrorMessage, insertedRecord.CommandText)); } else if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsAndInfo) { if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsInfoAndDebug) { logLine.AppendLine(string.Format("Command text successfully executed: {0}", insertedRecord.CommandText)); } logLine.Append(infoLogLineMinusMessage); logLine.AppendLine(string.Format("INSERT applied for keys \"{0}\" with values:", LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, insertedRecord.PrimaryKeyValues.ToList()))); } StringBuilder insertedMessage = new StringBuilder(); foreach (var fieldValuePair in insertedRecord.FieldValuePairs) { insertedMessage.AppendLine(string.Format("{0}{1}: {2}", PRE_MESSAGES_TABS, fieldValuePair.Key, fieldValuePair.Value == null ? "NULL" : "'" + fieldValuePair.Value + "'")); } logLine.AppendLine(insertedMessage.ToString()); WriteAndClearLogLineIfExceedsSize(jobInstance.Integration.Id, log, logLine); } foreach (var updatedRecord in entityBatch.RecordsToUpdate) { if (entityBatch.LoggingBehavior.MaxNumberOfUpdatesWithoutErrorToLog.HasValue && !updatedRecord.HasError) { countOfUpdatesWithoutError++; if (countOfUpdatesWithoutError > entityBatch.LoggingBehavior.MaxNumberOfUpdatesWithoutErrorToLog.Value) { continue; } } if (updatedRecord.HasError) { logLine.Append(errorLogLineMinusMessage); logLine.AppendLine(string.Format("UPDATE with keys '{0}' failed with error: {1}. Command text is: {2}. Failed values are:", LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, updatedRecord.PrimaryKeyValues.ToList()), updatedRecord.ErrorMessage, updatedRecord.CommandText)); } else if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsAndInfo) { if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsInfoAndDebug) { logLine.AppendLine(string.Format("Command text successfully executed: {0}", updatedRecord.CommandText)); } logLine.Append(infoLogLineMinusMessage); logLine.AppendLine(string.Format("UPDATE applied for keys \"{0}\" with values:", LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, updatedRecord.PrimaryKeyValues.ToList()))); } StringBuilder updatedMessage = new StringBuilder(); foreach (var fieldValuePair in updatedRecord.FieldValuePairs) { updatedMessage.AppendLine(string.Format("{0}{1}: {2} -> {3}", PRE_MESSAGES_TABS, fieldValuePair.Key, fieldValuePair.Value.OldValue == null ? "NULL" : "'" + fieldValuePair.Value.OldValue + "'", fieldValuePair.Value.NewValue == null ? "NULL" : "'" + fieldValuePair.Value.NewValue + "'")); } logLine.Append(updatedMessage.ToString()); WriteAndClearLogLineIfExceedsSize(jobInstance.Integration.Id, log, logLine); } foreach (var deletedRecord in entityBatch.RecordsToDelete) { if (entityBatch.LoggingBehavior.MaxNumberOfDeletionsWithoutErrorToLog.HasValue && !deletedRecord.HasError) { countOfDeletionsWithoutError++; if (countOfDeletionsWithoutError > entityBatch.LoggingBehavior.MaxNumberOfDeletionsWithoutErrorToLog.Value) { continue; } } if (deletedRecord.HasError) { logLine.Append(errorLogLineMinusMessage); logLine.AppendLine(string.Format("DELETE with keys \"{0}\" failed with error: {1}.", LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, deletedRecord.PrimaryKeyValues.ToList()), deletedRecord.ErrorMessage)); } else if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsAndInfo) { if (jobInstance.Integration.LoggingLevel >= LoggingLevel.ErrorsWarningsInfoAndDebug) { logLine.AppendLine(string.Format("Command text successfully executed: {0}", deletedRecord.CommandText)); } logLine.Append(infoLogLineMinusMessage); logLine.AppendLine(string.Format("DELETE applied for keys '{0}'.", LogHelper.GetKeysAndValuesAsText(entityBatch.EntityDefinition.PrimaryKeyColumnNames, deletedRecord.PrimaryKeyValues.ToList()))); } WriteAndClearLogLineIfExceedsSize(jobInstance.Integration.Id, log, logLine); } } log.Write(logLine); log.Flush(); log.Close(); } break; } catch (IOException) { if (i == NUMBER_OF_FILEACCESS_RETRIES) { throw; } else { Thread.Sleep(DELAY_ON_FILEACCES_RETRY); } } } } }
public static void WriteToLog(JobInstance jobInstance, JobStepInstance jobStepInstance, JobBatch jobBatch) { foreach (var logger in registeredLoggers) { try { logger.Logger.WriteToLog(jobInstance, jobStepInstance, jobBatch); } catch (Exception ex) { WriteExceptionToLog(jobInstance.Integration, jobBatch.AssociatedDataSource.DataSource, jobInstance, jobStepInstance, ex); } } }
public Resource Add(JobBatch batch) { Cpu.Add(batch.Cpu * batch.Size, batch.BeginTime, batch.Duration); Mem.Add(batch.Mem * batch.Size, batch.BeginTime, batch.Duration); return(this); }
public Resource Subtract(JobBatch batch) { Cpu.Subtract(batch.Cpu * batch.Size, batch.BeginTime, batch.Duration); Mem.Subtract(batch.Mem * batch.Size, batch.BeginTime, batch.Duration); return(this); }