private async Task CreateAndProcessJobsAsync()
{
_log.Info($"IncrementalExtractionService.CreateAndProcessJobsAsync execute: {DateTime.Now}");
//get the last job from DB, use the later time between job.EndTime and _startOptions.StartTime to be the cutoffStartTime
var latestJob = await _jobAndTaskService.GetLatestJobAsync();
// takes the later date from the two
var cutoffStartTime = (latestJob != null && latestJob.EndTime > _startOptions.StartTime) ? latestJob.EndTime : _startOptions.StartTime;
//don't get data that is less than 3 days old, so we don't lose any data
var cutOffEndTime = DateTime.Now.AddHours(-72);
var newJobStartTime = cutoffStartTime;
var newJobEndTime = newJobStartTime.AddHours(_startOptions.TimeIntervalInHours);
while (newJobEndTime <= cutOffEndTime)
{
var success =
await _jobManager.CreateAndProcessJobsAsync(newJobStartTime, newJobEndTime);
if (!success)
{
_log.Info($"IncrementalExtractionService.CreateAndProcessJobsAsync: something went wrong while creating new job startTime-{newJobStartTime}, endTime-{newJobEndTime}, refer to JobManager log entry for more details.");
break;
}
newJobStartTime = newJobEndTime;
newJobEndTime = newJobStartTime.AddHours(_startOptions.TimeIntervalInHours);
}
}
