public void Fetch()
{
VC.Setup(_virtualCluster, VC.NoProxy, null);
using (DatabaseObjectsDataContext context = new DatabaseObjectsDataContext(Properties.Settings.Default.JobStatisticsConnectionString))
{
#if DEBUG
context.Log = Console.Out;
#endif
int lookbackDays = Properties.Settings.Default.LookbackDays;
// load all data stored in database that submit time is not earlier than X days ago
var storedJobStatistics = context.JobStatistics
.Where(js => js.SubmitTime != null && js.SubmitTime >= DateTime.Now.AddDays(-lookbackDays))
.ToDictionary(js => js.Id);
// get jobs from server (Cosmos)
List <JobInfo> serverJobs = new List <JobInfo>();
var jobList = VC.GetJobsList(_virtualCluster);
serverJobs.AddRange(jobList);
// find out new jobs and get job statistic information for new jobs.
var newJobs = serverJobs.Where(job => !storedJobStatistics.ContainsKey(job.ID)).ToList();
var newJobBatches = SplitJobsToBatches(newJobs);
int finishedNewBatchCount = 0;
foreach (var jobBatch in newJobBatches)
{
var stats = new List <JobStatistic>();
foreach (var job in jobBatch)
{
var statistic = GetJobStatistic(job);
if (statistic != null)
{
stats.Add(statistic);
}
}
if (stats.Any())
{
context.JobStatistics.InsertAllOnSubmit(stats);
}
context.SubmitChanges();
finishedNewBatchCount++;
Console.WriteLine("{0}/{1} new job batches has been processed", finishedNewBatchCount, newJobBatches.Count);
// sleep for a while to avoid impact cosmos server too much
System.Threading.Thread.Sleep(5000);
}
// find out updated jobs and get updated job statistic information
var updatedJobs = serverJobs.Where(job =>
{
if (!storedJobStatistics.ContainsKey(job.ID))
{
return(false);
}
var storedJobStatistic = storedJobStatistics[job.ID];
JobInfo.JobState jobState = (JobInfo.JobState)storedJobStatistic.State;
if (IsJobStateFinal(jobState))
{
return(false);
}
if (jobState == JobInfo.JobState.Queued && job.State == JobInfo.JobState.Queued)
{
return(false);
}
return(true);
}).ToList();
var updatedJobBatches = SplitJobsToBatches(updatedJobs);
int finishedUpdatingBatchCount = 0;
foreach (var jobBatch in updatedJobBatches)
{
foreach (var job in jobBatch)
{
var storedJobStatistic = storedJobStatistics[job.ID];
var newJobStatistic = GetJobStatistic(job);
if (newJobStatistic != null)
{
UpdateJobStatistic(storedJobStatistic, newJobStatistic);
}
}
context.SubmitChanges();
finishedUpdatingBatchCount++;
Console.WriteLine("{0}/{1} updating job batches has been processed", finishedUpdatingBatchCount, updatedJobBatches.Count);
// sleep for a while to avoid impact cosmos server too much
System.Threading.Thread.Sleep(5000);
}
}
}
private bool IsJobStateFinal(JobInfo.JobState state)
{
return(state != JobInfo.JobState.Queued && state != JobInfo.JobState.Running);
}
