public void Fetch() { VC.Setup(_virtualCluster, VC.NoProxy, null); using (DatabaseObjectsDataContext context = new DatabaseObjectsDataContext(Properties.Settings.Default.JobStatisticsConnectionString)) { #if DEBUG context.Log = Console.Out; #endif int lookbackDays = Properties.Settings.Default.LookbackDays; // load all data stored in database that submit time is not earlier than X days ago var storedJobStatistics = context.JobStatistics .Where(js => js.SubmitTime != null && js.SubmitTime >= DateTime.Now.AddDays(-lookbackDays)) .ToDictionary(js => js.Id); // get jobs from server (Cosmos) List <JobInfo> serverJobs = new List <JobInfo>(); var jobList = VC.GetJobsList(_virtualCluster); serverJobs.AddRange(jobList); // find out new jobs and get job statistic information for new jobs. var newJobs = serverJobs.Where(job => !storedJobStatistics.ContainsKey(job.ID)).ToList(); var newJobBatches = SplitJobsToBatches(newJobs); int finishedNewBatchCount = 0; foreach (var jobBatch in newJobBatches) { var stats = new List <JobStatistic>(); foreach (var job in jobBatch) { var statistic = GetJobStatistic(job); if (statistic != null) { stats.Add(statistic); } } if (stats.Any()) { context.JobStatistics.InsertAllOnSubmit(stats); } context.SubmitChanges(); finishedNewBatchCount++; Console.WriteLine("{0}/{1} new job batches has been processed", finishedNewBatchCount, newJobBatches.Count); // sleep for a while to avoid impact cosmos server too much System.Threading.Thread.Sleep(5000); } // find out updated jobs and get updated job statistic information var updatedJobs = serverJobs.Where(job => { if (!storedJobStatistics.ContainsKey(job.ID)) { return(false); } var storedJobStatistic = storedJobStatistics[job.ID]; JobInfo.JobState jobState = (JobInfo.JobState)storedJobStatistic.State; if (IsJobStateFinal(jobState)) { return(false); } if (jobState == JobInfo.JobState.Queued && job.State == JobInfo.JobState.Queued) { return(false); } return(true); }).ToList(); var updatedJobBatches = SplitJobsToBatches(updatedJobs); int finishedUpdatingBatchCount = 0; foreach (var jobBatch in updatedJobBatches) { foreach (var job in jobBatch) { var storedJobStatistic = storedJobStatistics[job.ID]; var newJobStatistic = GetJobStatistic(job); if (newJobStatistic != null) { UpdateJobStatistic(storedJobStatistic, newJobStatistic); } } context.SubmitChanges(); finishedUpdatingBatchCount++; Console.WriteLine("{0}/{1} updating job batches has been processed", finishedUpdatingBatchCount, updatedJobBatches.Count); // sleep for a while to avoid impact cosmos server too much System.Threading.Thread.Sleep(5000); } } }
private bool IsJobStateFinal(JobInfo.JobState state) { return(state != JobInfo.JobState.Queued && state != JobInfo.JobState.Running); }