private List <string> GetUserAliasesForTracking()
{
using (var context = new DatabaseObjectsDataContext(_databaseConnectionString))
{
return(context.UserAliasesForTrackings.Select(u => u.Alias.ToLowerInvariant()).ToList());
}
}
private List <JobStatistic> GetJobStatistics(
AlertingRule rule,
IEnumerable <Guid> jobIds,
IEnumerable <string> jobNameFilters,
IDictionary <string, string> userAliases,
int lookbackDays)
{
using (var context = new DatabaseObjectsDataContext(_databaseConnectionString))
{
DateTime minimumJobEndTime = DateTime.UtcNow.AddDays(-lookbackDays);
return(jobIds
.Select(id => context.JobStatistics.Where(stat => stat.Id == id).First())
.Where(jobStat => jobStat.EndTime == null || jobStat.EndTime > minimumJobEndTime)
.Where(jobStat => !rule.ShouldFilterJobName || jobNameFilters.All(filter => jobStat.Name.IndexOf(filter, StringComparison.InvariantCultureIgnoreCase) < 0))
.Where(jobStat => userAliases.ContainsKey(jobStat.TrueUserName.ToLowerInvariant()))
.ToList());
}
}
private static void UpdateAliasesInDatabase(IEnumerable <string> uniqueAliases)
{
using (DatabaseObjectsDataContext context = new DatabaseObjectsDataContext(Properties.Settings.Default.JobStatisticsConnectionString))
{
#if DEBUG
context.Log = Console.Out;
#endif
// find out all records to be deleted
var uniqueAliasesDictionary = uniqueAliases.ToDictionary(a => a);
var allAliasesToBeDeleted = context.UserAliasesForTrackings
.ToList()
.Except(uniqueAliases.Select(a => new UserAliasesForTracking()
{
Alias = a
}), new UserAliasesForTrackingEquityComparer())
.ToList();
context.UserAliasesForTrackings.DeleteAllOnSubmit(allAliasesToBeDeleted);
// find out all records to be added
var allAliasesToBeAdded = uniqueAliases
.Except(context.UserAliasesForTrackings.Select(a => a.Alias))
.Select(a => new UserAliasesForTracking()
{
Alias = a
})
.ToList();
context.UserAliasesForTrackings.InsertAllOnSubmit(allAliasesToBeAdded);
// submit changes
context.SubmitChanges();
}
}
public void Fetch()
{
VC.Setup(_virtualCluster, VC.NoProxy, null);
using (DatabaseObjectsDataContext context = new DatabaseObjectsDataContext(Properties.Settings.Default.JobStatisticsConnectionString))
{
#if DEBUG
context.Log = Console.Out;
#endif
int lookbackDays = Properties.Settings.Default.LookbackDays;
// load all data stored in database that submit time is not earlier than X days ago
var storedJobStatistics = context.JobStatistics
.Where(js => js.SubmitTime != null && js.SubmitTime >= DateTime.Now.AddDays(-lookbackDays))
.ToDictionary(js => js.Id);
// get jobs from server (Cosmos)
List <JobInfo> serverJobs = new List <JobInfo>();
var jobList = VC.GetJobsList(_virtualCluster);
serverJobs.AddRange(jobList);
// find out new jobs and get job statistic information for new jobs.
var newJobs = serverJobs.Where(job => !storedJobStatistics.ContainsKey(job.ID)).ToList();
var newJobBatches = SplitJobsToBatches(newJobs);
int finishedNewBatchCount = 0;
foreach (var jobBatch in newJobBatches)
{
var stats = new List <JobStatistic>();
foreach (var job in jobBatch)
{
var statistic = GetJobStatistic(job);
if (statistic != null)
{
stats.Add(statistic);
}
}
if (stats.Any())
{
context.JobStatistics.InsertAllOnSubmit(stats);
}
context.SubmitChanges();
finishedNewBatchCount++;
Console.WriteLine("{0}/{1} new job batches has been processed", finishedNewBatchCount, newJobBatches.Count);
// sleep for a while to avoid impact cosmos server too much
System.Threading.Thread.Sleep(5000);
}
// find out updated jobs and get updated job statistic information
var updatedJobs = serverJobs.Where(job =>
{
if (!storedJobStatistics.ContainsKey(job.ID))
{
return(false);
}
var storedJobStatistic = storedJobStatistics[job.ID];
JobInfo.JobState jobState = (JobInfo.JobState)storedJobStatistic.State;
if (IsJobStateFinal(jobState))
{
return(false);
}
if (jobState == JobInfo.JobState.Queued && job.State == JobInfo.JobState.Queued)
{
return(false);
}
return(true);
}).ToList();
var updatedJobBatches = SplitJobsToBatches(updatedJobs);
int finishedUpdatingBatchCount = 0;
foreach (var jobBatch in updatedJobBatches)
{
foreach (var job in jobBatch)
{
var storedJobStatistic = storedJobStatistics[job.ID];
var newJobStatistic = GetJobStatistic(job);
if (newJobStatistic != null)
{
UpdateJobStatistic(storedJobStatistic, newJobStatistic);
}
}
context.SubmitChanges();
finishedUpdatingBatchCount++;
Console.WriteLine("{0}/{1} updating job batches has been processed", finishedUpdatingBatchCount, updatedJobBatches.Count);
// sleep for a while to avoid impact cosmos server too much
System.Threading.Thread.Sleep(5000);
}
}
}
