private DT.TimeStatus GetTimeStatus(IPersistenceManager pm)
{
return(pm.UseTransaction(() => {
var factTaskDao = pm.FactTaskDao;
var factTasks = factTaskDao.GetAll();
var completedTasks = factTaskDao.GetCompletedTasks()
.OrderByDescending(x => x.EndTime)
.Take(LAST_TASKS);
var lastCalculatingTimes = completedTasks
.GroupBy(x => 1)
.Select(x => new {
Min = x.Min(y => y.CalculatingTime),
Max = x.Max(y => y.CalculatingTime),
Avg = (long)x.Average(y => (long?)y.CalculatingTime)
}).FirstOrDefault();
var calculatingTasks = factTasks.Where(x => x.TaskState == TaskState.Calculating);
int count = calculatingTasks.Count() / 3;
return new DT.TimeStatus {
MinCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Min : 0,
MaxCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Max : 0,
AvgWaitingTime = count > 0 ? (long)calculatingTasks.OrderBy(x => x.StartTime).Take(count).Average(x => x.InitialWaitingTime) : 0,
AvgCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Avg : 0,
TotalCpuTime = factTasks.Sum(x => (long?)x.CalculatingTime) ?? 0,
StandardDeviationCalculatingTime = (long)StandardDeviation(completedTasks.Select(x => (double)x.CalculatingTime)),
BeginDate = factTasks.Where(x => x.StartTime.HasValue).OrderBy(x => x.StartTime).Select(x => x.StartTime).FirstOrDefault()
};
}));
}
private void AggregateClientInfos(IPersistenceManager pm, List <FactClientInfo> clientInfos)
{
var factClientInfoDao = pm.FactClientInfoDao;
var last = clientInfos.Last();
var infos = clientInfos.GroupBy(x => x.ClientId).Select(x => new {
NumUsedCores = (int)x.Average(y => y.NumUsedCores),
NumTotalCores = (int)x.Average(y => y.NumTotalCores),
UsedMemory = (int)x.Average(y => y.UsedMemory),
TotalMemory = (int)x.Average(y => y.TotalMemory),
CpuUtilization = Math.Round(x.Average(y => y.CpuUtilization), 2),
IdleTime = x.Sum(y => y.IdleTime),
OfflineTime = x.Sum(y => y.OfflineTime),
UnavailableTime = x.Sum(y => y.UnavailableTime)
}).SingleOrDefault();
if (infos != null)
{
pm.UseTransaction(() => {
last.NumUsedCores = infos.NumUsedCores;
last.NumTotalCores = infos.NumTotalCores;
last.UsedMemory = infos.UsedMemory;
last.TotalMemory = infos.TotalMemory;
last.CpuUtilization = infos.CpuUtilization;
last.IdleTime = infos.IdleTime;
last.OfflineTime = infos.OfflineTime;
last.UnavailableTime = infos.UnavailableTime;
clientInfos.Remove(last);
factClientInfoDao.Delete(clientInfos);
pm.SubmitChanges();
});
}
}
private void UpdateUserPriority(IPersistenceManager pm, DA.UserPriority up)
{
pm.UseTransaction(() => {
up.DateEnqueued = DateTime.Now;
pm.SubmitChanges();
});
}
private IEnumerable<DT.TaskStatus> GetTaskStatus(IPersistenceManager pm) {
return pm.UseTransaction(() => {
var query = pm.DataContext.ExecuteQuery<UserTaskStatus>(SQL_USER_TASK_STATUS).ToList();
return query.Select(uts => new DT.TaskStatus {
User = new DT.User {
Id = uts.UserId.ToString(),
Name = ServiceLocator.Instance.UserManager.GetUserNameById(uts.UserId)
},
CalculatingTasks = uts.CalculatingTasks,
WaitingTasks = uts.WaitingTasks
}).OrderBy(x => x.User.Name);
});
}
private IEnumerable <DT.TaskStatus> GetTaskStatus(IPersistenceManager pm)
{
return(pm.UseTransaction(() => {
var query = pm.DataContext.ExecuteQuery <UserTaskStatus>(SQL_USER_TASK_STATUS).ToList();
return query.Select(uts => new DT.TaskStatus {
User = new DT.User {
Id = uts.UserId.ToString(),
Name = ServiceLocator.Instance.UserManager.GetUserNameById(uts.UserId)
},
CalculatingTasks = uts.CalculatingTasks,
WaitingTasks = uts.WaitingTasks
}).OrderBy(x => x.User.Name);
}));
}
private void AggregateJob(IPersistenceManager pm, Guid id)
{
var jobDao = pm.JobDao;
var factTaskDao = pm.FactTaskDao;
pm.UseTransaction(() => {
if (jobDao.Exists(id))
{
return;
}
var jobTimeData = factTaskDao.GetByJobId(id).GroupBy(x => x.JobId).Select(x => new {
CalculatingTime = x.Sum(y => y.CalculatingTime),
WaitingTime = x.Sum(y => y.WaitingTime),
TransferTime = x.Sum(y => y.TransferTime),
InitialWaitingTime = x.Sum(y => y.InitialWaitingTime),
StartTime = x.Min(y => y.StartTime),
EndTime = x.Max(y => y.EndTime)
}).SingleOrDefault();
if (jobTimeData != null)
{
factTaskDao.DeleteByJobId(id);
factTaskDao.Save(new FactTask {
TaskId = Guid.NewGuid(),
JobId = id,
CalculatingTime = jobTimeData.CalculatingTime,
WaitingTime = jobTimeData.WaitingTime,
TransferTime = jobTimeData.TransferTime,
InitialWaitingTime = jobTimeData.InitialWaitingTime,
StartTime = jobTimeData.StartTime,
EndTime = jobTimeData.EndTime,
NumCalculationRuns = 0,
NumRetries = 0,
CoresRequired = 0,
MemoryRequired = 0,
Priority = 0,
TaskState = TaskState.Finished
});
}
pm.SubmitChanges();
});
}
private void AssignTask(IPersistenceManager pm, DA.Slave slave, Guid taskId)
{
const DA.TaskState transferring = DA.TaskState.Transferring;
DateTime now = DateTime.Now;
var taskDao = pm.TaskDao;
var stateLogDao = pm.StateLogDao;
pm.UseTransaction(() => {
var task = taskDao.GetById(taskId);
stateLogDao.Save(new DA.StateLog {
State = transferring,
DateTime = now,
TaskId = taskId,
SlaveId = slave.ResourceId,
UserId = null,
Exception = null
});
task.State = transferring;
task.LastHeartbeat = now;
pm.SubmitChanges();
});
}
private void AggregateClient(IPersistenceManager pm, Guid clientId, DateTime start, DateTime end, int entries)
{
var factClientInfoDao = pm.FactClientInfoDao;
var clientInfos = pm.UseTransaction(() => factClientInfoDao.GetByClientId(clientId)
.Where(x => x.Time >= start && x.Time <= end)
.OrderBy(x => x.Time)
.ToList()
);
var e = clientInfos.GetEnumerator();
if (!e.MoveNext())
{
return;
}
do
{
var infosToAggregate = new List <FactClientInfo> {
e.Current
};
var prev = e.Current;
while (e.MoveNext() && infosToAggregate.Count() != entries)
{
var cur = e.Current;
if (prev.IsAllowedToCalculate != cur.IsAllowedToCalculate || prev.SlaveState != cur.SlaveState)
{
break;
}
infosToAggregate.Add(cur);
prev = cur;
}
if (infosToAggregate.Count() >= 2)
{
AggregateClientInfos(pm, infosToAggregate);
}
} while (e.Current != null);
}
private void AggregateClient(IPersistenceManager pm, Guid clientId, DateTime start, DateTime end, int entries) {
var factClientInfoDao = pm.FactClientInfoDao;
var clientInfos = pm.UseTransaction(() => factClientInfoDao.GetByClientId(clientId)
.Where(x => x.Time >= start && x.Time <= end)
.OrderBy(x => x.Time)
.ToList()
);
var e = clientInfos.GetEnumerator();
if (!e.MoveNext()) return;
do {
var infosToAggregate = new List<FactClientInfo> { e.Current };
var prev = e.Current;
while (e.MoveNext() && infosToAggregate.Count() != entries) {
var cur = e.Current;
if (prev.IsAllowedToCalculate != cur.IsAllowedToCalculate || prev.SlaveState != cur.SlaveState)
break;
infosToAggregate.Add(cur);
prev = cur;
}
if (infosToAggregate.Count() >= 2) {
AggregateClientInfos(pm, infosToAggregate);
}
} while (e.Current != null);
}
private void AggregateJob(IPersistenceManager pm, Guid id) {
var jobDao = pm.JobDao;
var factTaskDao = pm.FactTaskDao;
pm.UseTransaction(() => {
if (jobDao.Exists(id)) return;
var jobTimeData = factTaskDao.GetByJobId(id).GroupBy(x => x.JobId).Select(x => new {
CalculatingTime = x.Sum(y => y.CalculatingTime),
WaitingTime = x.Sum(y => y.WaitingTime),
TransferTime = x.Sum(y => y.TransferTime),
InitialWaitingTime = x.Sum(y => y.InitialWaitingTime),
StartTime = x.Min(y => y.StartTime),
EndTime = x.Max(y => y.EndTime)
}).SingleOrDefault();
if (jobTimeData != null) {
factTaskDao.DeleteByJobId(id);
factTaskDao.Save(new FactTask {
TaskId = Guid.NewGuid(),
JobId = id,
CalculatingTime = jobTimeData.CalculatingTime,
WaitingTime = jobTimeData.WaitingTime,
TransferTime = jobTimeData.TransferTime,
InitialWaitingTime = jobTimeData.InitialWaitingTime,
StartTime = jobTimeData.StartTime,
EndTime = jobTimeData.EndTime,
NumCalculationRuns = 0,
NumRetries = 0,
CoresRequired = 0,
MemoryRequired = 0,
Priority = 0,
TaskState = TaskState.Finished
});
}
pm.SubmitChanges();
});
}
private void AggregateClientInfos(IPersistenceManager pm, List<FactClientInfo> clientInfos) {
var factClientInfoDao = pm.FactClientInfoDao;
var last = clientInfos.Last();
var infos = clientInfos.GroupBy(x => x.ClientId).Select(x => new {
NumUsedCores = (int)x.Average(y => y.NumUsedCores),
NumTotalCores = (int)x.Average(y => y.NumTotalCores),
UsedMemory = (int)x.Average(y => y.UsedMemory),
TotalMemory = (int)x.Average(y => y.TotalMemory),
CpuUtilization = Math.Round(x.Average(y => y.CpuUtilization), 2),
IdleTime = x.Sum(y => y.IdleTime),
OfflineTime = x.Sum(y => y.OfflineTime),
UnavailableTime = x.Sum(y => y.UnavailableTime)
}).SingleOrDefault();
if (infos != null) {
pm.UseTransaction(() => {
last.NumUsedCores = infos.NumUsedCores;
last.NumTotalCores = infos.NumTotalCores;
last.UsedMemory = infos.UsedMemory;
last.TotalMemory = infos.TotalMemory;
last.CpuUtilization = infos.CpuUtilization;
last.IdleTime = infos.IdleTime;
last.OfflineTime = infos.OfflineTime;
last.UnavailableTime = infos.UnavailableTime;
clientInfos.Remove(last);
factClientInfoDao.Delete(clientInfos);
pm.SubmitChanges();
});
}
}
private void AssignTask(IPersistenceManager pm, DA.Slave slave, Guid taskId) {
const DA.TaskState transferring = DA.TaskState.Transferring;
DateTime now = DateTime.Now;
var taskDao = pm.TaskDao;
var stateLogDao = pm.StateLogDao;
pm.UseTransaction(() => {
var task = taskDao.GetById(taskId);
stateLogDao.Save(new DA.StateLog {
State = transferring,
DateTime = now,
TaskId = taskId,
SlaveId = slave.ResourceId,
UserId = null,
Exception = null
});
task.State = transferring;
task.LastHeartbeat = now;
pm.SubmitChanges();
});
}
/// <summary>
/// Update the progress of each task
/// Checks if all the task sent by heartbeat are supposed to be calculated by this slave
/// </summary>
private IEnumerable <MessageContainer> UpdateTasks(IPersistenceManager pm, Heartbeat heartbeat, bool isAllowedToCalculate)
{
var taskDao = pm.TaskDao;
var jobDao = pm.JobDao;
var assignedJobResourceDao = pm.AssignedJobResourceDao;
var actions = new List <MessageContainer>();
if (heartbeat.JobProgress == null || !heartbeat.JobProgress.Any())
{
return(actions);
}
var jobIdsWithStatisticsPending = jobDao.GetJobIdsByState(DA.JobState.StatisticsPending).ToList();
// select all tasks and statelogs with one query
var taskIds = heartbeat.JobProgress.Select(x => x.Key).ToList();
var taskInfos = pm.UseTransaction(() =>
(from task in taskDao.GetAll()
where taskIds.Contains(task.TaskId)
let lastStateLog = task.StateLogs.OrderByDescending(x => x.DateTime).FirstOrDefault()
select new {
TaskId = task.TaskId,
JobId = task.JobId,
State = task.State,
Command = task.Command,
SlaveId = lastStateLog != null ? lastStateLog.SlaveId : default(Guid)
}).ToList()
);
// process the jobProgresses
foreach (var jobProgress in heartbeat.JobProgress)
{
var progress = jobProgress;
var curTask = taskInfos.SingleOrDefault(x => x.TaskId == progress.Key);
if (curTask == null)
{
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, progress.Key));
LogFactory.GetLogger(this.GetType().Namespace).Log("Task on slave " + heartbeat.SlaveId + " does not exist in DB: " + jobProgress.Key);
}
else if (jobIdsWithStatisticsPending.Contains(curTask.JobId))
{
// parenting job of current task has been requested for deletion (indicated by job state "Statistics Pending")
// update task execution time
pm.UseTransaction(() => {
taskDao.UpdateExecutionTime(curTask.TaskId, progress.Value.TotalMilliseconds);
});
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
LogFactory.GetLogger(this.GetType().Namespace).Log("Abort task " + curTask.TaskId + " on slave " + heartbeat.SlaveId + ". The parenting job " + curTask.JobId + " was requested to be deleted.");
}
else if (curTask.SlaveId == Guid.Empty || curTask.SlaveId != heartbeat.SlaveId)
{
// assigned slave does not match heartbeat
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not supposed to calculate task: " + curTask.TaskId);
}
else if (!isAllowedToCalculate)
{
actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not allowed to calculate any tasks tue to a downtime. The task is paused.");
}
else if (!assignedJobResourceDao.CheckJobGrantedForResource(curTask.JobId, heartbeat.SlaveId))
{
// slaveId (and parent resourceGroupIds) are not among the assigned resources ids for task-parenting job
// this might happen when (a) job-resource assignment has been changed (b) slave is moved to different group
actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not granted to calculate task: " + curTask.TaskId + " of job: " + curTask.JobId);
}
else
{
// update task execution time
pm.UseTransaction(() => {
taskDao.UpdateExecutionTime(curTask.TaskId, progress.Value.TotalMilliseconds);
});
switch (curTask.Command)
{
case DA.Command.Stop:
actions.Add(new MessageContainer(MessageContainer.MessageType.StopTask, curTask.TaskId));
break;
case DA.Command.Pause:
actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
break;
case DA.Command.Abort:
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
break;
}
}
}
return(actions);
}
/// <summary>
/// Update the progress of each task
/// Checks if all the task sent by heartbeat are supposed to be calculated by this slave
/// </summary>
private IEnumerable<MessageContainer> UpdateTasks(IPersistenceManager pm, Heartbeat heartbeat, bool isAllowedToCalculate) {
var taskDao = pm.TaskDao;
var assignedResourceDao = pm.AssignedResourceDao;
var actions = new List<MessageContainer>();
if (heartbeat.JobProgress == null || !heartbeat.JobProgress.Any())
return actions;
if (!isAllowedToCalculate && heartbeat.JobProgress.Count != 0) {
actions.Add(new MessageContainer(MessageContainer.MessageType.PauseAll));
} else {
// select all tasks and statelogs with one query
var taskIds = heartbeat.JobProgress.Select(x => x.Key).ToList();
var taskInfos = pm.UseTransaction(() =>
(from task in taskDao.GetAll()
where taskIds.Contains(task.TaskId)
let lastStateLog = task.StateLogs.OrderByDescending(x => x.DateTime).FirstOrDefault()
select new {
TaskId = task.TaskId,
Command = task.Command,
SlaveId = lastStateLog != null ? lastStateLog.SlaveId : default(Guid)
}).ToList()
);
// process the jobProgresses
foreach (var jobProgress in heartbeat.JobProgress) {
var progress = jobProgress;
var curTask = taskInfos.SingleOrDefault(x => x.TaskId == progress.Key);
if (curTask == null) {
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, progress.Key));
LogFactory.GetLogger(this.GetType().Namespace).Log("Task on slave " + heartbeat.SlaveId + " does not exist in DB: " + jobProgress.Key);
} else {
var slaveId = curTask.SlaveId;
if (slaveId == Guid.Empty || slaveId != heartbeat.SlaveId) {
// assigned slave does not match heartbeat
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not supposed to calculate task: " + curTask.TaskId);
} else if (!assignedResourceDao.TaskIsAllowedToBeCalculatedBySlave(curTask.TaskId, heartbeat.SlaveId)) {
// assigned resources ids of task do not match with slaveId (and parent resourceGroupIds); this might happen when slave is moved to different group
actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
} else {
// update task execution time
pm.UseTransaction(() => {
taskDao.UpdateExecutionTime(curTask.TaskId, progress.Value.TotalMilliseconds);
});
switch (curTask.Command) {
case DA.Command.Stop:
actions.Add(new MessageContainer(MessageContainer.MessageType.StopTask, curTask.TaskId));
break;
case DA.Command.Pause:
actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
break;
case DA.Command.Abort:
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
break;
}
}
}
}
}
return actions;
}
private DT.TimeStatus GetTimeStatus(IPersistenceManager pm) {
return pm.UseTransaction(() => {
var factTaskDao = pm.FactTaskDao;
var factTasks = factTaskDao.GetAll();
var completedTasks = factTaskDao.GetCompletedTasks()
.OrderByDescending(x => x.EndTime)
.Take(LAST_TASKS);
var lastCalculatingTimes = completedTasks
.GroupBy(x => 1)
.Select(x => new {
Min = x.Min(y => y.CalculatingTime),
Max = x.Max(y => y.CalculatingTime),
Avg = (long)x.Average(y => (long?)y.CalculatingTime)
}).FirstOrDefault();
var calculatingTasks = factTasks.Where(x => x.TaskState == TaskState.Calculating);
int count = calculatingTasks.Count() / 3;
return new DT.TimeStatus {
MinCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Min : 0,
MaxCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Max : 0,
AvgWaitingTime = count > 0 ? (long)calculatingTasks.OrderBy(x => x.StartTime).Take(count).Average(x => x.InitialWaitingTime) : 0,
AvgCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Avg : 0,
TotalCpuTime = factTasks.Sum(x => Convert.ToInt64(x.CalculatingTime)),
StandardDeviationCalculatingTime = (long)StandardDeviation(completedTasks.Select(x => (double)x.CalculatingTime)),
BeginDate = factTasks.Where(x => x.StartTime.HasValue).OrderBy(x => x.StartTime).Select(x => x.StartTime).FirstOrDefault()
};
});
}
/// <summary>
/// Update the progress of each task
/// Checks if all the task sent by heartbeat are supposed to be calculated by this slave
/// </summary>
private IEnumerable <MessageContainer> UpdateTasks(IPersistenceManager pm, Heartbeat heartbeat, bool isAllowedToCalculate)
{
var taskDao = pm.TaskDao;
var assignedResourceDao = pm.AssignedResourceDao;
var actions = new List <MessageContainer>();
if (heartbeat.JobProgress == null || !heartbeat.JobProgress.Any())
{
return(actions);
}
if (!isAllowedToCalculate && heartbeat.JobProgress.Count != 0)
{
actions.Add(new MessageContainer(MessageContainer.MessageType.PauseAll));
}
else
{
// select all tasks and statelogs with one query
var taskIds = heartbeat.JobProgress.Select(x => x.Key).ToList();
var taskInfos = pm.UseTransaction(() =>
(from task in taskDao.GetAll()
where taskIds.Contains(task.TaskId)
let lastStateLog = task.StateLogs.OrderByDescending(x => x.DateTime).FirstOrDefault()
select new {
TaskId = task.TaskId,
Command = task.Command,
SlaveId = lastStateLog != null ? lastStateLog.SlaveId : default(Guid)
}).ToList()
);
// process the jobProgresses
foreach (var jobProgress in heartbeat.JobProgress)
{
var progress = jobProgress;
var curTask = taskInfos.SingleOrDefault(x => x.TaskId == progress.Key);
if (curTask == null)
{
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, progress.Key));
LogFactory.GetLogger(this.GetType().Namespace).Log("Task on slave " + heartbeat.SlaveId + " does not exist in DB: " + jobProgress.Key);
}
else
{
var slaveId = curTask.SlaveId;
if (slaveId == Guid.Empty || slaveId != heartbeat.SlaveId)
{
// assigned slave does not match heartbeat
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not supposed to calculate task: " + curTask.TaskId);
}
else if (!assignedResourceDao.TaskIsAllowedToBeCalculatedBySlave(curTask.TaskId, heartbeat.SlaveId))
{
// assigned resources ids of task do not match with slaveId (and parent resourceGroupIds); this might happen when slave is moved to different group
actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
}
else
{
// update task execution time
pm.UseTransaction(() => {
taskDao.UpdateExecutionTime(curTask.TaskId, progress.Value.TotalMilliseconds);
});
switch (curTask.Command)
{
case DA.Command.Stop:
actions.Add(new MessageContainer(MessageContainer.MessageType.StopTask, curTask.TaskId));
break;
case DA.Command.Pause:
actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
break;
case DA.Command.Abort:
actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
break;
}
}
}
}
}
return(actions);
}
private void UpdateUserPriority(IPersistenceManager pm, DA.UserPriority up) {
pm.UseTransaction(() => {
up.DateEnqueued = DateTime.Now;
pm.SubmitChanges();
});
}