private void taskManager_TaskFailed(object sender, EventArgs <Tuple <SlaveTask, TaskData, Exception> > e)
{
try {
SlaveStatusInfo.DecrementUsedCores(e.Value.Item1.CoresNeeded);
heartbeatManager.AwakeHeartBeatThread();
SlaveTask slaveTask = e.Value.Item1;
TaskData taskData = e.Value.Item2;
Exception exception = e.Value.Item3;
Task task = wcfService.GetTask(slaveTask.TaskId);
if (task == null)
{
throw new TaskNotFoundException(slaveTask.TaskId);
}
task.ExecutionTime = slaveTask.ExecutionTime;
if (taskData != null)
{
wcfService.UpdateTaskData(task, taskData, configManager.GetClientInfo().Id, TaskState.Failed, exception.ToString());
}
else
{
wcfService.UpdateJobState(task.Id, TaskState.Failed, exception.ToString());
}
SlaveClientCom.Instance.LogMessage(exception.Message);
}
catch (TaskNotFoundException ex) {
SlaveStatusInfo.IncrementTasksFailed();
SlaveClientCom.Instance.LogMessage(ex.ToString());
}
catch (Exception ex) {
SlaveStatusInfo.IncrementTasksFailed();
SlaveClientCom.Instance.LogMessage(ex.ToString());
}
}
public void StartTaskAsync(Task task, TaskData taskData)
{
SlaveTask slaveTask = null;
slaveTasksLocker.EnterUpgradeableReadLock();
try {
if (slaveTasks.ContainsKey(task.Id))
{
SlaveStatusInfo.IncrementTasksFailed();
throw new TaskAlreadyRunningException(task.Id);
}
else
{
slaveTask = new SlaveTask(pluginManager, task.CoresNeeded, log);
AddSlaveTask(task, slaveTask);
SlaveStatusInfo.IncrementTasksFetched();
}
}
finally { slaveTasksLocker.ExitUpgradeableReadLock(); }
if (slaveTask != null)
{
try {
slaveTask.StartJobAsync(task, taskData);
}
catch (Exception) {
RemoveSlaveTask(task.Id, slaveTask); // clean up and rethrow
slaveTask.DisposeAppDomain();
throw;
}
}
}
private void HandleCalculateTask(object taskIdObj)
{
Guid taskId = (Guid)taskIdObj;
Task task = null;
int usedCores = 0;
try {
task = wcfService.GetTask(taskId);
if (task == null)
{
throw new TaskNotFoundException(taskId);
}
if (ConfigManager.Instance.GetFreeCores() < task.CoresNeeded)
{
throw new OutOfCoresException();
}
if (ConfigManager.Instance.GetFreeMemory() < task.MemoryNeeded)
{
throw new OutOfMemoryException();
}
SlaveStatusInfo.IncrementUsedCores(task.CoresNeeded); usedCores = task.CoresNeeded;
TaskData taskData = wcfService.GetTaskData(taskId);
if (taskData == null)
{
throw new TaskDataNotFoundException(taskId);
}
task = wcfService.UpdateJobState(taskId, TaskState.Calculating, null);
if (task == null)
{
throw new TaskNotFoundException(taskId);
}
taskManager.StartTaskAsync(task, taskData);
}
catch (TaskNotFoundException) {
SlaveStatusInfo.DecrementUsedCores(usedCores);
throw;
}
catch (TaskDataNotFoundException) {
SlaveStatusInfo.DecrementUsedCores(usedCores);
throw;
}
catch (TaskAlreadyRunningException) {
SlaveStatusInfo.DecrementUsedCores(usedCores);
throw;
}
catch (OutOfCoresException) {
wcfService.UpdateJobState(taskId, TaskState.Waiting, "No more cores available");
throw;
}
catch (OutOfMemoryException) {
wcfService.UpdateJobState(taskId, TaskState.Waiting, "No more memory available");
throw;
}
catch (Exception e) {
SlaveStatusInfo.DecrementUsedCores(usedCores);
wcfService.UpdateJobState(taskId, TaskState.Failed, e.ToString());
throw;
}
}
private void taskManager_TaskAborted(object sender, EventArgs <SlaveTask> e)
{
var slaveTask = e.Value;
var task = wcfService.GetTask(slaveTask.TaskId);
wcfService.UpdateJobState(task.Id, TaskState.Aborted, null);
SlaveStatusInfo.DecrementUsedCores(e.Value.CoresNeeded);
}
private void AbortTaskAsync(Guid jobId)
{
TS.Task.Factory.StartNew(HandleAbortTask, jobId)
.ContinueWith((t) => {
SlaveStatusInfo.IncrementTasksFailed();
SlaveClientCom.Instance.LogMessage(t.Exception.ToString());
}, TaskContinuationOptions.OnlyOnFaulted);
}
private void slaveTask_TaskStarted(object sender, EventArgs <Guid> e)
{
SlaveTask slaveTask;
slaveTasksLocker.EnterUpgradeableReadLock();
try {
slaveTask = slaveTasks[e.Value].Task;
}
finally { slaveTasksLocker.ExitUpgradeableReadLock(); }
SlaveStatusInfo.IncrementTasksStarted();
OnTaskStarted(slaveTask);
}
private void slaveTask_TaskFailed(object sender, EventArgs <Guid, Exception> e)
{
SlaveTask slaveTask;
slaveTasksLocker.EnterUpgradeableReadLock();
try {
slaveTask = slaveTasks[e.Value].Task;
RemoveSlaveTask(e.Value, slaveTask);
}
finally { slaveTasksLocker.ExitUpgradeableReadLock(); }
TaskData taskData = null;
try {
taskData = slaveTask.GetTaskData();
}
catch { /* taskData will be null */ }
SlaveStatusInfo.IncrementTasksFailed();
OnTaskFailed(slaveTask, taskData, e.Value2);
}
private void taskManager_TaskStopped(object sender, EventArgs <SlaveTask, TaskData> e)
{
try {
SlaveStatusInfo.DecrementUsedCores(e.Value.CoresNeeded);
heartbeatManager.AwakeHeartBeatThread();
Task task = wcfService.GetTask(e.Value.TaskId);
if (task == null)
{
throw new TaskNotFoundException(e.Value.TaskId);
}
task.ExecutionTime = e.Value.ExecutionTime;
TaskData taskData = e.Value2;
wcfService.UpdateTaskData(task, taskData, configManager.GetClientInfo().Id, TaskState.Finished);
}
catch (TaskNotFoundException ex) {
SlaveClientCom.Instance.LogMessage(ex.ToString());
}
catch (Exception ex) {
SlaveClientCom.Instance.LogMessage(ex.ToString());
}
}
public void AbortTask(Guid taskId)
{
SlaveTask slaveTask = null;
slaveTasksLocker.EnterUpgradeableReadLock();
try {
if (!slaveTasks.ContainsKey(taskId))
{
throw new TaskNotRunningException(taskId);
}
slaveTask = slaveTasks[taskId].Task;
if (!slaveTask.IsPrepared)
{
throw new AppDomainNotCreatedException();
}
RemoveSlaveTask(taskId, slaveTask);
}
finally { slaveTasksLocker.ExitUpgradeableReadLock(); }
slaveTask.DisposeAppDomain();
SlaveStatusInfo.IncrementTasksAborted();
OnTaskAborted(slaveTask);
}
private void slaveTask_TaskStopped(object sender, EventArgs <Guid> e)
{
SlaveTask slaveTask;
slaveTasksLocker.EnterUpgradeableReadLock();
try {
slaveTask = slaveTasks[e.Value].Task;
RemoveSlaveTask(e.Value, slaveTask);
}
finally { slaveTasksLocker.ExitUpgradeableReadLock(); }
TaskData taskData = null;
try {
taskData = slaveTask.GetTaskData();
SlaveStatusInfo.IncrementTasksFinished();
OnTaskStopped(slaveTask, taskData);
}
catch (Exception ex) {
RemoveSlaveTask(e.Value, slaveTask);
SlaveStatusInfo.IncrementTasksFailed();
OnTaskFailed(slaveTask, taskData, ex);
}
}
private void taskManager_TaskAborted(object sender, EventArgs <SlaveTask> e)
{
SlaveStatusInfo.DecrementUsedCores(e.Value.CoresNeeded);
}