/// <summary>
/// Gets error type (encoded as TaskStateEvent) based on the exception type in IFailedTask.
/// For unknown exceptions or exceptions that doesn't belong to defined IMRU task exceptions
/// treat then as application error.
/// </summary>
/// <param name="failedTask"></param>
/// <returns></returns>
private TaskStateEvent GetTaskErrorEventByExceptionType(IFailedTask failedTask)
{
var exception = failedTask.AsError();
var innerExceptionType = exception.InnerException != null ? exception.InnerException.GetType().ToString() : "InnerException null";
var innerExceptionMsg = exception.InnerException != null ? exception.InnerException.Message : "No InnerException";
if (failedTask.GetActiveContext().IsPresent())
{
Logger.Log(Level.Info, "GetTaskErrorEventByExceptionType: with task id: {0}, exception type {1}, innerException type {2}, InnerExceptionMessage {3}, evaluator id: {4}",
failedTask.Id,
exception.GetType(),
innerExceptionType,
innerExceptionMsg,
failedTask.GetActiveContext().Value.EvaluatorId);
}
else
{
Logger.Log(Level.Info, "GetTaskErrorEventByExceptionType: with task id: {0}, exception type {1}, innerException type {2}, InnerExceptionMessage {3}",
failedTask.Id,
exception.GetType(),
innerExceptionType,
innerExceptionMsg);
}
if (exception is IMRUTaskAppException)
{
_numberOfAppErrors++;
return TaskStateEvent.FailedTaskAppError;
}
if (exception is IMRUTaskGroupCommunicationException)
{
return TaskStateEvent.FailedTaskCommunicationError;
}
if (exception is IMRUTaskSystemException)
{
return TaskStateEvent.FailedTaskSystemError;
}
// special case for communication error during group communication initialization
if (exception is TaskClientCodeException)
{
// try extract cause and check whether it is InjectionException for GroupCommClient
if (exception.InnerException != null &&
exception.InnerException is InjectionException &&
exception.InnerException.Message.Contains("GroupCommClient"))
{
Logger.Log(Level.Info, "GetTaskErrorEventByExceptionType:FailedTaskCommunicationError with task id {0}", failedTask.Id);
return TaskStateEvent.FailedTaskCommunicationError;
}
}
Logger.Log(Level.Info, "GetTaskErrorEventByExceptionType for un-hanlded exception with task id {0} and exception type {1}", failedTask.Id, exception.GetType());
return TaskStateEvent.FailedTaskSystemError;
}
/// <summary>
/// This method is called when receiving IFailedTask event during task submitting or running
/// Removes the task from running tasks if the task was running
/// Updates the task state to fail based on the error message in the failed task
/// </summary>
/// <param name="failedTask"></param>
internal void RecordFailedTaskDuringRunningOrSubmissionState(IFailedTask failedTask)
{
//// Remove the task from running tasks if it exists there
_runningTasks.Remove(failedTask.Id);
UpdateState(failedTask.Id, GetTaskErrorEventByExceptionType(failedTask));
}
/// <summary>
/// This method is called when receiving IFailedTask event during system shutting down.
/// If the task failed because it receives the close command from driver, update the task state to TaskClosedByDriver.
/// Task could fail by communication error or any other application or system error during this time, as long as it is not
/// TaskFailedByEvaluatorFailure, update the task state based on the error received.
/// </summary>
/// <param name="failedTask"></param>
internal void RecordFailedTaskDuringSystemShuttingDownState(IFailedTask failedTask)
{
Logger.Log(Level.Info, "RecordFailedTaskDuringSystemShuttingDownState, exceptionType: {0}", GetTaskErrorEventByExceptionType(failedTask).ToString());
var taskState = GetTaskState(failedTask.Id);
if (taskState == StateMachine.TaskState.TaskWaitingForClose)
{
UpdateState(failedTask.Id, TaskStateEvent.ClosedTask);
}
else if (taskState != StateMachine.TaskState.TaskFailedByEvaluatorFailure)
{
UpdateState(failedTask.Id, GetTaskErrorEventByExceptionType(failedTask));
}
}
public async Task DispatchFailedTaskEvent(IFailedTask failedTaskEvent)
{
await DispatchAsync(_failedTaskDispatcher, failedTaskEvent);
}
/// <summary>
/// Creates a mock IFailedEvaluator with the specified IFailedTask associated
/// </summary>
/// <param name="evaluatorId"></param>
/// <param name="failedTask"></param>
/// <returns></returns>
private static IFailedEvaluator CreateMockFailedEvaluator(string evaluatorId, IFailedTask failedTask)
{
var failedEvalutor = Substitute.For<IFailedEvaluator>();
failedEvalutor.Id.Returns(evaluatorId);
failedEvalutor.FailedTask.Returns(Optional<IFailedTask>.Of(failedTask));
return failedEvalutor;
}
private string GetEndPoint(IFailedTask failedTask)
{
return(GetEndPointFromTaskId(failedTask.Id) ?? GetEndPointFromContext(failedTask.GetActiveContext()));
}
public void OnNext(IFailedTask value)
{
Logger.Log(Level.Info, FailedTaskMessage + " " + value.Id + " " + value.GetActiveContext().Value.EvaluatorId);
}
public void OnNext(IFailedTask value)
{
}
/// <summary>
/// This method is called when receiving IFailedTask event during task submitting or running
/// Removes the task from running tasks if the task was running
/// Updates the task state to fail based on the error message in the failed task
/// </summary>
/// <param name="failedTask"></param>
internal void RecordFailedTaskDuringRunningOrSubmissionState(IFailedTask failedTask)
{
//// Remove the task from running tasks if it exists there
_runningTasks.Remove(failedTask.Id);
UpdateState(failedTask.Id, GetTaskErrorEventByExceptionType(failedTask));
}
/// <summary>
/// Creates a mock IFailedEvaluator with the specified IFailedTask associated
/// </summary>
/// <param name="evaluatorId"></param>
/// <param name="failedTask"></param>
/// <returns></returns>
private static IFailedEvaluator CreateMockFailedEvaluator(string evaluatorId, IFailedTask failedTask)
{
var failedEvalutor = Substitute.For <IFailedEvaluator>();
failedEvalutor.Id.Returns(evaluatorId);
failedEvalutor.FailedTask.Returns(Optional <IFailedTask> .Of(failedTask));
return(failedEvalutor);
}
public void OnNext(IFailedTask value)
{
throw new Exception("Did not expect Failed Task.");
}
/// <summary>
/// Gets error type based on the exception type in IFailedTask
/// </summary>
/// <param name="failedTask"></param>
/// <returns></returns>
private TaskStateEvent GetTaskErrorEventByExceptionType(IFailedTask failedTask)
{
var exception = failedTask.AsError();
if (exception is IMRUTaskAppException)
{
_numberOfAppErrors++;
return TaskStateEvent.FailedTaskAppError;
}
if (exception is IMRUTaskGroupCommunicationException)
{
return TaskStateEvent.FailedTaskCommunicationError;
}
else
{
return TaskStateEvent.FailedTaskSystemError;
}
}
/// <summary>
/// This method is called when receiving IFailedTask event during system shutting down.
/// If the task failed because it receives the close command from driver, update the task state to TaskClosedByDriver.
/// Task could fail by communication error or any other application or system error during this time, as long as it is not
/// TaskFailedByEvaluatorFailure, update the task state based on the error received.
/// </summary>
/// <param name="failedTask"></param>
internal void RecordFailedTaskDuringSystemShuttingDownState(IFailedTask failedTask)
{
var taskState = GetTaskState(failedTask.Id);
if (taskState == StateMachine.TaskState.TaskWaitingForClose)
{
UpdateState(failedTask.Id, TaskStateEvent.ClosedTask);
}
else if (taskState != StateMachine.TaskState.TaskFailedByEvaluatorFailure)
{
UpdateState(failedTask.Id, GetTaskErrorEventByExceptionType(failedTask));
}
}
public void OnNext(IFailedTask value)
{
throw new Exception(UnexpectedFailedTask);
}