/// <summary>
/// Receives and routes heartbeats from Evaluators.
/// </summary>
/// <param name="evaluatorHearBeatProto"></param>
private void Handle(IRemoteMessage <EvaluatorHeartbeatProto> evaluatorHearBeatProto)
{
EvaluatorHeartbeatProto heartbeat = evaluatorHearBeatProto.Message;
EvaluatorStatusProto status = heartbeat.evaluator_status;
string evaluatorId = status.evaluator_id;
LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "Heartbeat from Evaluator {0} with state {1} timestamp {2}", evaluatorId, status.state, heartbeat.timestamp));
_sanityChecker.check(evaluatorId, heartbeat.timestamp);
lock (_evaluators)
{
if (_evaluators.ContainsKey(evaluatorId))
{
EvaluatorManager evaluatorManager = _evaluators[evaluatorId];
evaluatorManager.Handle(evaluatorHearBeatProto);
}
else
{
string msg = "Contact from unkonwn evaluator with id: " + evaluatorId;
if (heartbeat.evaluator_status != null)
{
msg += " with state" + status.state;
}
LOGGER.Log(Level.Error, msg);
Exceptions.Throw(new InvalidOperationException(msg), LOGGER);
}
}
}
private void OnException(Exception e)
{
lock (_heartBeatManager)
{
Logger.Log(Level.Error, "Evaluator {0} failed with exception {1}.", _evaluatorId, e);
_state = State.FAILED;
byte[] errorBytes = null;
try
{
errorBytes = ByteUtilities.SerializeToBinaryFormat(e);
}
catch (SerializationException se)
{
errorBytes = ByteUtilities.SerializeToBinaryFormat(
NonSerializableEvaluatorException.UnableToSerialize(e, se));
}
var evaluatorStatusProto = new EvaluatorStatusProto
{
evaluator_id = _evaluatorId,
error = errorBytes,
state = _state
};
_heartBeatManager.OnNext(evaluatorStatusProto);
_contextManager.Dispose();
_evaluatorExitLogger.LogExit(false);
}
}
public EvaluatorStatusProto GetEvaluatorStatus()
{
Logger.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "Evaluator state : {0}", _state));
EvaluatorStatusProto evaluatorStatusProto = new EvaluatorStatusProto
{
evaluator_id = _evaluatorId,
state = _state
};
return(evaluatorStatusProto);
}
/// <summary>
/// Called with a specific EvaluatorStatus that must be delivered to the driver
/// </summary>
/// <param name="evaluatorStatusProto"></param>
public void OnNext(EvaluatorStatusProto evaluatorStatusProto)
{
LOGGER.Log(Level.Verbose, "Before acquiring lock: HeartbeatManager::OnNext(EvaluatorStatusProto)");
lock (this)
{
LOGGER.Log(Level.Verbose, "HeartbeatManager::OnNext(EvaluatorStatusProto)");
EvaluatorHeartbeatProto heartbeatProto = new EvaluatorHeartbeatProto()
{
timestamp = CurrentTimeMilliSeconds(),
evaluator_status = evaluatorStatusProto
};
LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "Triggered a heartbeat: {0}.", heartbeatProto));
Send(heartbeatProto);
}
}
private EvaluatorHeartbeatProto GetEvaluatorHeartbeatProto(
EvaluatorStatusProto evaluatorStatusProto,
ICollection <ContextStatusProto> contextStatusProtos,
Optional <TaskStatusProto> taskStatusProto)
{
EvaluatorHeartbeatProto evaluatorHeartbeatProto = new EvaluatorHeartbeatProto()
{
timestamp = CurrentTimeMilliSeconds(),
evaluator_status = evaluatorStatusProto
};
foreach (ContextStatusProto contextStatusProto in contextStatusProtos)
{
evaluatorHeartbeatProto.context_status.Add(contextStatusProto);
}
if (taskStatusProto.IsPresent())
{
evaluatorHeartbeatProto.task_status = taskStatusProto.Value;
}
return(evaluatorHeartbeatProto);
}
private void Handle(Exception e)
{
lock (_heartBeatManager)
{
Logger.Log(Level.Error, string.Format(CultureInfo.InvariantCulture, "evaluator {0} failed with exception", _evaluatorId), e);
_state = State.FAILED;
string errorMessage = string.Format(
CultureInfo.InvariantCulture,
"failed with error [{0}] with message [{1}] and stack trace [{2}]",
e,
e.Message,
e.StackTrace);
EvaluatorStatusProto evaluatorStatusProto = new EvaluatorStatusProto()
{
evaluator_id = _evaluatorId,
error = ByteUtilities.StringToByteArrays(errorMessage),
state = _state
};
_heartBeatManager.OnNext(evaluatorStatusProto);
_contextManager.Dispose();
}
}
public void Handle(IRemoteMessage <EvaluatorHeartbeatProto> evaluatorHearBeatProtoMessage)
{
lock (_evaluatorDescriptor)
{
EvaluatorHeartbeatProto heartbeatProto = evaluatorHearBeatProtoMessage.Message;
if (heartbeatProto.evaluator_status != null)
{
EvaluatorStatusProto status = heartbeatProto.evaluator_status;
if (status.error != null)
{
Handle(new EvaluatorException(Id, ByteUtilities.ByteArrarysToString(status.error)));
return;
}
else if (_state == STATE.SUBMITTED)
{
string evaluatorRId = evaluatorHearBeatProtoMessage.Identifier.ToString();
LOGGER.Log(Level.Info, "TODO: REPLACE THIS " + evaluatorRId);
// TODO
// _evaluatorControlHandler = _remoteManager.getHandler(evaluatorRID, EvaluatorRuntimeProtocol.EvaluatorControlProto.class);
_state = STATE.RUNNING;
LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "Evaluator {0} is running", _evaluatorId));
}
}
LOGGER.Log(Level.Info, "Evaluator heartbeat: " + heartbeatProto);
EvaluatorStatusProto evaluatorStatusProto = heartbeatProto.evaluator_status;
foreach (ContextStatusProto contextStatusProto in heartbeatProto.context_status)
{
Handle(contextStatusProto, heartbeatProto.task_status != null);
}
if (heartbeatProto.task_status != null)
{
Handle(heartbeatProto.task_status);
}
if (evaluatorStatusProto.state == State.FAILED)
{
_state = STATE.FAILED;
EvaluatorException e = evaluatorStatusProto.error != null ?
new EvaluatorException(_evaluatorId, ByteUtilities.ByteArrarysToString(evaluatorStatusProto.error)) :
new EvaluatorException(_evaluatorId, "unknown cause");
LOGGER.Log(Level.Warning, "Failed evaluator: " + Id + e.Message);
Handle(e);
}
else if (evaluatorStatusProto.state == State.DONE)
{
LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "Evaluator {0} done", Id));
_state = STATE.DONE;
// TODO
// dispatcher.onNext(CompletedEvaluator.class, new CompletedEvaluator() {
//@Override
//public String getId() {
// return EvaluatorManager.this.evaluatorId;
Dispose();
}
}
LOGGER.Log(Level.Info, "DONE with evaluator heartbeat");
}
