public void OnNext(ICloseEvent value)
{
Logger.Log(Level.Info, "TaskRuntime::OnNext(ICloseEvent value)");
_closeHandlerFuture.Get().OnNext(value);
// TODO: send a heartbeat
}
/// <summary>
/// Called by the remote manager to process messages received from the java bridge.
/// </summary>
/// <param name="message">A byte buffer containing a serialzied message.</param>
public void OnNext(IRemoteMessage <byte[]> message)
{
Logger.Log(Level.Verbose, "Message received: {0}", message.Identifier);
// Deserialize the message and invoke the appropriate handler.
_serializer.Read(message.Message, _fMessageObserver.Get());
}
/// <summary>
/// Register the event handlers
/// </summary>
private void SubscribeHandlers()
{
Subscribe(_startHandler.Get());
Subscribe(_stopHandler.Get());
Subscribe(_runtimeStartHandler.Get());
Subscribe(_runtimeStopHandler.Get());
Subscribe(_idleHandler.Get());
}
public void OnNext(Alarm value)
{
LOGGER.Log(Level.Verbose, "Before acquiring lock: HeartbeatManager::OnNext(Alarm)");
lock (this)
{
LOGGER.Log(Level.Verbose, "HeartbeatManager::OnNext(Alarm)");
if (_evaluatorSettings.OperationState == EvaluatorOperationState.OPERATIONAL && EvaluatorRuntime.State == State.RUNNING)
{
EvaluatorHeartbeatProto evaluatorHeartbeatProto = GetEvaluatorHeartbeatProto();
LOGGER.Log(Level.Verbose, string.Format(CultureInfo.InvariantCulture, "Triggered a heartbeat: {0}. {1}Node Health: {2}", evaluatorHeartbeatProto, Environment.NewLine, MachineStatus.ToString()));
Send(evaluatorHeartbeatProto);
}
else
{
LOGGER.Log(Level.Verbose, "Ignoring regular heartbeat since Evaluator operation state is [{0}] and runtime state is [{1}]. ", EvaluatorSettings.OperationState, EvaluatorRuntime.State);
// Do not try to recover if Evaluator is done.
if (IsEvaluatorStateCompleted(EvaluatorRuntime.State))
{
return;
}
if (_evaluatorSettings.OperationState == EvaluatorOperationState.RECOVERY)
{
var driverConnection = _driverConnection.Get();
try
{
var driverInformation = driverConnection.GetDriverInformation();
if (driverInformation == null)
{
LOGGER.Log(Level.Verbose, "In RECOVERY mode, cannot retrieve driver information, will try again later.");
}
else
{
LOGGER.Log(
Level.Info,
string.Format(CultureInfo.InvariantCulture, "Detect driver restarted at {0} and is running on endpoint {1} with services {2}. Now trying to re-establish connection", driverInformation.DriverStartTime, driverInformation.DriverRemoteIdentifier, driverInformation.NameServerId));
Recover(driverInformation);
}
}
catch (Exception e)
{
// we do not want any exception to stop the query for driver status
Utilities.Diagnostics.Exceptions.Caught(e, Level.Warning, LOGGER);
}
}
}
_clock.ScheduleAlarm(_heartBeatPeriodInMillSeconds, this);
}
}
public void OnNext(ISuspendEvent value)
{
Logger.Log(Level.Info, "TaskRuntime::OnNext(ISuspendEvent value)");
try
{
_suspendHandlerFuture.Get().OnNext(value);
}
catch (Exception ex)
{
var suspendEx = new TaskSuspendHandlerException("Unable to suspend task.", ex);
Utilities.Diagnostics.Exceptions.CaughtAndThrow(suspendEx, Level.Error, Logger);
}
}
public void OnNext(IdleClock value)
{
string message = string.Format(
CultureInfo.InvariantCulture,
"IdleClock: [{0}], RuntimeState [{1}], Outstanding container requests [{2}], Container allocation count[{3}]",
value + Environment.NewLine,
_runtimeStatusProto.state + Environment.NewLine,
_runtimeStatusProto.outstanding_container_requests + Environment.NewLine,
_runtimeStatusProto.container_allocation.Count);
LOGGER.Log(Level.Info, message);
lock (_evaluators)
{
if (_runtimeStatusProto.state == State.RUNNING &&
_runtimeStatusProto.outstanding_container_requests == 0 &&
_runtimeStatusProto.container_allocation.Count == 0)
{
LOGGER.Log(Level.Info, "Idle runtime shutdown");
_clockFuture.Get().Dispose();
}
}
}
private object GetInstance(INode node)
{
InjectionPlan plan = (InjectionPlan)GetInjectionPlan(node);
object u = InjectFromPlan(plan);
while (pendingFutures.Count != 0)
{
IEnumerator <object> i = pendingFutures.GetEnumerator();
i.MoveNext();
IInjectionFuture <object> f = (IInjectionFuture <object>)i.Current;
pendingFutures.Remove(f);
f.Get();
}
return(u);
}
public void OnNext(ISuspendEvent value)
{
Logger.Log(Level.Info, "TaskRuntime::OnNext(ISuspendEvent value)");
_suspendHandlerFuture.Get().OnNext(value);
}
public override FlyingCar getMyCar()
{
FlyingCar c = fCar.Get();
return(c);
}
public virtual FlyingCar getMyCar()
{
FlyingCar c = fcar.Get();
return(c);
}
/// <summary>
/// Registers a <see cref="TaskMessageObserver"/> for a given <see cref="taskSourceId"/>.
/// If the <see cref="TaskMessageObserver"/> has already been initialized, it will return
/// the existing one.
/// </summary>
public TaskMessageObserver RegisterAndGetForTask(string taskSourceId)
{
// Add a TaskMessage observer for each upstream/downstream source.
return(_taskMessageObservers.GetOrAdd(taskSourceId, new TaskMessageObserver(_networkService.Get())));
}
public void Send(EvaluatorHeartbeatProto evaluatorHeartbeatProto)
{
lock (_queuedHeartbeats)
{
// Do not send a heartbeat if Evaluator has already signaled that it was done.
if (_isCompletedHeartbeatQueued)
{
LOGGER.Log(Level.Warning, "Evaluator trying to schedule a heartbeat after a completed heartbeat has already been scheduled or sent.");
return;
}
if (IsEvaluatorStateCompleted(evaluatorHeartbeatProto.evaluator_status.state))
{
_isCompletedHeartbeatQueued = true;
}
if (_evaluatorSettings.OperationState == EvaluatorOperationState.RECOVERY)
{
LOGGER.Log(Level.Warning, string.Format(CultureInfo.InvariantCulture, "In RECOVERY mode, heartbeat queued as [{0}]. ", evaluatorHeartbeatProto));
_queuedHeartbeats.Enqueue(evaluatorHeartbeatProto);
return;
}
// NOT during recovery, try to send
REEFMessage payload = new REEFMessage(evaluatorHeartbeatProto);
try
{
_observer.OnNext(payload);
_heartbeatFailures = 0; // reset failure counts if we are having intermittent (not continuous) failures
}
catch (Exception e)
{
if (evaluatorHeartbeatProto.task_status == null || evaluatorHeartbeatProto.task_status.state != State.RUNNING)
{
Utilities.Diagnostics.Exceptions.Throw(e, "Lost communications to driver when no task is running, recovery NOT supported for such scenario", LOGGER);
}
_heartbeatFailures++;
_queuedHeartbeats.Enqueue(evaluatorHeartbeatProto);
LOGGER.Log(Level.Error, string.Format(CultureInfo.InvariantCulture, "Sending heartbeat to driver experienced #{0} failure. Hearbeat queued as: [{1}]. ", _heartbeatFailures, evaluatorHeartbeatProto), e);
if (_driverConnection.Get() is MissingDriverConnection)
{
if (_heartbeatFailures >= _maxHeartbeatRetriesForNonRecoveryMode)
{
var msg =
string.Format(CultureInfo.InvariantCulture,
"Have encountered {0} heartbeat failures. Limit of heartbeat sending failures exceeded. Driver reconnect logic is not implemented, failing evaluator.",
_heartbeatFailures);
LOGGER.Log(Level.Error, msg);
throw new ReefRuntimeException(msg, e);
}
}
else
{
if (_heartbeatFailures >= _maxHeartbeatRetries)
{
LOGGER.Log(Level.Warning,
"Heartbeat communications to driver reached max of {0} failures. Driver is considered dead/unreachable",
_heartbeatFailures);
LOGGER.Log(Level.Info, "Entering RECOVERY mode!!!");
ContextManager.HandleDriverConnectionMessage(
new DriverConnectionMessageImpl(DriverConnectionState.Disconnected));
LOGGER.Log(Level.Info, "instantiate driver reconnect implementation: " + _driverConnection);
_evaluatorSettings.OperationState = EvaluatorOperationState.RECOVERY;
// clean heartbeat failure
_heartbeatFailures = 0;
}
}
}
}
}
public void Run()
{
_driverClientService.Get().Start();
_driverClientService.Get().AwaitTermination();
Log.Log(Level.Info, "bridge clock terminated");
}
public void Dispose()
{
_driverServiceClient.Get().OnShutdown();
}
