Ejemplo n.º 1
0
 private EvaluatorHeartbeatProto GetEvaluatorHeartbeatProto()
 {
     return(GetEvaluatorHeartbeatProto(
                EvaluatorRuntime.GetEvaluatorStatus(),
                ContextManager.GetContextStatusCollection(),
                ContextManager.GetTaskStatus()));
 }
Ejemplo n.º 2
0
 /// <summary>
 /// Called with a specific TaskStatus that must be delivered to the driver
 /// </summary>
 /// <param name="taskStatusProto"></param>
 public void OnNext(TaskStatusProto taskStatusProto)
 {
     LOGGER.Log(Level.Verbose, "Before acquiring lock: HeartbeatManager::OnNext(TaskStatusProto)");
     lock (this)
     {
         LOGGER.Log(Level.Verbose, "HeartbeatManager::OnNext(TaskStatusProto)");
         EvaluatorHeartbeatProto heartbeatProto = GetEvaluatorHeartbeatProto(
             EvaluatorRuntime.GetEvaluatorStatus(),
             ContextManager.GetContextStatusCollection(),
             Optional <TaskStatusProto> .Of(taskStatusProto));
         LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "Triggered a heartbeat: {0}.", heartbeatProto));
         Send(heartbeatProto);
     }
 }
Ejemplo n.º 3
0
        public void OnNext(Alarm value)
        {
            LOGGER.Log(Level.Verbose, "Before acquiring lock: HeartbeatManager::OnNext(Alarm)");
            lock (this)
            {
                LOGGER.Log(Level.Verbose, "HeartbeatManager::OnNext(Alarm)");

                if (_evaluatorSettings.OperationState == EvaluatorOperationState.OPERATIONAL && EvaluatorRuntime.State == State.RUNNING)
                {
                    try
                    {
                        EvaluatorHeartbeatProto evaluatorHeartbeatProto = GetEvaluatorHeartbeatProto();
                        LOGGER.Log(Level.Verbose,
                                   string.Format(CultureInfo.InvariantCulture, "Triggered a heartbeat: {0}. {1}Node Health: {2}", evaluatorHeartbeatProto, Environment.NewLine, MachineStatus));
                        Send(evaluatorHeartbeatProto);
                    }
                    catch (Exception e)
                    {
                        Utilities.Diagnostics.Exceptions.Caught(e, Level.Error, LOGGER);
                        EvaluatorRuntime.OnException(e);
                    }
                }
                else
                {
                    LOGGER.Log(Level.Verbose, "Ignoring regular heartbeat since Evaluator operation state is [{0}] and runtime state is [{1}]. ", EvaluatorSettings.OperationState, EvaluatorRuntime.State);

                    // Do not try to recover if Evaluator is done.
                    if (IsEvaluatorStateCompleted(EvaluatorRuntime.State))
                    {
                        return;
                    }

                    if (_evaluatorSettings.OperationState == EvaluatorOperationState.RECOVERY)
                    {
                        var driverConnection = _driverConnection.Get();
                        try
                        {
                            var driverInformation = driverConnection.GetDriverInformation();
                            if (driverInformation == null)
                            {
                                LOGGER.Log(Level.Verbose,
                                           "In RECOVERY mode, cannot retrieve driver information, will try again later.");
                            }
                            else
                            {
                                var msg = string.Format(CultureInfo.InvariantCulture,
                                                        "Detect driver restarted at {0} and is running on endpoint {1} with services {2}. Now trying to re-establish connection",
                                                        driverInformation.DriverStartTime,
                                                        driverInformation.DriverRemoteIdentifier,
                                                        driverInformation.NameServerId);
                                LOGGER.Log(Level.Info, msg);
                                Recover(driverInformation);
                            }
                        }
                        catch (NotImplementedException)
                        {
                            LOGGER.Log(Level.Error, "Reaching EvaluatorOperation RECOVERY mode, however, there is no IDriverConnection implemented for HA.");
                            throw;
                        }
                        catch (Exception e)
                        {
                            // we do not want any exception to stop the query for driver status
                            Utilities.Diagnostics.Exceptions.Caught(e, Level.Warning, LOGGER);
                        }
                    }
                }

                _clock.ScheduleAlarm(_heartBeatPeriodInMillSeconds, this);
            }
        }