Пример #1
0
        public void OnNext(ICloseEvent value)
        {
            Logger.Log(Level.Info, "TaskRuntime::OnNext(ICloseEvent value)");
            _closeHandlerFuture.Get().OnNext(value);

            // TODO: send a heartbeat
        }
Пример #2
0
        /// <summary>
        /// Called by the remote manager to process messages received from the java bridge.
        /// </summary>
        /// <param name="message">A byte buffer containing a serialzied message.</param>
        public void OnNext(IRemoteMessage <byte[]> message)
        {
            Logger.Log(Level.Verbose, "Message received: {0}", message.Identifier);

            // Deserialize the message and invoke the appropriate handler.
            _serializer.Read(message.Message, _fMessageObserver.Get());
        }
Пример #3
0
 /// <summary>
 /// Register the event handlers
 /// </summary>
 private void SubscribeHandlers()
 {
     Subscribe(_startHandler.Get());
     Subscribe(_stopHandler.Get());
     Subscribe(_runtimeStartHandler.Get());
     Subscribe(_runtimeStopHandler.Get());
     Subscribe(_idleHandler.Get());
 }
Пример #4
0
        public void OnNext(Alarm value)
        {
            LOGGER.Log(Level.Verbose, "Before acquiring lock: HeartbeatManager::OnNext(Alarm)");
            lock (this)
            {
                LOGGER.Log(Level.Verbose, "HeartbeatManager::OnNext(Alarm)");

                if (_evaluatorSettings.OperationState == EvaluatorOperationState.OPERATIONAL && EvaluatorRuntime.State == State.RUNNING)
                {
                    EvaluatorHeartbeatProto evaluatorHeartbeatProto = GetEvaluatorHeartbeatProto();
                    LOGGER.Log(Level.Verbose, string.Format(CultureInfo.InvariantCulture, "Triggered a heartbeat: {0}. {1}Node Health: {2}", evaluatorHeartbeatProto, Environment.NewLine, MachineStatus.ToString()));
                    Send(evaluatorHeartbeatProto);
                }
                else
                {
                    LOGGER.Log(Level.Verbose, "Ignoring regular heartbeat since Evaluator operation state is [{0}] and runtime state is [{1}]. ", EvaluatorSettings.OperationState, EvaluatorRuntime.State);

                    // Do not try to recover if Evaluator is done.
                    if (IsEvaluatorStateCompleted(EvaluatorRuntime.State))
                    {
                        return;
                    }

                    if (_evaluatorSettings.OperationState == EvaluatorOperationState.RECOVERY)
                    {
                        var driverConnection = _driverConnection.Get();

                        try
                        {
                            var driverInformation = driverConnection.GetDriverInformation();
                            if (driverInformation == null)
                            {
                                LOGGER.Log(Level.Verbose, "In RECOVERY mode, cannot retrieve driver information, will try again later.");
                            }
                            else
                            {
                                LOGGER.Log(
                                    Level.Info,
                                    string.Format(CultureInfo.InvariantCulture, "Detect driver restarted at {0} and is running on endpoint {1} with services {2}. Now trying to re-establish connection", driverInformation.DriverStartTime, driverInformation.DriverRemoteIdentifier, driverInformation.NameServerId));
                                Recover(driverInformation);
                            }
                        }
                        catch (Exception e)
                        {
                            // we do not want any exception to stop the query for driver status
                            Utilities.Diagnostics.Exceptions.Caught(e, Level.Warning, LOGGER);
                        }
                    }
                }

                _clock.ScheduleAlarm(_heartBeatPeriodInMillSeconds, this);
            }
        }
Пример #5
0
 public void OnNext(ISuspendEvent value)
 {
     Logger.Log(Level.Info, "TaskRuntime::OnNext(ISuspendEvent value)");
     try
     {
         _suspendHandlerFuture.Get().OnNext(value);
     }
     catch (Exception ex)
     {
         var suspendEx = new TaskSuspendHandlerException("Unable to suspend task.", ex);
         Utilities.Diagnostics.Exceptions.CaughtAndThrow(suspendEx, Level.Error, Logger);
     }
 }
Пример #6
0
        public void OnNext(IdleClock value)
        {
            string message = string.Format(
                CultureInfo.InvariantCulture,
                "IdleClock: [{0}], RuntimeState [{1}], Outstanding container requests [{2}], Container allocation count[{3}]",
                value + Environment.NewLine,
                _runtimeStatusProto.state + Environment.NewLine,
                _runtimeStatusProto.outstanding_container_requests + Environment.NewLine,
                _runtimeStatusProto.container_allocation.Count);

            LOGGER.Log(Level.Info, message);

            lock (_evaluators)
            {
                if (_runtimeStatusProto.state == State.RUNNING &&
                    _runtimeStatusProto.outstanding_container_requests == 0 &&
                    _runtimeStatusProto.container_allocation.Count == 0)
                {
                    LOGGER.Log(Level.Info, "Idle runtime shutdown");
                    _clockFuture.Get().Dispose();
                }
            }
        }
Пример #7
0
        private object GetInstance(INode node)
        {
            InjectionPlan plan = (InjectionPlan)GetInjectionPlan(node);
            object        u    = InjectFromPlan(plan);

            while (pendingFutures.Count != 0)
            {
                IEnumerator <object> i = pendingFutures.GetEnumerator();
                i.MoveNext();
                IInjectionFuture <object> f = (IInjectionFuture <object>)i.Current;
                pendingFutures.Remove(f);
                f.Get();
            }
            return(u);
        }
Пример #8
0
 public void OnNext(ISuspendEvent value)
 {
     Logger.Log(Level.Info, "TaskRuntime::OnNext(ISuspendEvent value)");
     _suspendHandlerFuture.Get().OnNext(value);
 }
Пример #9
0
            public override FlyingCar getMyCar()
            {
                FlyingCar c = fCar.Get();

                return(c);
            }
Пример #10
0
            public virtual FlyingCar getMyCar()
            {
                FlyingCar c = fcar.Get();

                return(c);
            }
Пример #11
0
 /// <summary>
 /// Registers a <see cref="TaskMessageObserver"/> for a given <see cref="taskSourceId"/>.
 /// If the <see cref="TaskMessageObserver"/> has already been initialized, it will return
 /// the existing one.
 /// </summary>
 public TaskMessageObserver RegisterAndGetForTask(string taskSourceId)
 {
     // Add a TaskMessage observer for each upstream/downstream source.
     return(_taskMessageObservers.GetOrAdd(taskSourceId, new TaskMessageObserver(_networkService.Get())));
 }
Пример #12
0
        public void Send(EvaluatorHeartbeatProto evaluatorHeartbeatProto)
        {
            lock (_queuedHeartbeats)
            {
                // Do not send a heartbeat if Evaluator has already signaled that it was done.
                if (_isCompletedHeartbeatQueued)
                {
                    LOGGER.Log(Level.Warning, "Evaluator trying to schedule a heartbeat after a completed heartbeat has already been scheduled or sent.");
                    return;
                }

                if (IsEvaluatorStateCompleted(evaluatorHeartbeatProto.evaluator_status.state))
                {
                    _isCompletedHeartbeatQueued = true;
                }

                if (_evaluatorSettings.OperationState == EvaluatorOperationState.RECOVERY)
                {
                    LOGGER.Log(Level.Warning, string.Format(CultureInfo.InvariantCulture, "In RECOVERY mode, heartbeat queued as [{0}]. ", evaluatorHeartbeatProto));
                    _queuedHeartbeats.Enqueue(evaluatorHeartbeatProto);
                    return;
                }

                // NOT during recovery, try to send
                REEFMessage payload = new REEFMessage(evaluatorHeartbeatProto);
                try
                {
                    _observer.OnNext(payload);
                    _heartbeatFailures = 0; // reset failure counts if we are having intermittent (not continuous) failures
                }
                catch (Exception e)
                {
                    if (evaluatorHeartbeatProto.task_status == null || evaluatorHeartbeatProto.task_status.state != State.RUNNING)
                    {
                        Utilities.Diagnostics.Exceptions.Throw(e, "Lost communications to driver when no task is running, recovery NOT supported for such scenario", LOGGER);
                    }

                    _heartbeatFailures++;

                    _queuedHeartbeats.Enqueue(evaluatorHeartbeatProto);
                    LOGGER.Log(Level.Error, string.Format(CultureInfo.InvariantCulture, "Sending heartbeat to driver experienced #{0} failure. Hearbeat queued as: [{1}]. ", _heartbeatFailures, evaluatorHeartbeatProto), e);

                    if (_driverConnection.Get() is MissingDriverConnection)
                    {
                        if (_heartbeatFailures >= _maxHeartbeatRetriesForNonRecoveryMode)
                        {
                            var msg =
                                string.Format(CultureInfo.InvariantCulture,
                                              "Have encountered {0} heartbeat failures. Limit of heartbeat sending failures exceeded. Driver reconnect logic is not implemented, failing evaluator.",
                                              _heartbeatFailures);
                            LOGGER.Log(Level.Error, msg);
                            throw new ReefRuntimeException(msg, e);
                        }
                    }
                    else
                    {
                        if (_heartbeatFailures >= _maxHeartbeatRetries)
                        {
                            LOGGER.Log(Level.Warning,
                                       "Heartbeat communications to driver reached max of {0} failures. Driver is considered dead/unreachable",
                                       _heartbeatFailures);
                            LOGGER.Log(Level.Info, "Entering RECOVERY mode!!!");
                            ContextManager.HandleDriverConnectionMessage(
                                new DriverConnectionMessageImpl(DriverConnectionState.Disconnected));

                            LOGGER.Log(Level.Info, "instantiate driver reconnect implementation: " + _driverConnection);
                            _evaluatorSettings.OperationState = EvaluatorOperationState.RECOVERY;

                            // clean heartbeat failure
                            _heartbeatFailures = 0;
                        }
                    }
                }
            }
        }
Пример #13
0
 public void Run()
 {
     _driverClientService.Get().Start();
     _driverClientService.Get().AwaitTermination();
     Log.Log(Level.Info, "bridge clock terminated");
 }
Пример #14
0
 public void Dispose()
 {
     _driverServiceClient.Get().OnShutdown();
 }