private void StreamHealthCheckMsgHandler(StreamHealthCheckMsg msg) { if (_resource == null || msg.Resource == null || msg.Resource.Id != _resource.Id) { return; } try { _resource.Content.Sequence = msg.Resource.Content.Sequence; _resource.Content.MatchStatus = msg.Resource.Content.MatchStatus; _logger.Debug( $"Listener state for {msg.Resource} has " + $"sequence={msg.Resource.Content.Sequence} " + $"processedSequence={msg.CurrentSequence} " + (msg.Resource.Content.Sequence > msg.CurrentSequence ? $"missedSequence={msg.Resource.Content.Sequence - msg.CurrentSequence} " : "") + $"State={msg.StreamingState} " + $"isMatchOver={msg.Resource.IsMatchOver}"); var streamIsValid = _streamHealthCheckValidation.ValidateStream(msg.Resource, msg.StreamingState, msg.CurrentSequence); var connectToStreamServer = _streamHealthCheckValidation.CanConnectToStreamServer(msg.Resource, msg.StreamingState); if (!streamIsValid) { _logger.Warn($"Detected invalid stream for {msg.Resource}"); if (_streamInvalidDetected) { Context.Parent.Tell(new StopStreamingMsg()); } else { _streamInvalidDetected = true; Context.Parent.Tell(new SuspendAndReprocessSnapshotMsg()); } } else { _streamInvalidDetected = false; } if (connectToStreamServer) { Context.Parent.Tell(new ConnectToStreamServerMsg()); } } catch (Exception ex) { _logger.Error($"Error occured on Stream Health Check for {_resource} - exception - {ex}"); throw; } }
private void StreamHealthCheckMsgHandler(StreamHealthCheckMsg msg) { msg.StreamingState = State; msg.CurrentSequence = _currentSequence; _logger.Info( $"{_resource} Stream health check message arrived - State = {State}; CurrentSequence = {_currentSequence}"); _streamHealthCheckActor.Tell(msg); }
private void LogState(StreamHealthCheckMsg msg) { var delay = (DateTime.Now - msg.Time).TotalSeconds; _logger.Debug( $"Listener state for {msg.Resource} has " + $"processedSequence={msg.CurrentSequence} " + (msg.Resource.Content.Sequence > msg.CurrentSequence ? $"missedSequence={msg.Resource.Content.Sequence - msg.CurrentSequence} " : "") + $"State={msg.StreamingState} " + $"isMatchOver={msg.Resource.IsMatchOver} messageTime={msg.Time} delay={delay.ToString("N")} sec"); }
private bool StopStreamingDueToMatchOver(StreamHealthCheckMsg msg) { if (msg.Resource.IsMatchOver) { if (matchOverDetectedTime == null) { matchOverDetectedTime = DateTime.Now; } else { if (matchOverDetectedTime.Value.AddMinutes(10) > DateTime.Now) { Context.Parent.Tell(new StopStreamingMsg()); return(true); } } } return(false); }
private void StreamHealthCheckMsgHandler(StreamHealthCheckMsg msg) { if (_resource == null || msg.Resource == null || msg.Resource.Id != _resource.Id) { return; } _resource.Content.Sequence = msg.Resource.Content.Sequence; _resource.Content.MatchStatus = msg.Resource.Content.MatchStatus; LogState(msg); if (StopStreamingDueToMatchOver(msg)) { return; } if (!ValidateTime()) { return; } try { var isSequenceValid = _streamHealthCheckValidation.IsSequenceValid(msg.Resource, msg.StreamingState, msg.CurrentSequence); var SequenceUpdated = msg.CurrentSequence > lastProcessedSequence && lastProcessedSequence > 0; var isInErroredState = msg.StreamingState == StreamListenerState.Errored; _erroredStateCount = isInErroredState ? _erroredStateCount + 1 : 0; var isNeedStopErroredState = isInErroredState && _erroredStateCount >= _settings.MaxInErroredState - 1; //var streamIsValid if (!isSequenceValid && !SequenceUpdated || isNeedStopErroredState) { _logger.Warn($"StreamHealthCheckMsgHandler: Detected {(_streamInvalidDetected ? "invalid" : "suspicious")} stream {msg.Resource}, isNeedStopErroredState={isNeedStopErroredState}, " + $"isSequenceValid={isSequenceValid}, SequenceUpdated={SequenceUpdated}"); if (_streamInvalidDetected) { Context.Parent.Tell(new StopStreamingMsg()); } else { _streamInvalidDetected = true; Context.Parent.Tell(new SuspendMessage(SuspensionReason.HEALTH_CHECK_FALURE)); } } else { _streamInvalidDetected = false; } if (_streamHealthCheckValidation.CanConnectToStreamServer(msg.Resource, msg.StreamingState)) { Context.Parent.Tell(new ConnectToStreamServerMsg()); } } catch (Exception ex) { _logger.Error($"Error occured on Stream Health Check for {_resource} - exception - {ex}"); throw; } lastExecute = DateTime.UtcNow; lastProcessedSequence = msg.CurrentSequence; }