private bool?StartRecovery()
{
var duration = TimeProviderAccessor.Current.Now - _recoveryOperation.LastStartTime;
if (duration.TotalSeconds > _minIntervalBetweenRecoveryRequests)
{
return(_recoveryOperation.Start());
}
ExecutionLog.Info($"Producer={_producer.Id}: Recovery operation skipped.");
return(null);
}
private bool?StartRecovery()
{
var duration = TimeProviderAccessor.Current.Now - _recoveryOperation.LastAttemptTime;
if (duration.TotalSeconds > _minIntervalBetweenRecoveryRequests)
{
_lastRecoveryMessage = DateTime.Now;
return(_recoveryOperation.Start());
}
ExecutionLog.LogInformation($"Producer={_producer.Id}: Recovery operation skipped. Last done at {_recoveryOperation.LastAttemptTime}");
return(null);
}
private bool?StartRecovery()
{
if (!_connectionDownTimestamp.Equals(DateTime.MinValue))
{
ExecutionLog.LogWarning($"Producer={_producer.Id}: Recovery operation skipped (feed connection is down).");
return(null);
}
var duration = TimeProviderAccessor.Current.Now - _recoveryOperation.LastAttemptTime;
if (duration.TotalSeconds > _minIntervalBetweenRecoveryRequests)
{
_lastRecoveryMessage = DateTime.Now;
return(_recoveryOperation.Start());
}
ExecutionLog.LogInformation($"Producer={_producer.Id}: Recovery operation skipped. Last done at {_recoveryOperation.LastAttemptTime}");
return(null);
}
/// <summary>
/// Processes a message received on the system's session
/// </summary>
/// <remarks>
/// This method does:
/// - starts recovery operations if needed
/// - interrupt running recoveries on non-subscribed alive(s) and alive violation(s)
/// - set LastTimestampBeforeDisconnect property on the producer.
/// This method does not:
/// - determine if the user is behind (or not) with message processing - this is done in CheckStatus(..) method
/// - attempt to determine whether the recovery has timed-out - this is done in CheckStatus(..) method
/// </remarks>
/// <param name="message">A <see cref="FeedMessage"/> received on the system session</param>
public void ProcessSystemMessage(FeedMessage message)
{
Guard.Argument(message, nameof(message)).NotNull();
var alive = message as alive;
if (alive?.ProducerId != Producer.Id)
{
return;
}
var newStatus = Status;
if (Status == ProducerRecoveryStatus.FatalError || _producer.IgnoreRecovery)
{
return;
}
lock (_syncLock)
{
try
{
// store the timestamp of most recent system alive before it is overridden by
// _timestampTracker.ProcessSystemAlive(alive); in case the current alive
// is not subscribed and ongoing recovery operation must be interrupted
var previousAliveTimestamp = _timestampTracker.SystemAliveTimestamp;
_timestampTracker.ProcessSystemAlive(alive);
// if current status is NotStarted or Error just start the recovery
if (Status == ProducerRecoveryStatus.NotStarted || Status == ProducerRecoveryStatus.Error)
{
try
{
if (_recoveryOperation.Start())
{
ExecutionLog.LogInformation($"Producer={_producer.Id}: Recovery operation started. Current status={Enum.GetName(typeof(ProducerRecoveryStatus), Status)}, After={SdkInfo.ToEpochTime(_producer.LastTimestampBeforeDisconnect)}.");
newStatus = ProducerRecoveryStatus.Started;
}
else
{
ExecutionLog.LogWarning($"Producer={_producer.Id}: An error occurred while starting recovery operation with after={SdkInfo.ToEpochTime(_producer.LastTimestampBeforeDisconnect)}. Retry will be made at next system alive.");
}
}
catch (RecoveryInitiationException ex)
{
ExecutionLog.LogCritical($"Producer id={Producer.Id} failed to execute recovery because 'after' is to much in the past. After={ex.After}. Stopping the feed.");
newStatus = ProducerRecoveryStatus.FatalError;
}
}
else
{
Debug.Assert(Status == ProducerRecoveryStatus.Started ||
Status == ProducerRecoveryStatus.Completed ||
Status == ProducerRecoveryStatus.Delayed);
// we are no longer in sync with the feed
if (alive.subscribed == 0)
{
if (_recoveryOperation.IsRunning)
{
Debug.Assert(Status == ProducerRecoveryStatus.Started);
var timestamp = SdkInfo.FromEpochTime(previousAliveTimestamp);
ExecutionLog.LogInformation($"Producer={_producer.Id}: Recovery operation interrupted. Current status={Enum.GetName(typeof(ProducerRecoveryStatus), Status)}, Timestamp={timestamp}.");
_recoveryOperation.Interrupt(timestamp);
}
else
{
Debug.Assert(Status == ProducerRecoveryStatus.Completed || Status == ProducerRecoveryStatus.Delayed);
try
{
if (_recoveryOperation.Start())
{
ExecutionLog.LogInformation($"Producer={_producer.Id}: Recovery operation started due to un-subscribed alive. Current status={Enum.GetName(typeof(ProducerRecoveryStatus), Status)}, After={SdkInfo.ToEpochTime(_producer.LastTimestampBeforeDisconnect)}.");
newStatus = ProducerRecoveryStatus.Started;
}
else
{
ExecutionLog.LogWarning($"Producer={_producer.Id}: An error occurred while starting recovery operation with after={SdkInfo.ToEpochTime(_producer.LastTimestampBeforeDisconnect)}. Retry will be made at next system alive.");
newStatus = ProducerRecoveryStatus.Error;
}
}
catch (RecoveryInitiationException ex)
{
ExecutionLog.LogCritical($"Producer id={Producer.Id} failed to execute recovery because 'after' is to much in the past. After={ex.After}. Stopping the feed.");
newStatus = ProducerRecoveryStatus.FatalError;
}
}
}
}
}
catch (Exception ex)
{
ExecutionLog.LogError($"An unexpected exception occurred while processing system message. Producer={_producer.Id}, message={message}. Exception:", ex);
}
if (newStatus != Status)
{
SetStatusAndRaiseEvent(null, newStatus);
}
}
}