private void Dropped(EventStoreSubscription subscription, SubscriptionDropReason subscriptionDropReason, Exception exception)
{
var message = string.Format("Subscription {0} dropped: {1} (Recovery currently not implemented){2}{3}",
subscription.StreamId, subscriptionDropReason, Environment.NewLine, exception);
_console.Error(message);
}
public void SetUp()
{
_connection = new FakeEventStoreConnection();
_raisedEvents = new List<ResolvedEvent>();
_dropEvent = new ManualResetEventSlim();
_raisedEventEvent = new ManualResetEventSlim();
_liveProcessingStarted = false;
_isDropped = false;
_dropReason = SubscriptionDropReason.Unknown;
_dropException = null;
var settings = new CatchUpSubscriptionSettings(1, 1, false, false);
_subscription = new EventStoreStreamCatchUpSubscription(_connection, new NoopLogger(), StreamId, null, null,
(subscription, ev) =>
{
_raisedEvents.Add(ev);
_raisedEventEvent.Set();
},
subscription =>
{
_liveProcessingStarted = true;
},
(subscription, reason, ex) =>
{
_isDropped = true;
_dropReason = reason;
_dropException = ex;
_dropEvent.Set();
},
settings);
}
private void Dropped(EventStoreCatchUpSubscription sub,SubscriptionDropReason reason,Exception ex)
{
//Reconnect if we drop
//TODO: check the reason and handle it appropriately
_view.ErrorMsg = "Subscription Dropped, press Enter to reconnect";
Subscribe();
}
protected override void Given()
{
_conn.AppendToStreamAsync(_stream, ExpectedVersion.Any,
new EventData(Guid.NewGuid(), "whatever", true, Encoding.UTF8.GetBytes("{'foo' : 2}"), new Byte[0]));
_conn.CreatePersistentSubscriptionAsync(_stream, "existing", _settings, DefaultData.AdminCredentials).Wait();
_conn.ConnectToPersistentSubscription(_stream, "existing" , (x, y) => { },
(sub, reason, ex) =>
{
_dropped.Set();
_reason = reason;
_exception = ex;
});
}
protected override void Given()
{
_response = MakeJsonPut(
string.Format("/subscriptions/{0}/{1}", _stream, _groupName),
new
{
ResolveLinkTos = true
}, _admin);
_connection.ConnectToPersistentSubscription(_stream, _groupName, (x, y) => { },
(sub, reason, e) =>
{
_dropped.Set();
_reason = reason;
_exception = e;
});
}
private void SubscriptionDropped(EventStoreCatchUpSubscription sub, SubscriptionDropReason reason,
Exception ex)
{
_logger.LogError(ex,
"SubscriptionDropped on StreamId {StreamId}, Projector {ProjectorId}, Reason: {Reason}",
sub.StreamId,
ProjectorId,
reason.ToString());
if (!_cancellationToken.IsCancellationRequested)
{
// _logger.LogInformation("Re Subscribe Subscription");
_subscriptionDropped = true;
// await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None)
// .ContinueWith(task => StartSubscriptionInternal(), CancellationToken.None);
//await StartSubscriptionInternal().ConfigureAwait(false);
}
}
void OnSubscriptionDropped(EventStoreSubscription sub, SubscriptionDropReason reason, Exception error)
{
if (reason == SubscriptionDropReason.UserInitiated)
{
return;
}
Logger.Error("Exchanges subscription dropped.", error);
try
{
subscription = connection.SubscribeToStreamAsync(ExchangeRepository.StreamName, true, OnNewVersion, OnSubscriptionDropped)
.GetAwaiter()
.GetResult();
circuitBreaker.Success();
}
catch (Exception ex)
{
circuitBreaker.Failure(ex).GetAwaiter().GetResult();
}
}
private void DropSubscription(SubscriptionDropReason reason, Exception exception)
{
if (Interlocked.CompareExchange(ref _unsubscribed, 1, 0) == 0)
{
if (reason != SubscriptionDropReason.UserInitiated)
{
if (exception == null) throw new Exception(string.Format("No exception provided for subscription drop reason '{0}", reason));
_source.TrySetException(exception);
}
if (reason == SubscriptionDropReason.UserInitiated && _subscription != null)
_publisher.Publish(new ClientMessage.UnsubscribeFromStream(Guid.NewGuid(), _correlationId, new NoopEnvelope(), SystemAccount.Principal));
if (_subscription != null)
ExecuteActionAsync(() => _subscriptionDropped(_subscription, reason, exception));
}
}
private void SubscriptionDropped(EventStorePersistentSubscriptionBase eventStorePersistentSubscriptionBase,
SubscriptionDropReason subscriptionDropReason, Exception ex)
{
if (_cancellationRequestedByUser)
{
if (_connection != null)
{
_connection.Dispose();
}
_logger.LogInformation(ex, $"Subscription stopped by user: {subscriptionDropReason.ToString()}");
_status = SubscriptionConnectionStatus.Disconnected;
return;
}
_logger.LogError(ex, $"Subscription dropped: {subscriptionDropReason.ToString()}");
_connection.Dispose();
_status = SubscriptionConnectionStatus.Disconnected;
Connect().Wait();
}
private void SubscriptionDropped(EventStoreCatchUpSubscription subscription, SubscriptionDropReason reason, Exception exception)
{
OnSubscriptionDropped(subscription, reason, exception);
if (reason == SubscriptionDropReason.UserInitiated)
{
return;
}
if (_subscriptionTerminatedRequested)
{
return;
}
if (reason == SubscriptionDropReason.EventHandlerException ||
reason == SubscriptionDropReason.CatchUpError ||
reason == SubscriptionDropReason.ProcessingQueueOverflow)
{
if (exception is ServerErrorException serverError)
{
OnServerError(serverError);
return;
}
StartNewSubscriptionEpoch();
}
if (reason == SubscriptionDropReason.ConnectionClosed)
{
Log.Information(
"Subscription {Name} with epoch {Epoch} dropped with {Reason}, which means that either the single node is dead and no other node found through gossip. Waiting for reconnect to start a new epoch.",
_subscriptionName,
_subscriptionEpoch,
SubscriptionDropReason.ConnectionClosed);
_subscriptionEpochStopping = true;
_catchUpSubscription.Stop();
return;
}
Log.Information("Subscription dropped with reason {Reason}", reason);
}
private void DropSubscription(SubscriptionDropReason reason, Exception error)
{
if (Interlocked.CompareExchange(ref _isDropped, 1, 0) == 0)
{
if (Verbose)
{
Log.Debug("Catch-up Subscription to {0}: dropping subscription, reason: {1} {2}.",
IsSubscribedToAll ? "<all>" : StreamId, reason, error == null ? string.Empty : error.ToString());
}
if (_subscription != null)
{
_subscription.Unsubscribe();
}
if (_subscriptionDropped != null)
{
_subscriptionDropped(this, reason, error);
}
_stopped.Set();
}
}
void HandleDrop(EventStoreSubscription subscription, SubscriptionDropReason reason, Exception exception)
{
Log.Info("Connection dropped because {Reason}", reason);
if (reason is SubscriptionDropReason.UserInitiated)
{
return;
}
_started = false;
var task = reason is SubscriptionDropReason.ConnectionClosed ?StartAfterDelay() : Start();
Task.Run(() => task);
async Task StartAfterDelay()
{
await Task.Delay(5000).ConfigureAwait(false);
await Start().ConfigureAwait(false);
}
}
void SubscriptionDropped(EventStoreCatchUpSubscription projection, SubscriptionDropReason subscriptionDropReason, Exception exception)
{
Connection.Connected -= Connection_Connected;
Subscription.Stop();
if (IsTransient(subscriptionDropReason))
{
ReconnectionCounter++;
if (ReconnectionCounter > MaxRecconectionAttemts)
{
LogAndFail();
}
Logger.LogWarning(exception, $"{StreamName} subscription dropped because of an transient error: ({subscriptionDropReason}). Reconnection attempt nr: {ReconnectionCounter}.");
Thread.Sleep(300);
Start(CurrentCheckpoint).Wait();
}
else
{
Logger.LogCritical(exception, $"{StreamName} subscription failed: ({subscriptionDropReason}).");
throw exception;
}
}
private void OnSubscriptionDropped(SubscriptionDropReason dropReason, Exception ex, Action <Exception> pumpError, Action <IEvent> pumpEvent)
{
// If the drop reasons indicate that this was something straightforward, we should recreate the subscription
if (dropReason == SubscriptionDropReason.UserInitiated ||
dropReason == SubscriptionDropReason.EventHandlerException)
{
// Wait for the ConnectionState == Connected before recreating
CreateConnectionStateChangedStream()
.Where(state => state == StreamingConnectionState.Connected)
.Take(1)
.Subscribe(async _ =>
{
_allEventsSubscription.Disposable = await SubscribeToAllEvents(pumpEvent, pumpError);
});
return;
}
// If the connection was dropped for another reason, don't recreate and instead pump an exception to the consumer
// TODO: Logging here
pumpError(ex);
}
private void SubscriptionDropped(EventStorePersistentSubscriptionBase sub, SubscriptionDropReason reason, Exception ex)
{
Live = false;
Logger.Write(LogLevel.Info, () => $"Disconnected from subscription. Reason: {reason} Exception: {ex}");
// Todo: is it possible to ACK an event from a reconnection?
//if (_toAck.Any())
// throw new InvalidOperationException(
// $"Eventstore subscription dropped and we need to ACK {_toAck.Count} more events");
// Need to clear ReadyEvents of events delivered but not processed before disconnect
Queued.Decrement(Id, _waitingEvents.Count);
lock (_lock) _waitingEvents.Clear();
if (reason == SubscriptionDropReason.UserInitiated)
{
return;
}
//Task.Run(Connect, _token);
}
internal void DropSubscription(SubscriptionDropReason reason, Exception error)
{
if (Interlocked.CompareExchange(ref _isDropped, 1, 0) != 0)
{
return;
}
if (Verbose)
{
Log.Debug("Catch-up Subscription {0} to {1}: dropping subscription, reason: {2} {3}.", SubscriptionName, IsSubscribedToAll ? "<all>" : StreamId, reason, error == null ? string.Empty : error.ToString());
}
StreamSubscription subscription = _subscription;
if (subscription != null)
{
subscription.Unsubscribe();
}
Action <CatchUpSubscription, SubscriptionDropReason, Exception> subscriptionDropped = _subscriptionDropped;
if (subscriptionDropped != null)
{
subscriptionDropped(this, reason, error);
}
_stopped.Set();
}
private void DropSubscription(SubscriptionDropReason reason, Exception exception)
{
if (Interlocked.CompareExchange(ref _unsubscribed, 1, 0) == 0)
{
if (reason != SubscriptionDropReason.UserInitiated)
{
if (exception == null)
{
throw new Exception(string.Format("No exception provided for subscription drop reason '{0}", reason));
}
_source.TrySetException(exception);
}
if (reason == SubscriptionDropReason.UserInitiated && _subscription != null)
{
_publisher.Publish(new ClientMessage.UnsubscribeFromStream(Guid.NewGuid(), _correlationId, new NoopEnvelope(), SystemAccount.Principal));
}
if (_subscription != null)
{
ExecuteActionAsync(() => _subscriptionDropped(_subscription, reason, exception));
}
}
}
private void SubscriptionDropped(SubscriptionDropReason reason, Exception exception)
{
_log.Trace($"Listener {ListenerName} subscription dropped, reason: {reason}, ex: {exception}");
_bus.Publish(new Domain.Foundation.StreamStoreMsgs.SubscriptionDropped(ListenerName, reason, exception));
_started = false;
// resubscribe
Thread.Sleep(_resubscribeTimeout);
_subscription?.Dispose();
if (reason == SubscriptionDropReason.UserInitiated)
{
// do not reconnect
return;
}
_log.Trace($"Listener {ListenerName} subscription restarting");
_subscription =
SubscribeToStreamFrom(
StreamName,
Position,
true,
eventAppeared: GotEvent,
liveProcessingStarted: () =>
{
_bus.Publish(new StreamStoreMsgs.CatchupSubscriptionBecameLive());
_liveLock.Set();
},
subscriptionDropped: (r, e) => SubscriptionDropped(reason, exception));
_started = true;
if (_resubscribeTimeout < 60000)
{
_resubscribeTimeout *= 2;
}
}
private void DropSubscription(SubscriptionDropReason reason, Exception error)
{
if (Interlocked.CompareExchange(ref _isDropped, 1, 0) == 0)
{
if (Verbose)
Log.Debug("Catch-up Subscription to {0}: dropping subscription, reason: {1} {2}.",
IsSubscribedToAll ? "<all>" : StreamId, reason, error == null ? string.Empty : error.ToString());
if (_subscription != null)
_subscription.Unsubscribe();
if (_subscriptionDropped != null)
_subscriptionDropped(this, reason, error);
_stopped.Set();
}
}
private void ServerSubscriptionDropped(EventStoreSubscription subscription, SubscriptionDropReason reason, Exception exc)
{
EnqueueSubscriptionDropNotification(reason, exc);
}
private void Dropped(EventStorePersistentSubscriptionBase sub, SubscriptionDropReason reason, Exception exception)
{
_exception = exception;
_reason = reason;
_resetEvent.Set();
}
public SubscriptionDropped(Guid correlationId, SubscriptionDropReason reason)
{
CorrelationId = correlationId;
Reason = reason;
}
private void SubscriptionDropped(EventStorePersistentSubscriptionBase eventStorePersistentSubscriptionBase,
SubscriptionDropReason subscriptionDropReason, Exception ex)
{
ConnectToSubscription();
}
private void SubscriptionDropped(EventStoreCatchUpSubscription eventStoreCatchUpSubscription, SubscriptionDropReason subscriptionDropReason, Exception ex)
{
if (ex != null)
{
_logger.Info(ex, "Event Store subscription dropped {0}", subscriptionDropReason.ToString());
}
else
{
_logger.Info("Event Store subscription dropped {0}", subscriptionDropReason.ToString());
}
_viewModelIsReady = false;
lock (_liveProcessingTimer)
{
if(!_liveProcessingTimer.Enabled)
_liveProcessingTimer.Start();
}
}
private static void SubscriptionDropped(EventStorePersistentSubscriptionBase arg1, SubscriptionDropReason arg2, Exception arg3)
{
Console.WriteLine(arg2);
Console.WriteLine(arg3.GetBaseException().Message);
}
private void TryToReconnect(EventStoreCatchUpSubscription catchUpSubscription, SubscriptionDropReason reason, Exception exception)
{
_console.Error("Projection subscription dropped: " + reason, exception);
Connect();
}
private static void Dropped(EventStoreCatchUpSubscription subscription, SubscriptionDropReason subscriptionDropReason, Exception exception)
{
}
private static void Dropped(EventStoreCatchUpSubscription arg1, SubscriptionDropReason arg2, Exception arg3)
{
}
private void OnSubscriptionDropped(EventStoreCatchUpSubscription subscription, SubscriptionDropReason reason, Exception exception)
{
_log.Error(string.Format("Event subscription stopped. Reason: {0}", reason), exception);
}
private void SubscriptionDropped(EventStoreSubscription subscription, SubscriptionDropReason reason, Exception exception)
{
_log.Error(exception, "Subscription dropped. Reason: {0}", reason);
SubscribeToStream();
}
private void ServerSubscriptionDropped(EventStoreSubscription subscription, SubscriptionDropReason reason, Exception exc)
{
EnqueueSubscriptionDropNotification(reason, exc);
}
private void Dropped(EventStoreSubscription subscription, SubscriptionDropReason subscriptionDropReason, Exception exception)
{
_console.Error("Subscription {0} dropped: {1} (Currently no recovery implemented){2}{3}", subscription.StreamId, subscriptionDropReason, Environment.NewLine, exception);
}
private async void SubscriptionDropped(EventStorePersistentSubscriptionBase subscription, SubscriptionDropReason reason, Exception ex) {
await ConnectSubscription();
}
private void SubscriptionDropped(EventStoreCatchUpSubscription _, SubscriptionDropReason reason, Exception ex)
{
if(reason == SubscriptionDropReason.UserInitiated)
{
return;
}
RecoverSubscription().Wait(TimeSpan.FromSeconds(2));
}
private void OnSubscriptionDropped(EventStoreSubscription subscription, SubscriptionDropReason reason, Exception exception)
{
}
/// <summary>
/// Catches up subscription dropped.
/// </summary>
/// <param name="eventStoreCatchUpSubscription">The event store catch up subscription.</param>
/// <param name="subscriptionDropReason">The subscription drop reason.</param>
/// <param name="exception">The exception.</param>
/// <param name="subscriptionGroupId">The subscription group identifier.</param>
private void CatchUpSubscriptionDropped(EventStoreCatchUpSubscription eventStoreCatchUpSubscription, SubscriptionDropReason subscriptionDropReason, Exception exception, Guid subscriptionGroupId)
{
if (this.SubscriptionDropped != null)
{
this.SubscriptionDropped(eventStoreCatchUpSubscription.StreamId, String.Empty, SubscriptionType.CatchUp,
subscriptionDropReason, exception, subscriptionGroupId);
}
else
{
Logger.LogInformation("Unable to process event as SubscriptionDropped Event handler is null");
}
}
void OnDropped(SubscriptionDropReason reason, Exception error) => _observer.OnDropped(reason.ToString(), error);
private void SubscriptionDropped(EventStoreSubscription arg1, SubscriptionDropReason arg2, Exception arg3)
{
Subsribe();
}
private void SubscriptionDropped(EventStorePersistentSubscriptionBase eventStorePersistentSubscriptionBase, SubscriptionDropReason subscriptionDropReason, Exception ex)
{
Console.WriteLine($"**** Connection dropped reason? '{subscriptionDropReason}' exception? '{ex.Message}'- Reconnecting ...");
ConnectToSubscription();
}
/// <summary>
/// Callback used to handle a dropped subscription to the Name category event stream.
/// </summary>
/// <param name="subscription"></param>
/// <param name="reason"></param>
/// <param name="exception"></param>
private static void ReconnectToStreamSubscription(EventStoreCatchUpSubscription subscription, SubscriptionDropReason reason, Exception exception)
{
throw new Exception($"Subscription to stream ended due to reason: {exception.Message}");
}
public SubscriptionDropped(string listenerName, SubscriptionDropReason reason, Exception exception)
{
ListenerName = listenerName;
Reason = reason;
Exception = exception;
}
public DropData(SubscriptionDropReason reason, Exception error)
{
Reason = reason;
Error = error;
}
private async void SubscriptionDropped(EventStoreCatchUpSubscription eventStoreCatchUpSubscription, SubscriptionDropReason subscriptionDropReason, Exception exception) => await HandleDroppedSubscription(new DroppedSubscription(subscription, exception.Message, subscriptionDropReason));
private void HandleSubscriptionDropped(EventStoreCatchUpSubscription subscription, SubscriptionDropReason reason, Exception ex)
{
}
private void SubscriptionDropped(EventStoreSubscription eventStorePersistentSubscriptionBase,
SubscriptionDropReason subscriptionDropReason, Exception ex)
{
ConnectToSubscription();
}
private void SubscriptionDropped(EventStoreSubscription sub, SubscriptionDropReason reason, Exception exception)
{
Log.Debug(_adapterName + " lost subscription due to " + reason);
}
private void HandleSubscriptionDropped(EventStoreCatchUpSubscription subscription, SubscriptionDropReason reason, Exception ex)
{
}
internal void DropSubscription(SubscriptionDropReason reason, Exception exc, TcpPackageConnection connection = null)
{
if (Interlocked.CompareExchange(ref _unsubscribed, 1, 0) == 0)
{
if (_verboseLogging)
_log.Debug("Subscription {0:B} to {1}: closing subscription, reason: {2}, exception: {3}...",
_correlationId, _streamId == string.Empty ? "<all>" : _streamId, reason, exc);
if (reason != SubscriptionDropReason.UserInitiated)
{
if (exc == null) throw new Exception(string.Format("No exception provided for subscription drop reason '{0}", reason));
_source.TrySetException(exc);
}
if (reason == SubscriptionDropReason.UserInitiated && _subscription != null && connection != null)
connection.EnqueueSend(CreateUnsubscriptionPackage());
if (_subscription != null)
ExecuteActionAsync(() => _subscriptionDropped(_subscription, reason, exc));
}
}
private void EnqueueSubscriptionDropNotification(SubscriptionDropReason reason, Exception error)
{
// if drop data was already set -- no need to enqueue drop again, somebody did that already
var dropData = new DropData(reason, error);
if (Interlocked.CompareExchange(ref _dropData, dropData, null) == null)
{
_liveQueue.Enqueue(DropSubscriptionEvent);
if (_allowProcessing)
EnsureProcessingPushQueue();
}
}
private void CommentChangedSubscriptionDropped(EventStoreSubscription sub, SubscriptionDropReason reason,
Exception ex)
{
SubscribeToStreamComment();
}
public DropData(SubscriptionDropReason reason, Exception error)
{
Reason = reason;
Error = error;
}
private void TodoChangedSubscriptionDropped(EventStoreSubscription sub, SubscriptionDropReason reason,
Exception ex)
{
SubscribeToStreamTodo();
}
public SubscriptionDropped(Guid correlationId, SubscriptionDropReason reason)
{
CorrelationId = correlationId;
Reason = reason;
}
public void OnSubscriptionDropped(EventStoreCatchUpSubscription subscription, SubscriptionDropReason reason, Exception error)
{
_logger.LogCritical("Catch up on {0} failure: {1}", _streamName, error.ToString());
}
void SubscriptionDropped(EventStorePersistentSubscriptionBase droppedSubscription, SubscriptionDropReason dropReason, Exception e)
{
if (dropReason == SubscriptionDropReason.UserInitiated)
{
return;
}
Logger.Error("Message pump subscription dropped: " + dropReason, e);
try
{
subscription = connection.ConnectToPersistentSubscription(inputQueue, inputQueue, OnEvent, SubscriptionDropped);
}
catch (Exception ex)
{
criticalError.Raise("Can't reconnect to EventStore.", ex);
}
}
private void SubscriptionDropped(EventStoreCatchUpSubscription arg1, SubscriptionDropReason arg2,
Exception arg3)
{
SubscribeMe();
}
private void SetupSubscription()
{
_connection.ConnectToPersistentSubscription(_stream,_groupName, (x, y) => { },
(sub, reason, ex) =>
{
_droppedReason = reason;
_exception = ex;
_dropped.Set();
}, DefaultData.AdminCredentials);
}
private void DisplaySubscriptionDropped(EventStoreSubscription subscription, SubscriptionDropReason reasonDropped, Exception exception)
{
Console.WriteLine("Opps");
}
public DroppedSubscription(Subscription subscription, string exceptionMessage, SubscriptionDropReason dropReason)
{
StreamId = subscription.StreamId;
ExceptionMessage = exceptionMessage;
DropReason = dropReason;
RetryPolicy = subscription.RetryPolicy;
}
private void SubscriptionDropped(EventStoreSubscription sub, SubscriptionDropReason reason, Exception exception)
{
Log.Debug(_adapterName + " lost subscription due to " + reason);
}
