public async Task NotifyPartitionReleasedAsync(T lease, ChangeFeedObserverCloseReason reason)
{
foreach (var observer in this.observers)
{
await observer.OnPartitionReleasedAsync(lease, reason);
}
}
public async Task StopAsync(ChangeFeedObserverCloseReason reason)
{
if (Interlocked.CompareExchange(ref this.isStarted, 0, 1) != 1)
{
// idempotent
return;
}
if (this.takerTask != null)
{
this.leaseTakerCancellationTokenSource.Cancel();
await this.takerTask;
}
await this.ShutdownAsync(reason);
this.shutdownComplete = true;
if (this.renewTask != null)
{
this.leaseRenewerCancellationTokenSource.Cancel();
await this.renewTask;
}
this.leaseTakerCancellationTokenSource = null;
this.leaseRenewerCancellationTokenSource = null;
}
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
Console.WriteLine($"Worker closed, {context.PartitionKeyRangeId}, reason {reason}");
return(Task.CompletedTask);
}
/// <summary>
/// Called when change feed observer is closed;
/// this function prints out observer partition key id and reason for shut down.
/// </summary>
/// <param name="context">The context specifying partition for this observer, etc.</param>
/// <param name="reason">Specifies the reason the observer is closed.</param>
/// <returns>A Task to allow asynchronous execution</returns>
public Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("Observer closed, {0}", context.PartitionKeyRangeId);
Console.WriteLine("Reason for shutdown, {0}", reason);
return(Task.CompletedTask);
}
async Task StopAsync(ChangeFeedObserverCloseReason reason)
{
if (Interlocked.CompareExchange(ref this.isShutdown, 1, 0) != 0)
{
return;
}
TraceLog.Informational(string.Format("Host '{0}': STOP signal received!", this.HostName));
List <Task> closingTasks = new List <Task>();
// Trigger stop for PartitionManager so it triggers shutdown of AcquireLease task and starts processor shutdown
closingTasks.Add(this.partitionManager.StopAsync(reason));
// Stop all workers.
TraceLog.Informational(string.Format("Host '{0}': Cancelling {1} workers.", this.HostName, this.partitionKeyRangeIdToWorkerMap.Count));
foreach (var item in this.partitionKeyRangeIdToWorkerMap.Values)
{
item.Cancellation.Cancel();
closingTasks.Add(item.Task);
}
// wait for everything to shutdown
TraceLog.Informational(string.Format("Host '{0}': Waiting for {1} closing tasks...", this.HostName, closingTasks.Count));
await Task.WhenAll(closingTasks.ToArray());
this.partitionKeyRangeIdToWorkerMap.Clear();
if (this.leaseManager is IDisposable)
{
((IDisposable)this.leaseManager).Dispose();
}
TraceLog.Informational(string.Format("Host '{0}': stopped.", this.HostName));
}
public async Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
_ctsClose.Cancel();
await _checkpointTask;
Console.WriteLine("Closed:" + context.PartitionKeyRangeId + " due to " + reason);
}
public async Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
if (reason == ChangeFeedObserverCloseReason.ObserverError)
{
await subscriber.OnErrorAsync(this, new InvalidOperationException("Change feed observer failed."));
}
}
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
if (context == null)
{
throw new ArgumentNullException("context", "Missing observer context");
}
return(Task.CompletedTask);
}
public Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
this.logger.LogInformation("Observer closed, {0} - Reason for shutdown {1}",
context.PartitionKeyRangeId,
reason);
return(Task.CompletedTask);
}
async Task RemoveLeaseAsync(T lease, bool hasOwnership, ChangeFeedObserverCloseReason closeReason = ChangeFeedObserverCloseReason.Unknown)
{
ChangeFeedObserverCloseReason reason =
closeReason != ChangeFeedObserverCloseReason.Unknown ? closeReason :
hasOwnership ? ChangeFeedObserverCloseReason.Shutdown : ChangeFeedObserverCloseReason.LeaseLost;
if (lease != null && this.currentlyOwnedPartitions != null && this.currentlyOwnedPartitions.TryRemove(lease.PartitionId, out lease))
{
TraceLog.Informational(string.Format("Host '{0}' successfully removed PartitionId '{1}' with lease token '{2}' from currently owned partitions.", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
try
{
if (hasOwnership)
{
this.keepRenewingDuringClose.TryAdd(lease.PartitionId, lease);
}
TraceLog.Informational(string.Format("Host '{0}' closing event processor for PartitionId '{1}' and lease token '{2}' with reason '{3}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken, reason));
// Notify the host that we lost partition so shutdown can be triggered on the host
await this.partitionObserverManager.NotifyPartitionReleasedAsync(lease, reason);
TraceLog.Informational(string.Format("Host '{0}' closed event processor for PartitionId '{1}' and lease token '{2}' with reason '{3}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken, reason));
}
catch (Exception ex)
{
// Eat any exceptions during notification of observers
TraceLog.Exception(ex);
}
finally
{
if (hasOwnership)
{
this.keepRenewingDuringClose.TryRemove(lease.PartitionId, out lease);
}
}
if (hasOwnership)
{
try
{
await this.leaseManager.ReleaseAsync(lease);
TraceLog.Informational(string.Format("Host '{0}' successfully released lease on PartitionId '{1}' with lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
}
catch (LeaseLostException)
{
// We have already shutdown the processor so we can ignore any LeaseLost at this point
TraceLog.Informational(string.Format("Host '{0}' failed to release lease for PartitionId '{1}' with lease token '{2}' due to conflict.", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
}
catch (Exception ex)
{
TraceLog.Exception(ex);
}
}
}
}
async Task ShutdownAsync(ChangeFeedObserverCloseReason reason)
{
var shutdownTasks = this.currentlyOwnedPartitions.Values.Select <T, Task>((lease) =>
{
return(this.RemoveLeaseAsync(lease, true, reason));
});
await Task.WhenAll(shutdownTasks);
}
public override async Task RunAsync(CancellationToken shutdownToken)
{
await this.observer.OpenAsync(this.lease.CurrentLeaseToken).ConfigureAwait(false);
this.processorCancellation = CancellationTokenSource.CreateLinkedTokenSource(shutdownToken);
Task processorTask = this.processor.RunAsync(this.processorCancellation.Token);
processorTask.ContinueWith(_ => this.renewerCancellation.Cancel()).LogException();
Task renewerTask = this.renewer.RunAsync(this.renewerCancellation.Token);
renewerTask.ContinueWith(_ => this.processorCancellation.Cancel()).LogException();
ChangeFeedObserverCloseReason closeReason = shutdownToken.IsCancellationRequested ?
ChangeFeedObserverCloseReason.Shutdown :
ChangeFeedObserverCloseReason.Unknown;
try
{
await Task.WhenAll(processorTask, renewerTask).ConfigureAwait(false);
}
catch (LeaseLostException)
{
closeReason = ChangeFeedObserverCloseReason.LeaseLost;
throw;
}
catch (FeedRangeGoneException)
{
closeReason = ChangeFeedObserverCloseReason.LeaseGone;
throw;
}
catch (CosmosException)
{
closeReason = ChangeFeedObserverCloseReason.CosmosException;
throw;
}
catch (OperationCanceledException) when(shutdownToken.IsCancellationRequested)
{
closeReason = ChangeFeedObserverCloseReason.Shutdown;
}
catch (ChangeFeedProcessorUserException)
{
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
catch (Exception) when(processorTask.IsFaulted)
{
closeReason = ChangeFeedObserverCloseReason.Unknown;
throw;
}
finally
{
await this.observer.CloseAsync(this.lease.CurrentLeaseToken, closeReason).ConfigureAwait(false);
}
}
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
if (this.closeProcessor != null)
{
return(this.closeProcessor(context, reason));
}
else
{
return(Task.CompletedTask);
}
}
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
if (reason == ChangeFeedObserverCloseReason.LeaseLost)
{
// The lease is lost by ChangeFeedEventHost but it still exists and ChangeFeedEventHost
// keeps calling ProcessAsync which is IMHO a bug.
// There is an open issue on git hub: https://github.com/Azure/azure-documentdb-dotnet/issues/237
// Just be aware of this, if you create/dispose objects. Do not dispose objects immediatly in case of "LeaseLost"
// as workaround. At some later point in time the ChangeFeedEventHost disposes the Factory and creates a new one
}
return(Task.CompletedTask); // Framework 4.6 above
}
public override async Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
try
{
await this.changeFeedObserver.CloseAsync(context, reason).ConfigureAwait(false);
}
catch (Exception userException)
{
Logger.WarnException("Exception happened on Observer.CloseAsync", userException);
throw new ObserverException(userException);
}
}
public override async Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
try
{
await this.changeFeedObserver.CloseAsync(context, reason).ConfigureAwait(false);
}
catch (Exception userException)
{
Extensions.TraceException(userException);
DefaultTrace.TraceWarning("Exception happened on Observer.CloseAsync");
throw new ObserverException(userException);
}
}
public Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
Trace.TraceInformation("Observer closed, {0}", context.PartitionKeyRangeId);
Trace.TraceInformation("Reason for shutdown, {0}", reason);
return(Task.CompletedTask);
}
public Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
Log("CloseAsync." + reason.ToString());
return(Task.CompletedTask);
}
async Task IPartitionObserver <DocumentServiceLease> .OnPartitionReleasedAsync(DocumentServiceLease l, ChangeFeedObserverCloseReason reason)
{
#if DEBUG
Interlocked.Decrement(ref this.partitionCount);
#endif
TraceLog.Informational(string.Format("Host '{0}' releasing partition {1}...", this.HostName, l.PartitionId));
WorkerData workerData = null;
if (this.partitionKeyRangeIdToWorkerMap.TryGetValue(l.PartitionId, out workerData))
{
workerData.Cancellation.Cancel();
try
{
await workerData.Observer.CloseAsync(workerData.Context, reason);
}
catch (Exception ex)
{
// Eat all client exceptions.
TraceLog.Error(string.Format("IChangeFeedObserver.CloseAsync: exception: {0}", ex));
}
await workerData.Task;
this.partitionKeyRangeIdToWorkerMap.TryRemove(l.PartitionId, out workerData);
}
TraceLog.Informational(string.Format("Host '{0}' partition {1}: released!", this.HostName, workerData.Context.PartitionKeyRangeId));
}
public Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
return(Task.CompletedTask); // Note: requires targeting .Net 4.6+.
}
public Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
return(Task.CompletedTask);
}
public Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
Log.Information($"SampleObserver::CloseAsync for {context.PartitionKeyRangeId} with {reason}");
return(Task.CompletedTask);
}
public override Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
return(this.observer.CloseAsync(context, reason));
}
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
return(parent.CloseAsync(context, reason));
}
public async Task RunAsync(CancellationToken shutdownToken)
{
var context = new ChangeFeedObserverContext(this.lease.PartitionId);
await this.observer.OpenAsync(context).ConfigureAwait(false);
this.processorCancellation = CancellationTokenSource.CreateLinkedTokenSource(shutdownToken);
Task processorTask = this.processor.RunAsync(this.processorCancellation.Token);
processorTask.ContinueWith(_ => this.renewerCancellation.Cancel()).LogException();
Task renewerTask = this.renewer.RunAsync(this.renewerCancellation.Token);
renewerTask.ContinueWith(_ => this.processorCancellation.Cancel()).LogException();
ChangeFeedObserverCloseReason closeReason = shutdownToken.IsCancellationRequested ?
ChangeFeedObserverCloseReason.Shutdown :
ChangeFeedObserverCloseReason.Unknown;
try
{
await Task.WhenAll(processorTask, renewerTask).ConfigureAwait(false);
}
catch (LeaseLostException)
{
closeReason = ChangeFeedObserverCloseReason.LeaseLost;
throw;
}
catch (PartitionSplitException)
{
closeReason = ChangeFeedObserverCloseReason.LeaseGone;
throw;
}
catch (PartitionNotFoundException)
{
closeReason = ChangeFeedObserverCloseReason.ResourceGone;
throw;
}
catch (ReadSessionNotAvailableException)
{
closeReason = ChangeFeedObserverCloseReason.ReadSessionNotAvailable;
throw;
}
catch (OperationCanceledException) when(shutdownToken.IsCancellationRequested)
{
closeReason = ChangeFeedObserverCloseReason.Shutdown;
}
catch (ObserverException)
{
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
catch (Exception) when(processorTask.IsFaulted)
{
closeReason = ChangeFeedObserverCloseReason.Unknown;
throw;
}
finally
{
await this.observer.CloseAsync(context, closeReason).ConfigureAwait(false);
}
}
public Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
Console.WriteLine("Closing partition processing:" + context.PartitionKeyRangeId);
return(Task.CompletedTask);
}
public override Task CloseAsync(
string leaseToken,
ChangeFeedObserverCloseReason reason)
{
return(Task.CompletedTask);
}
/// <summary>
/// Called when change feed observer is closed;
/// this function prints out observer partition key id and reason for shut down.
/// </summary>
/// <param name="context">The context specifying partition for this observer, etc.</param>
/// <param name="reason">Specifies the reason the observer is closed.</param>
/// <returns>A Task to allow asynchronous execution</returns>
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
Console.WriteLine("Observer closed, {0}", context.PartitionKeyRangeId);
Console.WriteLine("Reason for shutdown, {0}", reason);
return(Task.FromResult <object>(null));
}
Task IChangeFeedObserver.CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason) => Task.CompletedTask;
public async Task TryReleasePartitionAsync(string partitionId, bool hasOwnership, ChangeFeedObserverCloseReason closeReason)
{
T lease;
if (this.currentlyOwnedPartitions.TryGetValue(partitionId, out lease))
{
await this.RemoveLeaseAsync(lease, hasOwnership, closeReason);
}
}
