async Task <T> RenewLeaseAsync(T lease)
{
T renewedLease = null;
try
{
TraceLog.Informational(string.Format("Host '{0}' renewing lease for PartitionId '{1}' with lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
renewedLease = await this.leaseManager.RenewAsync(lease);
}
catch (LeaseLostException)
{
TraceLog.Informational(string.Format("Host '{0}' got LeaseLostException trying to renew lease for PartitionId '{1}' with lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
}
catch (Exception ex)
{
TraceLog.Exception(ex);
// Eat any exceptions during renew and keep going.
// Consider the lease as renewed. Maybe lease store outage is causing the lease to not get renewed.
renewedLease = lease;
}
finally
{
TraceLog.Informational(string.Format("Host '{0}' attempted to renew lease for PartitionId '{1}' and lease token '{2}' with result: '{3}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken, renewedLease != null));
}
return(renewedLease);
}
async Task RemoveLeaseAsync(T lease, bool hasOwnership, ChangeFeedObserverCloseReason closeReason = ChangeFeedObserverCloseReason.Unknown)
{
ChangeFeedObserverCloseReason reason =
closeReason != ChangeFeedObserverCloseReason.Unknown ? closeReason :
hasOwnership ? ChangeFeedObserverCloseReason.Shutdown : ChangeFeedObserverCloseReason.LeaseLost;
if (lease != null && this.currentlyOwnedPartitions != null && this.currentlyOwnedPartitions.TryRemove(lease.PartitionId, out lease))
{
TraceLog.Informational(string.Format("Host '{0}' successfully removed PartitionId '{1}' with lease token '{2}' from currently owned partitions.", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
try
{
if (hasOwnership)
{
this.keepRenewingDuringClose.TryAdd(lease.PartitionId, lease);
}
TraceLog.Informational(string.Format("Host '{0}' closing event processor for PartitionId '{1}' and lease token '{2}' with reason '{3}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken, reason));
// Notify the host that we lost partition so shutdown can be triggered on the host
await this.partitionObserverManager.NotifyPartitionReleasedAsync(lease, reason);
TraceLog.Informational(string.Format("Host '{0}' closed event processor for PartitionId '{1}' and lease token '{2}' with reason '{3}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken, reason));
}
catch (Exception ex)
{
// Eat any exceptions during notification of observers
TraceLog.Exception(ex);
}
finally
{
if (hasOwnership)
{
this.keepRenewingDuringClose.TryRemove(lease.PartitionId, out lease);
}
}
if (hasOwnership)
{
try
{
await this.leaseManager.ReleaseAsync(lease);
TraceLog.Informational(string.Format("Host '{0}' successfully released lease on PartitionId '{1}' with lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
}
catch (LeaseLostException)
{
// We have already shutdown the processor so we can ignore any LeaseLost at this point
TraceLog.Informational(string.Format("Host '{0}' failed to release lease for PartitionId '{1}' with lease token '{2}' due to conflict.", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
}
catch (Exception ex)
{
TraceLog.Exception(ex);
}
}
}
}
async Task AddLeaseAsync(T lease)
{
if (this.currentlyOwnedPartitions.TryAdd(lease.PartitionId, lease))
{
bool failedToInitialize = false;
try
{
TraceLog.Informational(string.Format("Host '{0}' opening event processor for PartitionId '{1}' and lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
await this.partitionObserverManager.NotifyPartitionAcquiredAsync(lease);
TraceLog.Informational(string.Format("Host '{0}' opened event processor for PartitionId '{1}' and lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
}
catch (Exception ex)
{
TraceLog.Informational(string.Format("Host '{0}' failed to initialize processor for PartitionId '{1}' and lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
failedToInitialize = true;
// Eat any exceptions during notification of observers
TraceLog.Exception(ex);
}
// We need to release the lease if we fail to initialize the processor, so some other node can pick up the parition
if (failedToInitialize)
{
await this.RemoveLeaseAsync(lease, true, ChangeFeedObserverCloseReason.ObserverError);
}
}
else
{
// We already acquired lease for this partition but it looks like we previously owned this partition
// and haven't completed the shutdown process for it yet. Release lease for possible others hosts to
// pick it up.
try
{
TraceLog.Warning(string.Format("Host '{0}' unable to add PartitionId '{1}' with lease token '{2}' to currently owned partitions.", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
await this.leaseManager.ReleaseAsync(lease);
TraceLog.Informational(string.Format("Host '{0}' successfully released lease on PartitionId '{1}' with lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
}
catch (LeaseLostException)
{
// We have already shutdown the processor so we can ignore any LeaseLost at this point
TraceLog.Informational(string.Format("Host '{0}' failed to release lease for PartitionId '{1}' with lease token '{2}' due to conflict.", this.workerName, lease.PartitionId, lease.ConcurrencyToken));
}
catch (Exception ex)
{
TraceLog.Exception(ex);
}
}
}
async Task <T> TryAcquireLeaseAsync(T lease)
{
try
{
return(await this.leaseManager.AcquireAsync(lease, this.workerName));
}
catch (LeaseLostException)
{
TraceLog.Informational(string.Format("Host '{0}' failed to acquire lease for PartitionId '{1}' due to conflict.", this.workerName, lease.PartitionId));
}
catch (Exception ex)
{
// Eat any exceptions during acquiring lease.
TraceLog.Exception(ex);
}
return(null);
}
async Task <T> TryStealLeaseAsync(T lease)
{
try
{
return(await this.leaseManager.AcquireAsync(lease, this.workerName));
}
catch (LeaseLostException)
{
// Concurrency issue in stealing the lease, someone else got it before us
TraceLog.Informational(string.Format("Host '{0}' failed to steal lease for PartitionId '{1}' due to conflict.", this.workerName, lease.PartitionId));
}
catch (Exception ex)
{
// Eat any exceptions during stealing
TraceLog.Exception(ex);
}
return(null);
}
public async Task <IDisposable> SubscribeAsync(IPartitionObserver <T> observer)
{
if (!this.observers.Contains(observer))
{
this.observers.Add(observer);
foreach (var lease in this.partitionManager.currentlyOwnedPartitions.Values)
{
try
{
await observer.OnPartitionAcquiredAsync(lease);
}
catch (Exception ex)
{
// Eat any exceptions during notification of observers
TraceLog.Exception(ex);
}
}
}
return(new Unsubscriber(this.observers, observer));
}
async Task LeaseTakerAsync()
{
while (this.isStarted == 1)
{
try
{
TraceLog.Informational(string.Format("Host '{0}' starting to check for available leases.", this.workerName));
var availableLeases = await this.TakeLeasesAsync();
if (availableLeases.Count > 0)
{
TraceLog.Informational(string.Format("Host '{0}' adding {1} leases...", this.workerName, availableLeases.Count));
}
var addLeaseTasks = new List <Task>();
foreach (var kvp in availableLeases)
{
addLeaseTasks.Add(this.AddLeaseAsync(kvp.Value));
}
await Task.WhenAll(addLeaseTasks.ToArray());
}
catch (Exception ex)
{
TraceLog.Exception(ex);
}
try
{
await Task.Delay(this.options.LeaseAcquireInterval, this.leaseTakerCancellationTokenSource.Token);
}
catch (OperationCanceledException)
{
TraceLog.Informational(string.Format("Host '{0}' AcquireLease task canceled.", this.workerName));
}
}
TraceLog.Informational(string.Format("Host '{0}' AcquireLease task completed.", this.workerName));
}
async Task LeaseRenewer()
{
while (this.isStarted == 1 || !this.shutdownComplete)
{
try
{
TraceLog.Informational(string.Format("Host '{0}' starting renewal of Leases.", this.workerName));
ConcurrentBag <T> renewedLeases = new ConcurrentBag <T>();
ConcurrentBag <T> failedToRenewLeases = new ConcurrentBag <T>();
List <Task> renewTasks = new List <Task>();
// Renew leases for all currently owned partitions in parallel
foreach (T lease in this.currentlyOwnedPartitions.Values)
{
renewTasks.Add(this.RenewLeaseAsync(lease).ContinueWith(renewResult =>
{
if (renewResult.Result != null)
{
renewedLeases.Add(renewResult.Result);
}
else
{
// Keep track of all failed attempts to renew so we can trigger shutdown for these partitions
failedToRenewLeases.Add(lease);
}
}));
}
// Renew leases for all partitions currently in shutdown
List <T> failedToRenewShutdownLeases = new List <T>();
foreach (T shutdownLeases in this.keepRenewingDuringClose.Values)
{
renewTasks.Add(this.RenewLeaseAsync(shutdownLeases).ContinueWith(renewResult =>
{
if (renewResult.Result != null)
{
renewedLeases.Add(renewResult.Result);
}
else
{
// Keep track of all failed attempts to renew shutdown leases so we can remove them from further renew attempts
failedToRenewShutdownLeases.Add(shutdownLeases);
}
}));
}
// Wait for all renews to complete
await Task.WhenAll(renewTasks.ToArray());
// Update renewed leases.
foreach (T lease in renewedLeases)
{
bool updateResult = this.currentlyOwnedPartitions.TryUpdate(lease.PartitionId, lease, lease);
if (!updateResult)
{
TraceLog.Warning(string.Format("Host '{0}' Renewed lease {1} but failed to update it in the map (ignorable).", this.workerName, lease));
}
}
// Trigger shutdown of all partitions we failed to renew leases
await failedToRenewLeases.ForEachAsync(
async lease => await this.RemoveLeaseAsync(lease, false, ChangeFeedObserverCloseReason.LeaseLost),
this.options.DegreeOfParallelism);
// Now remove all failed renewals of shutdown leases from further renewals
foreach (T failedToRenewShutdownLease in failedToRenewShutdownLeases)
{
T removedLease = null;
this.keepRenewingDuringClose.TryRemove(failedToRenewShutdownLease.PartitionId, out removedLease);
}
await Task.Delay(this.options.LeaseRenewInterval, this.leaseRenewerCancellationTokenSource.Token);
}
catch (OperationCanceledException)
{
TraceLog.Informational(string.Format("Host '{0}' Renewer task canceled.", this.workerName));
}
catch (Exception ex)
{
TraceLog.Exception(ex);
}
}
this.currentlyOwnedPartitions.Clear();
this.keepRenewingDuringClose.Clear();
TraceLog.Informational(string.Format("Host '{0}' Renewer task completed.", this.workerName));
}