async Task <T> RenewLeaseAsync(T lease) { T renewedLease = null; try { TraceLog.Informational(string.Format("Host '{0}' renewing lease for PartitionId '{1}' with lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); renewedLease = await this.leaseManager.RenewAsync(lease); } catch (LeaseLostException) { TraceLog.Informational(string.Format("Host '{0}' got LeaseLostException trying to renew lease for PartitionId '{1}' with lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); } catch (Exception ex) { TraceLog.Exception(ex); // Eat any exceptions during renew and keep going. // Consider the lease as renewed. Maybe lease store outage is causing the lease to not get renewed. renewedLease = lease; } finally { TraceLog.Informational(string.Format("Host '{0}' attempted to renew lease for PartitionId '{1}' and lease token '{2}' with result: '{3}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken, renewedLease != null)); } return(renewedLease); }
async Task RemoveLeaseAsync(T lease, bool hasOwnership, ChangeFeedObserverCloseReason closeReason = ChangeFeedObserverCloseReason.Unknown) { ChangeFeedObserverCloseReason reason = closeReason != ChangeFeedObserverCloseReason.Unknown ? closeReason : hasOwnership ? ChangeFeedObserverCloseReason.Shutdown : ChangeFeedObserverCloseReason.LeaseLost; if (lease != null && this.currentlyOwnedPartitions != null && this.currentlyOwnedPartitions.TryRemove(lease.PartitionId, out lease)) { TraceLog.Informational(string.Format("Host '{0}' successfully removed PartitionId '{1}' with lease token '{2}' from currently owned partitions.", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); try { if (hasOwnership) { this.keepRenewingDuringClose.TryAdd(lease.PartitionId, lease); } TraceLog.Informational(string.Format("Host '{0}' closing event processor for PartitionId '{1}' and lease token '{2}' with reason '{3}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken, reason)); // Notify the host that we lost partition so shutdown can be triggered on the host await this.partitionObserverManager.NotifyPartitionReleasedAsync(lease, reason); TraceLog.Informational(string.Format("Host '{0}' closed event processor for PartitionId '{1}' and lease token '{2}' with reason '{3}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken, reason)); } catch (Exception ex) { // Eat any exceptions during notification of observers TraceLog.Exception(ex); } finally { if (hasOwnership) { this.keepRenewingDuringClose.TryRemove(lease.PartitionId, out lease); } } if (hasOwnership) { try { await this.leaseManager.ReleaseAsync(lease); TraceLog.Informational(string.Format("Host '{0}' successfully released lease on PartitionId '{1}' with lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); } catch (LeaseLostException) { // We have already shutdown the processor so we can ignore any LeaseLost at this point TraceLog.Informational(string.Format("Host '{0}' failed to release lease for PartitionId '{1}' with lease token '{2}' due to conflict.", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); } catch (Exception ex) { TraceLog.Exception(ex); } } } }
async Task AddLeaseAsync(T lease) { if (this.currentlyOwnedPartitions.TryAdd(lease.PartitionId, lease)) { bool failedToInitialize = false; try { TraceLog.Informational(string.Format("Host '{0}' opening event processor for PartitionId '{1}' and lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); await this.partitionObserverManager.NotifyPartitionAcquiredAsync(lease); TraceLog.Informational(string.Format("Host '{0}' opened event processor for PartitionId '{1}' and lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); } catch (Exception ex) { TraceLog.Informational(string.Format("Host '{0}' failed to initialize processor for PartitionId '{1}' and lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); failedToInitialize = true; // Eat any exceptions during notification of observers TraceLog.Exception(ex); } // We need to release the lease if we fail to initialize the processor, so some other node can pick up the parition if (failedToInitialize) { await this.RemoveLeaseAsync(lease, true, ChangeFeedObserverCloseReason.ObserverError); } } else { // We already acquired lease for this partition but it looks like we previously owned this partition // and haven't completed the shutdown process for it yet. Release lease for possible others hosts to // pick it up. try { TraceLog.Warning(string.Format("Host '{0}' unable to add PartitionId '{1}' with lease token '{2}' to currently owned partitions.", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); await this.leaseManager.ReleaseAsync(lease); TraceLog.Informational(string.Format("Host '{0}' successfully released lease on PartitionId '{1}' with lease token '{2}'", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); } catch (LeaseLostException) { // We have already shutdown the processor so we can ignore any LeaseLost at this point TraceLog.Informational(string.Format("Host '{0}' failed to release lease for PartitionId '{1}' with lease token '{2}' due to conflict.", this.workerName, lease.PartitionId, lease.ConcurrencyToken)); } catch (Exception ex) { TraceLog.Exception(ex); } } }
async Task <T> TryAcquireLeaseAsync(T lease) { try { return(await this.leaseManager.AcquireAsync(lease, this.workerName)); } catch (LeaseLostException) { TraceLog.Informational(string.Format("Host '{0}' failed to acquire lease for PartitionId '{1}' due to conflict.", this.workerName, lease.PartitionId)); } catch (Exception ex) { // Eat any exceptions during acquiring lease. TraceLog.Exception(ex); } return(null); }
async Task <T> TryStealLeaseAsync(T lease) { try { return(await this.leaseManager.AcquireAsync(lease, this.workerName)); } catch (LeaseLostException) { // Concurrency issue in stealing the lease, someone else got it before us TraceLog.Informational(string.Format("Host '{0}' failed to steal lease for PartitionId '{1}' due to conflict.", this.workerName, lease.PartitionId)); } catch (Exception ex) { // Eat any exceptions during stealing TraceLog.Exception(ex); } return(null); }
public async Task <IDisposable> SubscribeAsync(IPartitionObserver <T> observer) { if (!this.observers.Contains(observer)) { this.observers.Add(observer); foreach (var lease in this.partitionManager.currentlyOwnedPartitions.Values) { try { await observer.OnPartitionAcquiredAsync(lease); } catch (Exception ex) { // Eat any exceptions during notification of observers TraceLog.Exception(ex); } } } return(new Unsubscriber(this.observers, observer)); }
async Task LeaseTakerAsync() { while (this.isStarted == 1) { try { TraceLog.Informational(string.Format("Host '{0}' starting to check for available leases.", this.workerName)); var availableLeases = await this.TakeLeasesAsync(); if (availableLeases.Count > 0) { TraceLog.Informational(string.Format("Host '{0}' adding {1} leases...", this.workerName, availableLeases.Count)); } var addLeaseTasks = new List <Task>(); foreach (var kvp in availableLeases) { addLeaseTasks.Add(this.AddLeaseAsync(kvp.Value)); } await Task.WhenAll(addLeaseTasks.ToArray()); } catch (Exception ex) { TraceLog.Exception(ex); } try { await Task.Delay(this.options.LeaseAcquireInterval, this.leaseTakerCancellationTokenSource.Token); } catch (OperationCanceledException) { TraceLog.Informational(string.Format("Host '{0}' AcquireLease task canceled.", this.workerName)); } } TraceLog.Informational(string.Format("Host '{0}' AcquireLease task completed.", this.workerName)); }
async Task LeaseRenewer() { while (this.isStarted == 1 || !this.shutdownComplete) { try { TraceLog.Informational(string.Format("Host '{0}' starting renewal of Leases.", this.workerName)); ConcurrentBag <T> renewedLeases = new ConcurrentBag <T>(); ConcurrentBag <T> failedToRenewLeases = new ConcurrentBag <T>(); List <Task> renewTasks = new List <Task>(); // Renew leases for all currently owned partitions in parallel foreach (T lease in this.currentlyOwnedPartitions.Values) { renewTasks.Add(this.RenewLeaseAsync(lease).ContinueWith(renewResult => { if (renewResult.Result != null) { renewedLeases.Add(renewResult.Result); } else { // Keep track of all failed attempts to renew so we can trigger shutdown for these partitions failedToRenewLeases.Add(lease); } })); } // Renew leases for all partitions currently in shutdown List <T> failedToRenewShutdownLeases = new List <T>(); foreach (T shutdownLeases in this.keepRenewingDuringClose.Values) { renewTasks.Add(this.RenewLeaseAsync(shutdownLeases).ContinueWith(renewResult => { if (renewResult.Result != null) { renewedLeases.Add(renewResult.Result); } else { // Keep track of all failed attempts to renew shutdown leases so we can remove them from further renew attempts failedToRenewShutdownLeases.Add(shutdownLeases); } })); } // Wait for all renews to complete await Task.WhenAll(renewTasks.ToArray()); // Update renewed leases. foreach (T lease in renewedLeases) { bool updateResult = this.currentlyOwnedPartitions.TryUpdate(lease.PartitionId, lease, lease); if (!updateResult) { TraceLog.Warning(string.Format("Host '{0}' Renewed lease {1} but failed to update it in the map (ignorable).", this.workerName, lease)); } } // Trigger shutdown of all partitions we failed to renew leases await failedToRenewLeases.ForEachAsync( async lease => await this.RemoveLeaseAsync(lease, false, ChangeFeedObserverCloseReason.LeaseLost), this.options.DegreeOfParallelism); // Now remove all failed renewals of shutdown leases from further renewals foreach (T failedToRenewShutdownLease in failedToRenewShutdownLeases) { T removedLease = null; this.keepRenewingDuringClose.TryRemove(failedToRenewShutdownLease.PartitionId, out removedLease); } await Task.Delay(this.options.LeaseRenewInterval, this.leaseRenewerCancellationTokenSource.Token); } catch (OperationCanceledException) { TraceLog.Informational(string.Format("Host '{0}' Renewer task canceled.", this.workerName)); } catch (Exception ex) { TraceLog.Exception(ex); } } this.currentlyOwnedPartitions.Clear(); this.keepRenewingDuringClose.Clear(); TraceLog.Informational(string.Format("Host '{0}' Renewer task completed.", this.workerName)); }