예제 #1
0
        Lease WhichLeaseToSteal(IEnumerable <Lease> stealableLeases, int haveLeaseCount)
        {
            IDictionary <string, int> countsByOwner = CountLeasesByOwner(stealableLeases);

            // Consider all leases might be already released where we won't have any entry in the return counts map.
            if (countsByOwner.Count == 0)
            {
                return(null);
            }

            var   biggestOwner   = countsByOwner.OrderByDescending(o => o.Value).First();
            Lease stealThisLease = null;

            // If the number of leases is a multiple of the number of hosts, then the desired configuration is
            // that all hosts own the name number of leases, and the difference between the "biggest" owner and
            // any other is 0.
            //
            // If the number of leases is not a multiple of the number of hosts, then the most even configuration
            // possible is for some hosts to have (leases/hosts) leases and others to have ((leases/hosts) + 1).
            // For example, for 16 partitions distributed over five hosts, the distribution would be 4, 3, 3, 3, 3,
            // or any of the possible reorderings.
            //
            // In either case, if the difference between this host and the biggest owner is 2 or more, then the
            // system is not in the most evenly-distributed configuration, so steal one lease from the biggest.
            // If there is a tie for biggest, we pick whichever appears first in the list because
            // it doesn't really matter which "biggest" is trimmed down.
            //
            // Stealing one at a time prevents flapping because it reduces the difference between the biggest and
            // this host by two at a time. If the starting difference is two or greater, then the difference cannot
            // end up below 0. This host may become tied for biggest, but it cannot become larger than the host that
            // it is stealing from.

            if ((biggestOwner.Value - haveLeaseCount) >= 2)
            {
                stealThisLease = stealableLeases.First(l => l.Owner == biggestOwner.Key);
                ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, $"Proposed to steal lease for partition {stealThisLease.PartitionId} from {biggestOwner.Key}");
            }

            return(stealThisLease);
        }
 public Task <bool> UpdateLeaseAsync(Lease lease)
 {
     return(UpdateLeaseCoreAsync((AzureBlobLease)lease));
 }
 public Task <bool> RenewLeaseAsync(Lease lease)
 {
     return(RenewLeaseCoreAsync((AzureBlobLease)lease));
 }
 public Task <bool> AcquireLeaseAsync(Lease lease)
 {
     return(AcquireLeaseCoreAsync((AzureBlobLease)lease));
 }
 public EventHubPartitionPump(EventProcessorHost host, Lease lease)
     : base(host, lease)
 {
 }
예제 #6
0
        async Task RunLoopAsync(CancellationToken cancellationToken) // throws Exception, ExceptionWithAction
        {
            while (!cancellationToken.IsCancellationRequested)
            {
                ILeaseManager leaseManager           = this.host.LeaseManager;
                Dictionary <string, Lease> allLeases = new Dictionary <string, Lease>();

                // Inspect all leases.
                // Acquire any expired leases.
                // Renew any leases that currently belong to us.
                IEnumerable <Task <Lease> > gettingAllLeases = leaseManager.GetAllLeases();
                List <Lease> leasesOwnedByOthers             = new List <Lease>();
                int          ourLeaseCount = 0;
                foreach (Task <Lease> getLeaseTask in gettingAllLeases)
                {
                    try
                    {
                        Lease possibleLease = await getLeaseTask.ConfigureAwait(false);

                        allLeases[possibleLease.PartitionId] = possibleLease;
                        if (await possibleLease.IsExpired().ConfigureAwait(false))
                        {
                            ProcessorEventSource.Log.PartitionPumpInfo(this.host.Id, possibleLease.PartitionId, "Trying to acquire lease.");
                            if (await leaseManager.AcquireLeaseAsync(possibleLease).ConfigureAwait(false))
                            {
                                ourLeaseCount++;
                            }
                            else
                            {
                                // Probably failed because another host stole it between get and acquire
                                leasesOwnedByOthers.Add(possibleLease);
                            }
                        }
                        else if (possibleLease.Owner == this.host.HostName)
                        {
                            ProcessorEventSource.Log.PartitionPumpInfo(this.host.Id, possibleLease.PartitionId, "Trying to renew lease.");

                            // Try to renew the lease. If successful then this lease belongs to us,
                            // if throws LeaseLostException then we don't own it anymore.
                            try
                            {
                                await leaseManager.RenewLeaseAsync(possibleLease).ConfigureAwait(false);

                                ourLeaseCount++;
                            }
                            catch (LeaseLostException)
                            {
                                // Probably failed because another host stole it between get and renew
                                leasesOwnedByOthers.Add(possibleLease);
                            }
                        }
                        else
                        {
                            leasesOwnedByOthers.Add(possibleLease);
                        }
                    }
                    catch (Exception e)
                    {
                        ProcessorEventSource.Log.EventProcessorHostWarning(this.host.Id, "Failure during getting/acquiring/renewing lease, skipping", e.ToString());
                        this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.CheckingLeases);
                    }
                }

                // Grab more leases if available and needed for load balancing
                if (leasesOwnedByOthers.Count > 0)
                {
                    Lease stealThisLease = WhichLeaseToSteal(leasesOwnedByOthers, ourLeaseCount);
                    if (stealThisLease != null)
                    {
                        try
                        {
                            ProcessorEventSource.Log.PartitionPumpStealLeaseStart(this.host.Id, stealThisLease.PartitionId);
                            if (await leaseManager.AcquireLeaseAsync(stealThisLease).ConfigureAwait(false))
                            {
                                // Succeeded in stealing lease
                                ProcessorEventSource.Log.PartitionPumpStealLeaseStop(this.host.Id, stealThisLease.PartitionId);
                            }
                            else
                            {
                                ProcessorEventSource.Log.EventProcessorHostWarning(this.host.Id,
                                                                                   "Failed to steal lease for partition " + stealThisLease.PartitionId, null);
                            }
                        }
                        catch (Exception e)
                        {
                            ProcessorEventSource.Log.EventProcessorHostError(this.host.Id,
                                                                             "Exception during stealing lease for partition " + stealThisLease.PartitionId, e.ToString());
                            this.host.EventProcessorOptions.NotifyOfException(this.host.HostName,
                                                                              stealThisLease.PartitionId, e, EventProcessorHostActionStrings.StealingLease);
                        }
                    }
                }

                // Update pump with new state of leases.
                foreach (string partitionId in allLeases.Keys)
                {
                    try
                    {
                        Lease updatedLease = allLeases[partitionId];
                        ProcessorEventSource.Log.EventProcessorHostInfo(this.host.Id, $"Lease on partition {updatedLease.PartitionId} owned by {updatedLease.Owner}");
                        if (updatedLease.Owner == this.host.HostName)
                        {
                            await this.CheckAndAddPumpAsync(partitionId, updatedLease).ConfigureAwait(false);
                        }
                        else
                        {
                            await this.RemovePumpAsync(partitionId, CloseReason.LeaseLost).ConfigureAwait(false);
                        }
                    }
                    catch (Exception e)
                    {
                        ProcessorEventSource.Log.EventProcessorHostError(this.host.Id, $"Exception during add/remove pump on partition {partitionId}", e.Message);
                        this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, partitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
                    }
                }

                try
                {
                    await Task.Delay(leaseManager.LeaseRenewInterval, cancellationToken).ConfigureAwait(false);
                }
                catch (TaskCanceledException)
                {
                    // Bail on the async work if we are canceled.
                }
            }
        }
예제 #7
0
 internal void SetLease(Lease newLease)
 {
     this.PartitionContext.Lease = newLease;
 }
예제 #8
0
        async Task RunLoopAsync(CancellationToken cancellationToken) // throws Exception, ExceptionWithAction
        {
            var loopStopwatch = new Stopwatch();

            while (!cancellationToken.IsCancellationRequested)
            {
                // Mark start time so we can use the duration taken to calculate renew interval.
                loopStopwatch.Restart();

                ILeaseManager leaseManager           = this.host.LeaseManager;
                Dictionary <string, Lease> allLeases = new Dictionary <string, Lease>();

                // Inspect all leases.
                // Acquire any expired leases.
                // Renew any leases that currently belong to us.
                IEnumerable <Task <Lease> > gettingAllLeases = leaseManager.GetAllLeases();
                List <Lease> leasesOwnedByOthers             = new List <Lease>();
                var          renewLeaseTasks = new List <Task>();
                int          ourLeaseCount   = 0;

                // First thing is first, renew owned leases.
                foreach (Task <Lease> getLeaseTask in gettingAllLeases)
                {
                    try
                    {
                        var lease = await getLeaseTask.ConfigureAwait(false);

                        allLeases[lease.PartitionId] = lease;
                        if (lease.Owner == this.host.HostName)
                        {
                            ourLeaseCount++;
                            ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, lease.PartitionId, "Trying to renew lease.");
                            renewLeaseTasks.Add(leaseManager.RenewLeaseAsync(lease).ContinueWith(renewResult =>
                            {
                                if (renewResult.IsFaulted || !renewResult.Result)
                                {
                                    // Might have failed due to intermittent error or lease-lost.
                                    // Just log here, expired leases will be picked by same or another host anyway.
                                    ProcessorEventSource.Log.PartitionPumpError(this.host.HostName, lease.PartitionId, "Failed to renew lease.", renewResult.Exception?.Message);
                                    this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, lease.PartitionId, renewResult.Exception, EventProcessorHostActionStrings.RenewingLease);
                                }
                            }));
                        }
                        else
                        {
                            leasesOwnedByOthers.Add(lease);
                        }
                    }
                    catch (Exception e)
                    {
                        ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, "Failure during checking lease.", e.ToString());
                        this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.CheckingLeases);
                    }
                }

                // Wait until we are done with renewing our own leases here.
                // In theory, this should never throw, error are logged and notified in the renew tasks.
                await Task.WhenAll(renewLeaseTasks.ToArray()).ConfigureAwait(false);

                ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, "Lease renewal is finished.");

                // Check any expired leases that we can grab here.
                foreach (var possibleLease in allLeases.Values)
                {
                    try
                    {
                        if (await possibleLease.IsExpired().ConfigureAwait(false))
                        {
                            bool isExpiredLeaseOwned = possibleLease.Owner == this.host.HostName;
                            ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, possibleLease.PartitionId, "Trying to acquire lease.");
                            if (await leaseManager.AcquireLeaseAsync(possibleLease).ConfigureAwait(false))
                            {
                                ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, possibleLease.PartitionId, "Acquired lease.");

                                // Don't double count if we have already counted this lease at the beginning of the loop.
                                if (!isExpiredLeaseOwned)
                                {
                                    ourLeaseCount++;
                                }
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        ProcessorEventSource.Log.PartitionPumpError(this.host.HostName, possibleLease.PartitionId, "Failure during acquiring lease", e.ToString());
                        this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, possibleLease.PartitionId, e, EventProcessorHostActionStrings.CheckingLeases);
                    }
                }

                // Grab more leases if available and needed for load balancing
                if (leasesOwnedByOthers.Count > 0)
                {
                    Lease stealThisLease = WhichLeaseToSteal(leasesOwnedByOthers, ourLeaseCount);
                    if (stealThisLease != null)
                    {
                        try
                        {
                            ProcessorEventSource.Log.PartitionPumpStealLeaseStart(this.host.HostName, stealThisLease.PartitionId);
                            if (await leaseManager.AcquireLeaseAsync(stealThisLease).ConfigureAwait(false))
                            {
                                // Succeeded in stealing lease
                                ProcessorEventSource.Log.PartitionPumpStealLeaseStop(this.host.HostName, stealThisLease.PartitionId);
                            }
                            else
                            {
                                ProcessorEventSource.Log.EventProcessorHostWarning(this.host.HostName,
                                                                                   "Failed to steal lease for partition " + stealThisLease.PartitionId, null);
                            }
                        }
                        catch (Exception e)
                        {
                            ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName,
                                                                             "Exception during stealing lease for partition " + stealThisLease.PartitionId, e.ToString());
                            this.host.EventProcessorOptions.NotifyOfException(this.host.HostName,
                                                                              stealThisLease.PartitionId, e, EventProcessorHostActionStrings.StealingLease);
                        }
                    }
                }

                // Update pump with new state of leases.
                foreach (string partitionId in allLeases.Keys)
                {
                    try
                    {
                        Lease updatedLease = allLeases[partitionId];
                        ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, $"Lease on partition {updatedLease.PartitionId} owned by {updatedLease.Owner}");
                        if (updatedLease.Owner == this.host.HostName)
                        {
                            await this.CheckAndAddPumpAsync(partitionId, updatedLease).ConfigureAwait(false);
                        }
                        else
                        {
                            await this.RemovePumpAsync(partitionId, CloseReason.LeaseLost).ConfigureAwait(false);
                        }
                    }
                    catch (Exception e)
                    {
                        ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, $"Exception during add/remove pump on partition {partitionId}", e.Message);
                        this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, partitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
                    }
                }

                try
                {
                    // Consider reducing the wait time with last lease-walkthrough's time taken.
                    var elapsedTime = loopStopwatch.Elapsed;
                    if (leaseManager.LeaseRenewInterval > elapsedTime)
                    {
                        await Task.Delay(leaseManager.LeaseRenewInterval.Subtract(elapsedTime), cancellationToken).ConfigureAwait(false);
                    }
                }
                catch (TaskCanceledException)
                {
                    // Bail on the async work if we are canceled.
                }
            }
        }
예제 #9
0
        async Task RunLoopAsync(CancellationToken cancellationToken) // throws Exception, ExceptionWithAction
        {
            var loopStopwatch = new Stopwatch();

            while (!cancellationToken.IsCancellationRequested)
            {
                // Mark start time so we can use the duration taken to calculate renew interval.
                loopStopwatch.Restart();

                ILeaseManager leaseManager        = this.host.LeaseManager;
                var           allLeases           = new ConcurrentDictionary <string, Lease>();
                var           leasesOwnedByOthers = new ConcurrentDictionary <string, Lease>();

                // Inspect all leases.
                // Acquire any expired leases.
                // Renew any leases that currently belong to us.
                IEnumerable <Lease> downloadedLeases;
                var renewLeaseTasks = new List <Task>();
                int ourLeaseCount   = 0;

                try
                {
                    try
                    {
                        downloadedLeases = await leaseManager.GetAllLeasesAsync().ConfigureAwait(false);
                    }
                    catch (Exception e)
                    {
                        ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, "Exception during downloading leases", e.Message);
                        this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.DownloadingLeases);

                        // Avoid tight spin if getallleases call keeps failing.
                        await Task.Delay(1000).ConfigureAwait(false);

                        continue;
                    }

                    // First things first, renew owned leases.
                    foreach (var lease in downloadedLeases)
                    {
                        var subjectLease = lease;

                        try
                        {
                            allLeases[subjectLease.PartitionId] = subjectLease;
                            if (subjectLease.Owner == this.host.HostName)
                            {
                                ourLeaseCount++;

                                // Get lease from partition since we need the token at this point.
                                if (!this.partitionPumps.TryGetValue(subjectLease.PartitionId, out var capturedPump))
                                {
                                    continue;
                                }

                                var capturedLease = capturedPump.Lease;

                                ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, capturedLease.PartitionId, "Trying to renew lease.");
                                renewLeaseTasks.Add(leaseManager.RenewLeaseAsync(capturedLease).ContinueWith(renewResult =>
                                {
                                    if (renewResult.IsFaulted)
                                    {
                                        // Might have failed due to intermittent error or lease-lost.
                                        // Just log here, expired leases will be picked by same or another host anyway.
                                        ProcessorEventSource.Log.PartitionPumpError(
                                            this.host.HostName,
                                            capturedLease.PartitionId,
                                            "Failed to renew lease.",
                                            renewResult.Exception?.Message);

                                        this.host.EventProcessorOptions.NotifyOfException(
                                            this.host.HostName,
                                            capturedLease.PartitionId,
                                            renewResult.Exception,
                                            EventProcessorHostActionStrings.RenewingLease);

                                        // Nullify the owner on the lease in case this host lost it.
                                        // This helps to remove pump earlier reducing duplicate receives.
                                        if (renewResult.Exception?.GetBaseException() is LeaseLostException)
                                        {
                                            allLeases[capturedLease.PartitionId].Owner = null;
                                        }
                                    }
                                }, cancellationToken));
                            }
                            else if (!await subjectLease.IsExpired().ConfigureAwait(false))
                            {
                                leasesOwnedByOthers[subjectLease.PartitionId] = subjectLease;
                            }
                        }
                        catch (Exception e)
                        {
                            ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, "Failure during checking lease.", e.ToString());
                            this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.CheckingLeases);
                        }
                    }

                    // Wait until we are done with renewing our own leases here.
                    // In theory, this should never throw, error are logged and notified in the renew tasks.
                    await Task.WhenAll(renewLeaseTasks).ConfigureAwait(false);

                    ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, "Lease renewal is finished.");

                    // Check any expired leases that we can grab here.
                    var expiredLeaseTasks = new List <Task>();
                    foreach (var possibleLease in allLeases.Values)
                    {
                        var subjectLease = possibleLease;

                        if (await subjectLease.IsExpired().ConfigureAwait(false))
                        {
                            expiredLeaseTasks.Add(Task.Run(async() =>
                            {
                                try
                                {
                                    // Get fresh content of lease subject to acquire.
                                    var downloadedLease = await leaseManager.GetLeaseAsync(subjectLease.PartitionId).ConfigureAwait(false);
                                    allLeases[subjectLease.PartitionId] = downloadedLease;

                                    // Check expired once more here incase another host have already leased this since we populated the list.
                                    if (await downloadedLease.IsExpired().ConfigureAwait(false))
                                    {
                                        ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, downloadedLease.PartitionId, "Trying to acquire lease.");
                                        if (await leaseManager.AcquireLeaseAsync(downloadedLease).ConfigureAwait(false))
                                        {
                                            ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, downloadedLease.PartitionId, "Acquired lease.");
                                            leasesOwnedByOthers.TryRemove(downloadedLease.PartitionId, out var removedLease);
                                            Interlocked.Increment(ref ourLeaseCount);
                                        }
                                        else
                                        {
                                            // Acquisition failed. Make sure we don't leave the lease as owned.
                                            allLeases[subjectLease.PartitionId].Owner = null;

                                            ProcessorEventSource.Log.EventProcessorHostWarning(this.host.HostName,
                                                                                               "Failed to acquire lease for partition " + downloadedLease.PartitionId, null);
                                        }
                                    }
                                }
                                catch (Exception e)
                                {
                                    ProcessorEventSource.Log.PartitionPumpError(this.host.HostName, subjectLease.PartitionId, "Failure during acquiring lease", e.ToString());
                                    this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, subjectLease.PartitionId, e, EventProcessorHostActionStrings.CheckingLeases);

                                    // Acquisition failed. Make sure we don't leave the lease as owned.
                                    allLeases[subjectLease.PartitionId].Owner = null;
                                }
                            }, cancellationToken));
                        }
                    }

                    await Task.WhenAll(expiredLeaseTasks).ConfigureAwait(false);

                    ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, "Expired lease check is finished.");

                    // Grab more leases if available and needed for load balancing
                    if (leasesOwnedByOthers.Count > 0)
                    {
                        Lease stealThisLease = WhichLeaseToSteal(leasesOwnedByOthers.Values, ourLeaseCount);

                        // Don't attempt to steal the lease if current host has a pump for this partition id
                        // This is possible when current pump is in failed state due to lease moved to some other host.
                        if (stealThisLease != null && !this.partitionPumps.ContainsKey(stealThisLease.PartitionId))
                        {
                            try
                            {
                                // Get fresh content of lease subject to acquire.
                                var downloadedLease = await leaseManager.GetLeaseAsync(stealThisLease.PartitionId).ConfigureAwait(false);

                                allLeases[stealThisLease.PartitionId] = downloadedLease;

                                // Don't attempt to steal if lease is already expired.
                                // Expired leases are picked up by other hosts quickly.
                                // Don't attempt to steal if owner has changed from the calculation time to refresh time.
                                if (!await downloadedLease.IsExpired().ConfigureAwait(false) &&
                                    downloadedLease.Owner == stealThisLease.Owner)
                                {
                                    ProcessorEventSource.Log.PartitionPumpStealLeaseStart(this.host.HostName, downloadedLease.PartitionId);
                                    if (await leaseManager.AcquireLeaseAsync(downloadedLease).ConfigureAwait(false))
                                    {
                                        // Succeeded in stealing lease
                                        ProcessorEventSource.Log.PartitionPumpStealLeaseStop(this.host.HostName, downloadedLease.PartitionId);
                                        ourLeaseCount++;
                                    }
                                    else
                                    {
                                        // Acquisition failed. Make sure we don't leave the lease as owned.
                                        allLeases[stealThisLease.PartitionId].Owner = null;

                                        ProcessorEventSource.Log.EventProcessorHostWarning(this.host.HostName,
                                                                                           "Failed to steal lease for partition " + downloadedLease.PartitionId, null);
                                    }
                                }
                            }
                            catch (Exception e)
                            {
                                ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName,
                                                                                 "Exception during stealing lease for partition " + stealThisLease.PartitionId, e.ToString());
                                this.host.EventProcessorOptions.NotifyOfException(this.host.HostName,
                                                                                  stealThisLease.PartitionId, e, EventProcessorHostActionStrings.StealingLease);

                                // Acquisition failed. Make sure we don't leave the lease as owned.
                                allLeases[stealThisLease.PartitionId].Owner = null;
                            }
                        }
                    }

                    // Update pump with new state of leases on owned partitions in parallel.
                    var createRemovePumpTasks = new List <Task>();
                    foreach (string partitionId in allLeases.Keys)
                    {
                        var subjectPartitionId = partitionId;

                        Lease updatedLease = allLeases[subjectPartitionId];
                        ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, $"Lease on partition {updatedLease.PartitionId} owned by {updatedLease.Owner}");

                        if (updatedLease.Owner == this.host.HostName)
                        {
                            createRemovePumpTasks.Add(Task.Run(async() =>
                            {
                                try
                                {
                                    await this.CheckAndAddPumpAsync(subjectPartitionId, updatedLease).ConfigureAwait(false);
                                }
                                catch (Exception e)
                                {
                                    ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, $"Exception during add pump on partition {subjectPartitionId}", e.Message);
                                    this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, subjectPartitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
                                }
                            }, cancellationToken));
                        }
                        else if (this.partitionPumps.ContainsKey(partitionId))
                        {
                            createRemovePumpTasks.Add(Task.Run(async() =>
                            {
                                try
                                {
                                    await this.TryRemovePumpAsync(subjectPartitionId, CloseReason.LeaseLost).ConfigureAwait(false);
                                }
                                catch (Exception e)
                                {
                                    ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, $"Exception during remove pump on partition {subjectPartitionId}", e.Message);
                                    this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, subjectPartitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
                                }
                            }, cancellationToken));
                        }
                    }

                    await Task.WhenAll(createRemovePumpTasks).ConfigureAwait(false);

                    ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, "Pump update is finished.");
                }
                catch (Exception e)
                {
                    // TaskCancelledException is expected furing host unregister.
                    if (e is TaskCanceledException)
                    {
                        continue;
                    }

                    // Loop should not exit unless signalled via cancellation token. Log any failures and continue.
                    ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, "Exception from partition manager main loop, continuing", e.Message);
                    this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.PartitionPumpManagement);
                }
                finally
                {
                    // Consider reducing the wait time with last lease-walkthrough's time taken.
                    var elapsedTime = loopStopwatch.Elapsed;
                    if (leaseManager.LeaseRenewInterval > elapsedTime)
                    {
                        await Task.Delay(leaseManager.LeaseRenewInterval.Subtract(elapsedTime), cancellationToken).ConfigureAwait(false);
                    }
                }
            }
        }