Lease WhichLeaseToSteal(IEnumerable <Lease> stealableLeases, int haveLeaseCount)
{
IDictionary <string, int> countsByOwner = CountLeasesByOwner(stealableLeases);
// Consider all leases might be already released where we won't have any entry in the return counts map.
if (countsByOwner.Count == 0)
{
return(null);
}
var biggestOwner = countsByOwner.OrderByDescending(o => o.Value).First();
Lease stealThisLease = null;
// If the number of leases is a multiple of the number of hosts, then the desired configuration is
// that all hosts own the name number of leases, and the difference between the "biggest" owner and
// any other is 0.
//
// If the number of leases is not a multiple of the number of hosts, then the most even configuration
// possible is for some hosts to have (leases/hosts) leases and others to have ((leases/hosts) + 1).
// For example, for 16 partitions distributed over five hosts, the distribution would be 4, 3, 3, 3, 3,
// or any of the possible reorderings.
//
// In either case, if the difference between this host and the biggest owner is 2 or more, then the
// system is not in the most evenly-distributed configuration, so steal one lease from the biggest.
// If there is a tie for biggest, we pick whichever appears first in the list because
// it doesn't really matter which "biggest" is trimmed down.
//
// Stealing one at a time prevents flapping because it reduces the difference between the biggest and
// this host by two at a time. If the starting difference is two or greater, then the difference cannot
// end up below 0. This host may become tied for biggest, but it cannot become larger than the host that
// it is stealing from.
if ((biggestOwner.Value - haveLeaseCount) >= 2)
{
stealThisLease = stealableLeases.First(l => l.Owner == biggestOwner.Key);
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, $"Proposed to steal lease for partition {stealThisLease.PartitionId} from {biggestOwner.Key}");
}
return(stealThisLease);
}
public Task <bool> UpdateLeaseAsync(Lease lease)
{
return(UpdateLeaseCoreAsync((AzureBlobLease)lease));
}
public Task <bool> RenewLeaseAsync(Lease lease)
{
return(RenewLeaseCoreAsync((AzureBlobLease)lease));
}
public Task <bool> AcquireLeaseAsync(Lease lease)
{
return(AcquireLeaseCoreAsync((AzureBlobLease)lease));
}
public EventHubPartitionPump(EventProcessorHost host, Lease lease)
: base(host, lease)
{
}
async Task RunLoopAsync(CancellationToken cancellationToken) // throws Exception, ExceptionWithAction
{
while (!cancellationToken.IsCancellationRequested)
{
ILeaseManager leaseManager = this.host.LeaseManager;
Dictionary <string, Lease> allLeases = new Dictionary <string, Lease>();
// Inspect all leases.
// Acquire any expired leases.
// Renew any leases that currently belong to us.
IEnumerable <Task <Lease> > gettingAllLeases = leaseManager.GetAllLeases();
List <Lease> leasesOwnedByOthers = new List <Lease>();
int ourLeaseCount = 0;
foreach (Task <Lease> getLeaseTask in gettingAllLeases)
{
try
{
Lease possibleLease = await getLeaseTask.ConfigureAwait(false);
allLeases[possibleLease.PartitionId] = possibleLease;
if (await possibleLease.IsExpired().ConfigureAwait(false))
{
ProcessorEventSource.Log.PartitionPumpInfo(this.host.Id, possibleLease.PartitionId, "Trying to acquire lease.");
if (await leaseManager.AcquireLeaseAsync(possibleLease).ConfigureAwait(false))
{
ourLeaseCount++;
}
else
{
// Probably failed because another host stole it between get and acquire
leasesOwnedByOthers.Add(possibleLease);
}
}
else if (possibleLease.Owner == this.host.HostName)
{
ProcessorEventSource.Log.PartitionPumpInfo(this.host.Id, possibleLease.PartitionId, "Trying to renew lease.");
// Try to renew the lease. If successful then this lease belongs to us,
// if throws LeaseLostException then we don't own it anymore.
try
{
await leaseManager.RenewLeaseAsync(possibleLease).ConfigureAwait(false);
ourLeaseCount++;
}
catch (LeaseLostException)
{
// Probably failed because another host stole it between get and renew
leasesOwnedByOthers.Add(possibleLease);
}
}
else
{
leasesOwnedByOthers.Add(possibleLease);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostWarning(this.host.Id, "Failure during getting/acquiring/renewing lease, skipping", e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.CheckingLeases);
}
}
// Grab more leases if available and needed for load balancing
if (leasesOwnedByOthers.Count > 0)
{
Lease stealThisLease = WhichLeaseToSteal(leasesOwnedByOthers, ourLeaseCount);
if (stealThisLease != null)
{
try
{
ProcessorEventSource.Log.PartitionPumpStealLeaseStart(this.host.Id, stealThisLease.PartitionId);
if (await leaseManager.AcquireLeaseAsync(stealThisLease).ConfigureAwait(false))
{
// Succeeded in stealing lease
ProcessorEventSource.Log.PartitionPumpStealLeaseStop(this.host.Id, stealThisLease.PartitionId);
}
else
{
ProcessorEventSource.Log.EventProcessorHostWarning(this.host.Id,
"Failed to steal lease for partition " + stealThisLease.PartitionId, null);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.Id,
"Exception during stealing lease for partition " + stealThisLease.PartitionId, e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName,
stealThisLease.PartitionId, e, EventProcessorHostActionStrings.StealingLease);
}
}
}
// Update pump with new state of leases.
foreach (string partitionId in allLeases.Keys)
{
try
{
Lease updatedLease = allLeases[partitionId];
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.Id, $"Lease on partition {updatedLease.PartitionId} owned by {updatedLease.Owner}");
if (updatedLease.Owner == this.host.HostName)
{
await this.CheckAndAddPumpAsync(partitionId, updatedLease).ConfigureAwait(false);
}
else
{
await this.RemovePumpAsync(partitionId, CloseReason.LeaseLost).ConfigureAwait(false);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.Id, $"Exception during add/remove pump on partition {partitionId}", e.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, partitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
}
}
try
{
await Task.Delay(leaseManager.LeaseRenewInterval, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
// Bail on the async work if we are canceled.
}
}
}
internal void SetLease(Lease newLease)
{
this.PartitionContext.Lease = newLease;
}
async Task RunLoopAsync(CancellationToken cancellationToken) // throws Exception, ExceptionWithAction
{
var loopStopwatch = new Stopwatch();
while (!cancellationToken.IsCancellationRequested)
{
// Mark start time so we can use the duration taken to calculate renew interval.
loopStopwatch.Restart();
ILeaseManager leaseManager = this.host.LeaseManager;
Dictionary <string, Lease> allLeases = new Dictionary <string, Lease>();
// Inspect all leases.
// Acquire any expired leases.
// Renew any leases that currently belong to us.
IEnumerable <Task <Lease> > gettingAllLeases = leaseManager.GetAllLeases();
List <Lease> leasesOwnedByOthers = new List <Lease>();
var renewLeaseTasks = new List <Task>();
int ourLeaseCount = 0;
// First thing is first, renew owned leases.
foreach (Task <Lease> getLeaseTask in gettingAllLeases)
{
try
{
var lease = await getLeaseTask.ConfigureAwait(false);
allLeases[lease.PartitionId] = lease;
if (lease.Owner == this.host.HostName)
{
ourLeaseCount++;
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, lease.PartitionId, "Trying to renew lease.");
renewLeaseTasks.Add(leaseManager.RenewLeaseAsync(lease).ContinueWith(renewResult =>
{
if (renewResult.IsFaulted || !renewResult.Result)
{
// Might have failed due to intermittent error or lease-lost.
// Just log here, expired leases will be picked by same or another host anyway.
ProcessorEventSource.Log.PartitionPumpError(this.host.HostName, lease.PartitionId, "Failed to renew lease.", renewResult.Exception?.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, lease.PartitionId, renewResult.Exception, EventProcessorHostActionStrings.RenewingLease);
}
}));
}
else
{
leasesOwnedByOthers.Add(lease);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, "Failure during checking lease.", e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.CheckingLeases);
}
}
// Wait until we are done with renewing our own leases here.
// In theory, this should never throw, error are logged and notified in the renew tasks.
await Task.WhenAll(renewLeaseTasks.ToArray()).ConfigureAwait(false);
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, "Lease renewal is finished.");
// Check any expired leases that we can grab here.
foreach (var possibleLease in allLeases.Values)
{
try
{
if (await possibleLease.IsExpired().ConfigureAwait(false))
{
bool isExpiredLeaseOwned = possibleLease.Owner == this.host.HostName;
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, possibleLease.PartitionId, "Trying to acquire lease.");
if (await leaseManager.AcquireLeaseAsync(possibleLease).ConfigureAwait(false))
{
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, possibleLease.PartitionId, "Acquired lease.");
// Don't double count if we have already counted this lease at the beginning of the loop.
if (!isExpiredLeaseOwned)
{
ourLeaseCount++;
}
}
}
}
catch (Exception e)
{
ProcessorEventSource.Log.PartitionPumpError(this.host.HostName, possibleLease.PartitionId, "Failure during acquiring lease", e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, possibleLease.PartitionId, e, EventProcessorHostActionStrings.CheckingLeases);
}
}
// Grab more leases if available and needed for load balancing
if (leasesOwnedByOthers.Count > 0)
{
Lease stealThisLease = WhichLeaseToSteal(leasesOwnedByOthers, ourLeaseCount);
if (stealThisLease != null)
{
try
{
ProcessorEventSource.Log.PartitionPumpStealLeaseStart(this.host.HostName, stealThisLease.PartitionId);
if (await leaseManager.AcquireLeaseAsync(stealThisLease).ConfigureAwait(false))
{
// Succeeded in stealing lease
ProcessorEventSource.Log.PartitionPumpStealLeaseStop(this.host.HostName, stealThisLease.PartitionId);
}
else
{
ProcessorEventSource.Log.EventProcessorHostWarning(this.host.HostName,
"Failed to steal lease for partition " + stealThisLease.PartitionId, null);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName,
"Exception during stealing lease for partition " + stealThisLease.PartitionId, e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName,
stealThisLease.PartitionId, e, EventProcessorHostActionStrings.StealingLease);
}
}
}
// Update pump with new state of leases.
foreach (string partitionId in allLeases.Keys)
{
try
{
Lease updatedLease = allLeases[partitionId];
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, $"Lease on partition {updatedLease.PartitionId} owned by {updatedLease.Owner}");
if (updatedLease.Owner == this.host.HostName)
{
await this.CheckAndAddPumpAsync(partitionId, updatedLease).ConfigureAwait(false);
}
else
{
await this.RemovePumpAsync(partitionId, CloseReason.LeaseLost).ConfigureAwait(false);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, $"Exception during add/remove pump on partition {partitionId}", e.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, partitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
}
}
try
{
// Consider reducing the wait time with last lease-walkthrough's time taken.
var elapsedTime = loopStopwatch.Elapsed;
if (leaseManager.LeaseRenewInterval > elapsedTime)
{
await Task.Delay(leaseManager.LeaseRenewInterval.Subtract(elapsedTime), cancellationToken).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
// Bail on the async work if we are canceled.
}
}
}
async Task RunLoopAsync(CancellationToken cancellationToken) // throws Exception, ExceptionWithAction
{
var loopStopwatch = new Stopwatch();
while (!cancellationToken.IsCancellationRequested)
{
// Mark start time so we can use the duration taken to calculate renew interval.
loopStopwatch.Restart();
ILeaseManager leaseManager = this.host.LeaseManager;
var allLeases = new ConcurrentDictionary <string, Lease>();
var leasesOwnedByOthers = new ConcurrentDictionary <string, Lease>();
// Inspect all leases.
// Acquire any expired leases.
// Renew any leases that currently belong to us.
IEnumerable <Lease> downloadedLeases;
var renewLeaseTasks = new List <Task>();
int ourLeaseCount = 0;
try
{
try
{
downloadedLeases = await leaseManager.GetAllLeasesAsync().ConfigureAwait(false);
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, "Exception during downloading leases", e.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.DownloadingLeases);
// Avoid tight spin if getallleases call keeps failing.
await Task.Delay(1000).ConfigureAwait(false);
continue;
}
// First things first, renew owned leases.
foreach (var lease in downloadedLeases)
{
var subjectLease = lease;
try
{
allLeases[subjectLease.PartitionId] = subjectLease;
if (subjectLease.Owner == this.host.HostName)
{
ourLeaseCount++;
// Get lease from partition since we need the token at this point.
if (!this.partitionPumps.TryGetValue(subjectLease.PartitionId, out var capturedPump))
{
continue;
}
var capturedLease = capturedPump.Lease;
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, capturedLease.PartitionId, "Trying to renew lease.");
renewLeaseTasks.Add(leaseManager.RenewLeaseAsync(capturedLease).ContinueWith(renewResult =>
{
if (renewResult.IsFaulted)
{
// Might have failed due to intermittent error or lease-lost.
// Just log here, expired leases will be picked by same or another host anyway.
ProcessorEventSource.Log.PartitionPumpError(
this.host.HostName,
capturedLease.PartitionId,
"Failed to renew lease.",
renewResult.Exception?.Message);
this.host.EventProcessorOptions.NotifyOfException(
this.host.HostName,
capturedLease.PartitionId,
renewResult.Exception,
EventProcessorHostActionStrings.RenewingLease);
// Nullify the owner on the lease in case this host lost it.
// This helps to remove pump earlier reducing duplicate receives.
if (renewResult.Exception?.GetBaseException() is LeaseLostException)
{
allLeases[capturedLease.PartitionId].Owner = null;
}
}
}, cancellationToken));
}
else if (!await subjectLease.IsExpired().ConfigureAwait(false))
{
leasesOwnedByOthers[subjectLease.PartitionId] = subjectLease;
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, "Failure during checking lease.", e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.CheckingLeases);
}
}
// Wait until we are done with renewing our own leases here.
// In theory, this should never throw, error are logged and notified in the renew tasks.
await Task.WhenAll(renewLeaseTasks).ConfigureAwait(false);
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, "Lease renewal is finished.");
// Check any expired leases that we can grab here.
var expiredLeaseTasks = new List <Task>();
foreach (var possibleLease in allLeases.Values)
{
var subjectLease = possibleLease;
if (await subjectLease.IsExpired().ConfigureAwait(false))
{
expiredLeaseTasks.Add(Task.Run(async() =>
{
try
{
// Get fresh content of lease subject to acquire.
var downloadedLease = await leaseManager.GetLeaseAsync(subjectLease.PartitionId).ConfigureAwait(false);
allLeases[subjectLease.PartitionId] = downloadedLease;
// Check expired once more here incase another host have already leased this since we populated the list.
if (await downloadedLease.IsExpired().ConfigureAwait(false))
{
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, downloadedLease.PartitionId, "Trying to acquire lease.");
if (await leaseManager.AcquireLeaseAsync(downloadedLease).ConfigureAwait(false))
{
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, downloadedLease.PartitionId, "Acquired lease.");
leasesOwnedByOthers.TryRemove(downloadedLease.PartitionId, out var removedLease);
Interlocked.Increment(ref ourLeaseCount);
}
else
{
// Acquisition failed. Make sure we don't leave the lease as owned.
allLeases[subjectLease.PartitionId].Owner = null;
ProcessorEventSource.Log.EventProcessorHostWarning(this.host.HostName,
"Failed to acquire lease for partition " + downloadedLease.PartitionId, null);
}
}
}
catch (Exception e)
{
ProcessorEventSource.Log.PartitionPumpError(this.host.HostName, subjectLease.PartitionId, "Failure during acquiring lease", e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, subjectLease.PartitionId, e, EventProcessorHostActionStrings.CheckingLeases);
// Acquisition failed. Make sure we don't leave the lease as owned.
allLeases[subjectLease.PartitionId].Owner = null;
}
}, cancellationToken));
}
}
await Task.WhenAll(expiredLeaseTasks).ConfigureAwait(false);
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, "Expired lease check is finished.");
// Grab more leases if available and needed for load balancing
if (leasesOwnedByOthers.Count > 0)
{
Lease stealThisLease = WhichLeaseToSteal(leasesOwnedByOthers.Values, ourLeaseCount);
// Don't attempt to steal the lease if current host has a pump for this partition id
// This is possible when current pump is in failed state due to lease moved to some other host.
if (stealThisLease != null && !this.partitionPumps.ContainsKey(stealThisLease.PartitionId))
{
try
{
// Get fresh content of lease subject to acquire.
var downloadedLease = await leaseManager.GetLeaseAsync(stealThisLease.PartitionId).ConfigureAwait(false);
allLeases[stealThisLease.PartitionId] = downloadedLease;
// Don't attempt to steal if lease is already expired.
// Expired leases are picked up by other hosts quickly.
// Don't attempt to steal if owner has changed from the calculation time to refresh time.
if (!await downloadedLease.IsExpired().ConfigureAwait(false) &&
downloadedLease.Owner == stealThisLease.Owner)
{
ProcessorEventSource.Log.PartitionPumpStealLeaseStart(this.host.HostName, downloadedLease.PartitionId);
if (await leaseManager.AcquireLeaseAsync(downloadedLease).ConfigureAwait(false))
{
// Succeeded in stealing lease
ProcessorEventSource.Log.PartitionPumpStealLeaseStop(this.host.HostName, downloadedLease.PartitionId);
ourLeaseCount++;
}
else
{
// Acquisition failed. Make sure we don't leave the lease as owned.
allLeases[stealThisLease.PartitionId].Owner = null;
ProcessorEventSource.Log.EventProcessorHostWarning(this.host.HostName,
"Failed to steal lease for partition " + downloadedLease.PartitionId, null);
}
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName,
"Exception during stealing lease for partition " + stealThisLease.PartitionId, e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName,
stealThisLease.PartitionId, e, EventProcessorHostActionStrings.StealingLease);
// Acquisition failed. Make sure we don't leave the lease as owned.
allLeases[stealThisLease.PartitionId].Owner = null;
}
}
}
// Update pump with new state of leases on owned partitions in parallel.
var createRemovePumpTasks = new List <Task>();
foreach (string partitionId in allLeases.Keys)
{
var subjectPartitionId = partitionId;
Lease updatedLease = allLeases[subjectPartitionId];
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, $"Lease on partition {updatedLease.PartitionId} owned by {updatedLease.Owner}");
if (updatedLease.Owner == this.host.HostName)
{
createRemovePumpTasks.Add(Task.Run(async() =>
{
try
{
await this.CheckAndAddPumpAsync(subjectPartitionId, updatedLease).ConfigureAwait(false);
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, $"Exception during add pump on partition {subjectPartitionId}", e.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, subjectPartitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
}
}, cancellationToken));
}
else if (this.partitionPumps.ContainsKey(partitionId))
{
createRemovePumpTasks.Add(Task.Run(async() =>
{
try
{
await this.TryRemovePumpAsync(subjectPartitionId, CloseReason.LeaseLost).ConfigureAwait(false);
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, $"Exception during remove pump on partition {subjectPartitionId}", e.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, subjectPartitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
}
}, cancellationToken));
}
}
await Task.WhenAll(createRemovePumpTasks).ConfigureAwait(false);
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, "Pump update is finished.");
}
catch (Exception e)
{
// TaskCancelledException is expected furing host unregister.
if (e is TaskCanceledException)
{
continue;
}
// Loop should not exit unless signalled via cancellation token. Log any failures and continue.
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, "Exception from partition manager main loop, continuing", e.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.PartitionPumpManagement);
}
finally
{
// Consider reducing the wait time with last lease-walkthrough's time taken.
var elapsedTime = loopStopwatch.Elapsed;
if (leaseManager.LeaseRenewInterval > elapsedTime)
{
await Task.Delay(leaseManager.LeaseRenewInterval.Subtract(elapsedTime), cancellationToken).ConfigureAwait(false);
}
}
}
}
