async Task RunLoopAsync(CancellationToken cancellationToken) // throws Exception, ExceptionWithAction
{
while (!cancellationToken.IsCancellationRequested)
{
ILeaseManager leaseManager = this.host.LeaseManager;
Dictionary <string, Lease> allLeases = new Dictionary <string, Lease>();
// Inspect all leases.
// Acquire any expired leases.
// Renew any leases that currently belong to us.
IEnumerable <Task <Lease> > gettingAllLeases = leaseManager.GetAllLeases();
List <Lease> leasesOwnedByOthers = new List <Lease>();
int ourLeaseCount = 0;
foreach (Task <Lease> getLeaseTask in gettingAllLeases)
{
try
{
Lease possibleLease = await getLeaseTask.ConfigureAwait(false);
allLeases[possibleLease.PartitionId] = possibleLease;
if (await possibleLease.IsExpired().ConfigureAwait(false))
{
ProcessorEventSource.Log.PartitionPumpInfo(this.host.Id, possibleLease.PartitionId, "Trying to acquire lease.");
if (await leaseManager.AcquireLeaseAsync(possibleLease).ConfigureAwait(false))
{
ourLeaseCount++;
}
else
{
// Probably failed because another host stole it between get and acquire
leasesOwnedByOthers.Add(possibleLease);
}
}
else if (possibleLease.Owner == this.host.HostName)
{
ProcessorEventSource.Log.PartitionPumpInfo(this.host.Id, possibleLease.PartitionId, "Trying to renew lease.");
// Try to renew the lease. If successful then this lease belongs to us,
// if throws LeaseLostException then we don't own it anymore.
try
{
await leaseManager.RenewLeaseAsync(possibleLease).ConfigureAwait(false);
ourLeaseCount++;
}
catch (LeaseLostException)
{
// Probably failed because another host stole it between get and renew
leasesOwnedByOthers.Add(possibleLease);
}
}
else
{
leasesOwnedByOthers.Add(possibleLease);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostWarning(this.host.Id, "Failure during getting/acquiring/renewing lease, skipping", e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.CheckingLeases);
}
}
// Grab more leases if available and needed for load balancing
if (leasesOwnedByOthers.Count > 0)
{
Lease stealThisLease = WhichLeaseToSteal(leasesOwnedByOthers, ourLeaseCount);
if (stealThisLease != null)
{
try
{
ProcessorEventSource.Log.PartitionPumpStealLeaseStart(this.host.Id, stealThisLease.PartitionId);
if (await leaseManager.AcquireLeaseAsync(stealThisLease).ConfigureAwait(false))
{
// Succeeded in stealing lease
ProcessorEventSource.Log.PartitionPumpStealLeaseStop(this.host.Id, stealThisLease.PartitionId);
}
else
{
ProcessorEventSource.Log.EventProcessorHostWarning(this.host.Id,
"Failed to steal lease for partition " + stealThisLease.PartitionId, null);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.Id,
"Exception during stealing lease for partition " + stealThisLease.PartitionId, e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName,
stealThisLease.PartitionId, e, EventProcessorHostActionStrings.StealingLease);
}
}
}
// Update pump with new state of leases.
foreach (string partitionId in allLeases.Keys)
{
try
{
Lease updatedLease = allLeases[partitionId];
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.Id, $"Lease on partition {updatedLease.PartitionId} owned by {updatedLease.Owner}");
if (updatedLease.Owner == this.host.HostName)
{
await this.CheckAndAddPumpAsync(partitionId, updatedLease).ConfigureAwait(false);
}
else
{
await this.RemovePumpAsync(partitionId, CloseReason.LeaseLost).ConfigureAwait(false);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.Id, $"Exception during add/remove pump on partition {partitionId}", e.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, partitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
}
}
try
{
await Task.Delay(leaseManager.LeaseRenewInterval, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
// Bail on the async work if we are canceled.
}
}
}
async Task RunLoopAsync(CancellationToken cancellationToken) // throws Exception, ExceptionWithAction
{
var loopStopwatch = new Stopwatch();
while (!cancellationToken.IsCancellationRequested)
{
// Mark start time so we can use the duration taken to calculate renew interval.
loopStopwatch.Restart();
ILeaseManager leaseManager = this.host.LeaseManager;
Dictionary <string, Lease> allLeases = new Dictionary <string, Lease>();
// Inspect all leases.
// Acquire any expired leases.
// Renew any leases that currently belong to us.
IEnumerable <Task <Lease> > gettingAllLeases = leaseManager.GetAllLeases();
List <Lease> leasesOwnedByOthers = new List <Lease>();
var renewLeaseTasks = new List <Task>();
int ourLeaseCount = 0;
// First thing is first, renew owned leases.
foreach (Task <Lease> getLeaseTask in gettingAllLeases)
{
try
{
var lease = await getLeaseTask.ConfigureAwait(false);
allLeases[lease.PartitionId] = lease;
if (lease.Owner == this.host.HostName)
{
ourLeaseCount++;
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, lease.PartitionId, "Trying to renew lease.");
renewLeaseTasks.Add(leaseManager.RenewLeaseAsync(lease).ContinueWith(renewResult =>
{
if (renewResult.IsFaulted || !renewResult.Result)
{
// Might have failed due to intermittent error or lease-lost.
// Just log here, expired leases will be picked by same or another host anyway.
ProcessorEventSource.Log.PartitionPumpError(this.host.HostName, lease.PartitionId, "Failed to renew lease.", renewResult.Exception?.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, lease.PartitionId, renewResult.Exception, EventProcessorHostActionStrings.RenewingLease);
}
}));
}
else
{
leasesOwnedByOthers.Add(lease);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, "Failure during checking lease.", e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, "N/A", e, EventProcessorHostActionStrings.CheckingLeases);
}
}
// Wait until we are done with renewing our own leases here.
// In theory, this should never throw, error are logged and notified in the renew tasks.
await Task.WhenAll(renewLeaseTasks.ToArray()).ConfigureAwait(false);
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, "Lease renewal is finished.");
// Check any expired leases that we can grab here.
foreach (var possibleLease in allLeases.Values)
{
try
{
if (await possibleLease.IsExpired().ConfigureAwait(false))
{
bool isExpiredLeaseOwned = possibleLease.Owner == this.host.HostName;
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, possibleLease.PartitionId, "Trying to acquire lease.");
if (await leaseManager.AcquireLeaseAsync(possibleLease).ConfigureAwait(false))
{
ProcessorEventSource.Log.PartitionPumpInfo(this.host.HostName, possibleLease.PartitionId, "Acquired lease.");
// Don't double count if we have already counted this lease at the beginning of the loop.
if (!isExpiredLeaseOwned)
{
ourLeaseCount++;
}
}
}
}
catch (Exception e)
{
ProcessorEventSource.Log.PartitionPumpError(this.host.HostName, possibleLease.PartitionId, "Failure during acquiring lease", e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, possibleLease.PartitionId, e, EventProcessorHostActionStrings.CheckingLeases);
}
}
// Grab more leases if available and needed for load balancing
if (leasesOwnedByOthers.Count > 0)
{
Lease stealThisLease = WhichLeaseToSteal(leasesOwnedByOthers, ourLeaseCount);
if (stealThisLease != null)
{
try
{
ProcessorEventSource.Log.PartitionPumpStealLeaseStart(this.host.HostName, stealThisLease.PartitionId);
if (await leaseManager.AcquireLeaseAsync(stealThisLease).ConfigureAwait(false))
{
// Succeeded in stealing lease
ProcessorEventSource.Log.PartitionPumpStealLeaseStop(this.host.HostName, stealThisLease.PartitionId);
}
else
{
ProcessorEventSource.Log.EventProcessorHostWarning(this.host.HostName,
"Failed to steal lease for partition " + stealThisLease.PartitionId, null);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName,
"Exception during stealing lease for partition " + stealThisLease.PartitionId, e.ToString());
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName,
stealThisLease.PartitionId, e, EventProcessorHostActionStrings.StealingLease);
}
}
}
// Update pump with new state of leases.
foreach (string partitionId in allLeases.Keys)
{
try
{
Lease updatedLease = allLeases[partitionId];
ProcessorEventSource.Log.EventProcessorHostInfo(this.host.HostName, $"Lease on partition {updatedLease.PartitionId} owned by {updatedLease.Owner}");
if (updatedLease.Owner == this.host.HostName)
{
await this.CheckAndAddPumpAsync(partitionId, updatedLease).ConfigureAwait(false);
}
else
{
await this.RemovePumpAsync(partitionId, CloseReason.LeaseLost).ConfigureAwait(false);
}
}
catch (Exception e)
{
ProcessorEventSource.Log.EventProcessorHostError(this.host.HostName, $"Exception during add/remove pump on partition {partitionId}", e.Message);
this.host.EventProcessorOptions.NotifyOfException(this.host.HostName, partitionId, e, EventProcessorHostActionStrings.PartitionPumpManagement);
}
}
try
{
// Consider reducing the wait time with last lease-walkthrough's time taken.
var elapsedTime = loopStopwatch.Elapsed;
if (leaseManager.LeaseRenewInterval > elapsedTime)
{
await Task.Delay(leaseManager.LeaseRenewInterval.Subtract(elapsedTime), cancellationToken).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
// Bail on the async work if we are canceled.
}
}
}