/// <summary>
/// Renew leases
/// </summary>
/// <returns>bool - false if we failed to renew all leases</returns>
private async Task <bool> RenewLeases()
{
bool allRenewed = true;
if (base.Cancellation.IsCancellationRequested)
{
return(false);
}
if (this.Logger.IsEnabled(LogLevel.Trace))
{
this.Logger.LogTrace("RenewLeases. QueueCount: {QueueCount}", this.myQueues.Count);
}
if (this.myQueues.Count <= 0)
{
return(allRenewed);
}
var results = await this.leaseProvider.Renew(this.options.LeaseCategory, this.myQueues.Select(queue => queue.AcquiredLease).ToArray());
//update myQueues list with successfully renewed leases
for (var i = 0; i < results.Count(); i++)
{
AcquireLeaseResult result = results[i];
switch (result.StatusCode)
{
case ResponseCode.OK:
{
this.myQueues[i].AcquiredLease = result.AcquiredLease;
break;
}
case ResponseCode.TransientFailure:
{
this.myQueues.RemoveAt(i);
allRenewed &= false;
this.Logger.LogWarning(result.FailureException, "Failed to renew lease {LeaseKey} due to transient error.", result.AcquiredLease.ResourceKey);
break;
}
// these can occure if lease has expired and/or someone else has taken it
case ResponseCode.InvalidToken:
case ResponseCode.LeaseNotAvailable:
{
this.myQueues.RemoveAt(i);
allRenewed &= false;
this.Logger.LogWarning(result.FailureException, "Failed to renew lease {LeaseKey} due to {Reason}.", result.AcquiredLease.ResourceKey, result.StatusCode);
break;
}
default:
{
this.myQueues.RemoveAt(i);
allRenewed &= false;
this.Logger.LogError(result.FailureException, "Unexpected response to renew of lease {LeaseKey}. StatusCode {StatusCode}.", result.AcquiredLease.ResourceKey, result.StatusCode);
break;
}
}
}
this.Logger.LogInformation("Renewed leases for {QueueCount} queues.", this.myQueues.Count);
return(allRenewed);
}
public async Task Provider_TryRenewLeaseWithWrongToken_Return_InvalidToken()
{
var leaseRequests = new List <LeaseRequest>()
{
new LeaseRequest(Guid.NewGuid().ToString(), TimeSpan.FromSeconds(15)), new LeaseRequest(Guid.NewGuid().ToString(), TimeSpan.FromSeconds(15))
};
//acquire
var results = await this.leaseProvider.Acquire(LeaseCategory, leaseRequests.ToArray());
var acquiredLeaseWithWrongToken = results.Select(result => new AcquiredLease(result.AcquiredLease.ResourceKey, result.AcquiredLease.Duration, Guid.NewGuid().ToString(), result.AcquiredLease.StartTimeUtc));
//renew with wrong token
var renewResults = await this.leaseProvider.Renew(LeaseCategory, acquiredLeaseWithWrongToken.ToArray());
for (int i = 0; i < renewResults.Count(); i++)
{
LeaseRequest request = leaseRequests[i];
AcquireLeaseResult result = renewResults[i];
Assert.Equal(ResponseCode.InvalidToken, result.StatusCode);
Assert.Equal(request.ResourceKey, result.AcquiredLease.ResourceKey);
}
}
private async Task AcquireLeasesToMeetExpectation(int expectedTotalLeaseCount, TimeSpan timeout)
{
if (base.Cancellation.IsCancellationRequested)
{
return;
}
if (this.Logger.IsEnabled(LogLevel.Trace))
{
this.Logger.LogTrace("AcquireLeasesToMeetExpectation. QueueCount: {QueueCount}, ExpectedTotalLeaseCount: {ExpectedTotalLeaseCount}", this.myQueues.Count, expectedTotalLeaseCount);
}
var leasesToAquire = expectedTotalLeaseCount - this.myQueues.Count;
if (leasesToAquire <= 0)
{
return;
}
// tracks how many remaining possible leases there are.
var possibleLeaseCount = this.queueSelector.Count - this.myQueues.Count;
if (this.Logger.IsEnabled(LogLevel.Debug))
{
this.Logger.LogDebug("Holding leased for {QueueCount} queues. Trying to acquire {AquireQueueCount} queues to reach {TargetQueueCount} of a possible {PossibleLeaseCount}", this.myQueues.Count, leasesToAquire, expectedTotalLeaseCount, possibleLeaseCount);
}
ValueStopwatch sw = ValueStopwatch.StartNew();
// try to acquire leases until we have no more to aquire or no more possible
while (!base.Cancellation.IsCancellationRequested && leasesToAquire > 0 && possibleLeaseCount > 0)
{
//select new queues to acquire
List <QueueId> expectedQueues = this.queueSelector.NextSelection(leasesToAquire, this.myQueues.Select(queue => queue.QueueId).ToList());
// build lease request from each queue
LeaseRequest[] leaseRequests = expectedQueues
.Select(queue => new LeaseRequest()
{
ResourceKey = queue.ToString(),
Duration = this.options.LeaseLength
})
.ToArray();
AcquireLeaseResult[] results = await this.leaseProvider.Acquire(this.options.LeaseCategory, leaseRequests);
//add successfully acquired queue to myQueues list
for (var i = 0; i < results.Length; i++)
{
AcquireLeaseResult result = results[i];
switch (result.StatusCode)
{
case ResponseCode.OK:
{
this.myQueues.Add(new AcquiredQueue(this.leaseOrder++, expectedQueues[i], result.AcquiredLease));
break;
}
case ResponseCode.TransientFailure:
{
this.Logger.LogWarning(result.FailureException, "Failed to acquire lease {LeaseKey} due to transient error.", result.AcquiredLease.ResourceKey);
break;
}
// this is expected much of the time
case ResponseCode.LeaseNotAvailable:
{
if (this.Logger.IsEnabled(LogLevel.Debug))
{
this.Logger.LogDebug(result.FailureException, "Failed to acquire lease {LeaseKey} due to {Reason}.", result.AcquiredLease.ResourceKey, result.StatusCode);
}
break;
}
// An acquire call should not return this code, so log as error
case ResponseCode.InvalidToken:
{
this.Logger.LogError(result.FailureException, "Failed to aquire acquire {LeaseKey} unexpected invalid token.", result.AcquiredLease.ResourceKey);
break;
}
default:
{
this.Logger.LogError(result.FailureException, "Unexpected response to acquire request of lease {LeaseKey}. StatusCode {StatusCode}.", result.AcquiredLease.ResourceKey, result.StatusCode);
break;
}
}
}
possibleLeaseCount -= expectedQueues.Count;
leasesToAquire = expectedTotalLeaseCount - this.myQueues.Count;
if (this.Logger.IsEnabled(LogLevel.Debug))
{
this.Logger.LogDebug("Holding leased for {QueueCount} queues. Trying to acquire {AquireQueueCount} queues to reach {TargetQueueCount} of a possible {PossibleLeaseCount} lease", this.myQueues.Count, leasesToAquire, expectedTotalLeaseCount, possibleLeaseCount);
}
if (sw.Elapsed > timeout)
{
// blown our alotted time, try again next period
break;
}
}
this.Logger.LogInformation("Holding leases for {QueueCount} of an expected {MinQueueCount} queues.", this.myQueues.Count, this.responsibility);
}
