private async Task <LeaseResponse> TryAcquireLeaseAsync(AcquireLeaseRequest request, CancellationToken token)
{
int delaySeconds = 2;
int triesLeft = 3;
while (triesLeft > 0)
{
triesLeft--;
var response = await this.leaseService.TryAcquireLeaseAsync(request);
if (response.Result != LeaseResult.TransientError)
{
return(response);
}
else if (triesLeft > 0)
{
await WaitFor(TimeSpan.FromSeconds(delaySeconds), token);
}
else
{
return(response);
}
delaySeconds = delaySeconds * 2;
}
// this should never happen
return(new LeaseResponse()
{
Result = LeaseResult.Error
});
}
private Task StartLeadershipTask(CancellationToken token,
BlockingCollection <ClientEvent> clientEvents)
{
return(Task.Run(async() =>
{
try
{
while (!token.IsCancellationRequested)
{
var request = new AcquireLeaseRequest()
{
ClientId = this.clientId,
ResourceGroup = this.resourceGroup
};
var response = await TryAcquireLeaseAsync(request, token);
if (response.Result == LeaseResult.Granted) // is now the Coordinator
{
await ExecuteLeaseRenewals(token, clientEvents, response.Lease);
}
else if (response.Result == LeaseResult.Denied) // is a Follower
{
PostFollowerEvent(response.Lease.ExpiryPeriod, clientEvents);
await WaitFor(GetInterval(response.Lease.HeartbeatPeriod), token);
}
else if (response.Result == LeaseResult.NoLease)
{
throw new RebalanserException($"The resource group {this.resourceGroup} does not exist.");
}
else if (response.IsErrorResponse())
{
throw new RebalanserException("An non-recoverable error occurred.", response.Exception);
}
else
{
throw new RebalanserException("A non-supported lease result was received"); // should never happen, just in case I screw up in the future
}
}
}
finally
{
clientEvents.CompleteAdding();
}
}));
}
public override Task <AcquireLeaseResponse> AcquireLease(AcquireLeaseRequest request, ServerCallContext context)
{
AcquireLeaseResponse.Types.Status status;
Lease lease = null;
SpotRobot.SpotInstance.Leases.TryGetValue(request.Resource, out var leaseTuple);
if (leaseTuple == null)
{
lease = new Lease
{
Resource = request.Resource,
Sequence = { 1 }
};
SpotRobot.SpotInstance.Leases.Add(request.Resource, new Tuple <string, Lease>(request.Header.ClientName, lease));
Console.WriteLine($"AcquireLease - registering new lease for resource \"{request.Resource}\" and client \"{request.Header.ClientName}\"");
status = AcquireLeaseResponse.Types.Status.Ok;
}
else if (leaseTuple.Item1 == request.Header.ClientName)
{
Console.WriteLine($"AcquireLease - lease already exists for resource \"{request.Resource}\" and client \"{request.Header.ClientName}\"");
status = AcquireLeaseResponse.Types.Status.Ok;
}
else
{
Console.WriteLine($"AcquireLease - lease already exists for resource \"{request.Resource}\" and but no for client \"{request.Header.ClientName}\"");
status = AcquireLeaseResponse.Types.Status.ResourceAlreadyClaimed;
}
return(Task.FromResult(new AcquireLeaseResponse
{
Header = HeaderBuilder.Build(request.Header, new CommonError {
Code = CommonError.Types.Code.Ok
}),
Status = status,
Lease = lease,
LeaseOwner = new LeaseOwner
{
ClientName = request.Header.ClientName,
}
}));
}
