private async Task <LeaseResponse> TryAcquireLeaseAsync(AcquireLeaseRequest request, CancellationToken token) { int delaySeconds = 2; int triesLeft = 3; while (triesLeft > 0) { triesLeft--; var response = await this.leaseService.TryAcquireLeaseAsync(request); if (response.Result != LeaseResult.TransientError) { return(response); } else if (triesLeft > 0) { await WaitFor(TimeSpan.FromSeconds(delaySeconds), token); } else { return(response); } delaySeconds = delaySeconds * 2; } // this should never happen return(new LeaseResponse() { Result = LeaseResult.Error }); }
private Task StartLeadershipTask(CancellationToken token, BlockingCollection <ClientEvent> clientEvents) { return(Task.Run(async() => { try { while (!token.IsCancellationRequested) { var request = new AcquireLeaseRequest() { ClientId = this.clientId, ResourceGroup = this.resourceGroup }; var response = await TryAcquireLeaseAsync(request, token); if (response.Result == LeaseResult.Granted) // is now the Coordinator { await ExecuteLeaseRenewals(token, clientEvents, response.Lease); } else if (response.Result == LeaseResult.Denied) // is a Follower { PostFollowerEvent(response.Lease.ExpiryPeriod, clientEvents); await WaitFor(GetInterval(response.Lease.HeartbeatPeriod), token); } else if (response.Result == LeaseResult.NoLease) { throw new RebalanserException($"The resource group {this.resourceGroup} does not exist."); } else if (response.IsErrorResponse()) { throw new RebalanserException("An non-recoverable error occurred.", response.Exception); } else { throw new RebalanserException("A non-supported lease result was received"); // should never happen, just in case I screw up in the future } } } finally { clientEvents.CompleteAdding(); } })); }
public override Task <AcquireLeaseResponse> AcquireLease(AcquireLeaseRequest request, ServerCallContext context) { AcquireLeaseResponse.Types.Status status; Lease lease = null; SpotRobot.SpotInstance.Leases.TryGetValue(request.Resource, out var leaseTuple); if (leaseTuple == null) { lease = new Lease { Resource = request.Resource, Sequence = { 1 } }; SpotRobot.SpotInstance.Leases.Add(request.Resource, new Tuple <string, Lease>(request.Header.ClientName, lease)); Console.WriteLine($"AcquireLease - registering new lease for resource \"{request.Resource}\" and client \"{request.Header.ClientName}\""); status = AcquireLeaseResponse.Types.Status.Ok; } else if (leaseTuple.Item1 == request.Header.ClientName) { Console.WriteLine($"AcquireLease - lease already exists for resource \"{request.Resource}\" and client \"{request.Header.ClientName}\""); status = AcquireLeaseResponse.Types.Status.Ok; } else { Console.WriteLine($"AcquireLease - lease already exists for resource \"{request.Resource}\" and but no for client \"{request.Header.ClientName}\""); status = AcquireLeaseResponse.Types.Status.ResourceAlreadyClaimed; } return(Task.FromResult(new AcquireLeaseResponse { Header = HeaderBuilder.Build(request.Header, new CommonError { Code = CommonError.Types.Code.Ok }), Status = status, Lease = lease, LeaseOwner = new LeaseOwner { ClientName = request.Header.ClientName, } })); }