private async Task TryConn(int retryCount, CancellationToken token)
{
var ipAddress = IPAddress.TryParse(Host, out var ip)
? ip
: (await Dns.GetHostEntryAsync(Host)).AddressList.FirstOrDefault();
var backoff = TimeSpan.FromMilliseconds(100);
for (var i = 0; i < retryCount && token.IsCancellationRequested == false; i++)
{
try
{
await _socket.ConnectAsync(ipAddress, Port);
_logger.LogInformation($"Connect to the RegionServer({Type}) at {Host}:{Port}");
return;
}
catch (Exception e)
{
_logger.LogWarning($"Connect RegionServer({Host}:{Port}) failed in {i + 1}th，try the locate again after {backoff}.\n{e}");
backoff = await TaskEx.SleepAndIncreaseBackoff(backoff, TimeSpan.FromSeconds(5), token);
}
}
_logger.LogError($"Connect RegionServer({Host}:{Port}) failed.");
}
private async Task <(RegionClient client, RegionInfo info)> TryLocateRegion(ICall rpc, CancellationToken token)
{
var backoff = BackoffStart;
while (rpc.FindRegionRetryCount < RetryCount && token.IsCancellationRequested == false)
{
rpc.FindRegionRetryCount++;
var reg = await FindRegionInfoForRPC(rpc.Table, rpc.Key, token);
if (reg == null)
{
_logger.LogWarning($"Locate region failed in {rpc.FindRegionRetryCount}th，try the locate again after {backoff}.");
backoff = await TaskEx.SleepAndIncreaseBackoff(backoff, BackoffIncrease, token);
continue;
}
var client = _cache.GetClient(reg.Host, reg.Port)
?? await new RegionClient(reg.Host, reg.Port, RegionType.ClientService)
{
TimeOut = Timeout,
EffectiveUser = EffectiveUser,
CallQueueSize = CallQueueSize
}
.Build(RetryCount, token);
reg.Client = client;
return(client, reg);
}
return(null, null);
}
protected async Task <TResult> TryLocateResource <TResult>(string resource, Func <byte[], TResult> getResultFunc, CancellationToken token)
{
var zkc = _zkHelper.CreateClient(ZkQuorum, Timeout);
var backoff = BackoffStart;
var result = default(TResult);
for (var i = 0; i < RetryCount && token.IsCancellationRequested == false; i++)
{
result = await _zkHelper.LocateResource(zkc, resource, getResultFunc);
if (result == null)
{
_logger.LogWarning($"Locate {resource} failed in {i + 1}th，try the locate again after {backoff}.");
backoff = await TaskEx.SleepAndIncreaseBackoff(backoff, BackoffIncrease, token);
}
else
{
break;
}
}
await zkc.closeAsync();
if (result == null)
{
_logger.LogWarning(
$"Locate {resource} failed in {RetryCount}th, please check your network.");
}
return(result);
}
private async Task <bool> CheckProcedureWithBackoff(ulong procId, CancellationToken token)
{
var backoff = BackoffStart;
var oldTime = DateTime.Now;
while (DateTime.Now - oldTime < Timeout)
{
var req = new GetProcedureStateCall(procId);
await _adminClient.QueueRPC(req);
var res = await _adminClient.GetRPCResult(req.CallId);
if (res == null)
{
return(false);
}
if (res.Msg is GetProcedureResultResponse rep)
{
switch (rep.State)
{
case GetProcedureResultResponse.Types.State.NotFound:
return(false);
case GetProcedureResultResponse.Types.State.Finished:
return(true);
}
}
else if (res.Error != null)
{
switch (res.Error)
{
case DoNotRetryIOException _:
return(false);
}
}
backoff = await TaskEx.SleepAndIncreaseBackoff(backoff, BackoffIncrease, token);
}
return(false);
}
