async Task RetryFailed <T>(IClientPool pool)
{
if (failed.Count == 0)
{
return;
}
foreach (var entry in failed)
{
IInvocation invocation = entry.Key;
var client = await pool.GetClient(entry.Value);
try
{
var refer = await client.Refer();
_source.WriteConsumerBefore(refer.Instance, entry.Value, invocation);
var result = await refer.Invoke <T>(invocation);
_source.WriteConsumerAfter(entry.Value, invocation, result);
await pool.Recovery(client);
failed.TryRemove(invocation, out URL cluster);
}
catch (Exception e)
{
_source.WriteConsumerError(entry.Value, invocation, e);
await pool.DestoryClient(client);
Logger().LogError(e, $"Failed retry to invoke method {invocation.MethodInfo.Name}, waiting again.");
}
}
}
public override async Task <IClusterResult <T> > DoInvoke <T>(IClientPool pool, ILoadBalance loadbalance, URL address, IList <URL> urls, IInvocation invocation)
{
var goodUrls = new List <URL>();
var badUrls = new List <BadUrl>();
Exception exception = null;
checkInvokers(urls, invocation, address);
var invoker = base.select(loadbalance, invocation, urls, null);
IResult <T> result;
var watch = Stopwatch.StartNew();
try
{
var client = await pool.GetClient(invoker);
try
{
var refer = await client.Refer();
_source.WriteConsumerBefore(refer.Instance, invoker, invocation);
result = await refer.Invoke <T>(invocation);
_source.WriteConsumerAfter(invoker, invocation, result);
await pool.Recovery(client);
goodUrls.Add(invoker);
}
catch (Exception ex)
{
await pool.DestoryClient(client).ConfigureAwait(false);
_source.WriteConsumerError(invoker, invocation, ex);
throw ex;
}
}
catch (Exception e)
{
Logger().LogError(e, $"Failback to invoke method {invocation.MethodInfo.Name}, wait for retry in background. Ignored exception:{e.Message}");
AddFailed <T>(pool, invocation, invoker);
watch.Stop();
result = new RpcResult <T>(watch.ElapsedMilliseconds); // ignore
exception = e;
badUrls.Add(new BadUrl {
Url = invoker, BadTime = DateTime.Now, CurrentException = exception
});
}
finally
{
if (watch.IsRunning)
{
watch.Stop();
}
}
return(new ClusterResult <T>(result, goodUrls, badUrls, exception, false));
}
public override async Task <IClusterResult <T> > DoInvoke <T>(IClientPool pool, ILoadBalance loadbalance, URL address, IList <URL> urls, IInvocation invocation)
{
checkInvokers(urls, invocation, address);
RpcContext.GetContext().SetInvokers(urls);
Exception exception = null;
var goodUrls = new List <URL>();
var badUrls = new List <BadUrl>();
var isThrow = false;
IResult <T> result = null;
foreach (var invoker in urls)
{
try
{
var client = await pool.GetClient(invoker);
try
{
var refer = await client.Refer();
_source.WriteConsumerBefore(refer.Instance, invoker, invocation);
result = await refer.Invoke <T>(invocation);
_source.WriteConsumerAfter(invoker, invocation, result);
await pool.Recovery(client);
goodUrls.Add(invoker);
}
catch (Exception ex)
{
await pool.DestoryClient(client);
_source.WriteConsumerError(invoker, invocation, ex);
throw ex;
}
}
catch (Exception e)
{
exception = e;
badUrls.Add(new BadUrl {
Url = invoker, BadTime = DateTime.Now, CurrentException = exception
});
Logger().LogWarning(e, e.Message);
}
}
if (exception != null)
{
isThrow = true;
}
var clusterResult = new ClusterResult <T>(result, goodUrls, badUrls, exception, isThrow);
return(clusterResult);
}
public override async Task <IClusterResult <T> > DoInvoke <T>(IClientPool pool, ILoadBalance loadbalance, URL address, IList <URL> urls, IInvocation invocation)
{
var goodUrls = new List <URL>();
var badUrls = new List <BadUrl>();
Exception exception = null;
checkInvokers(urls, invocation, address);
var invoker = base.select(loadbalance, invocation, urls, null);
IResult <T> result;
try
{
var client = await pool.GetClient(invoker);
try
{
var refer = await client.Refer();
_source.WriteConsumerBefore(refer.Instance, invoker, invocation);
result = await refer.Invoke <T>(invocation);
_source.WriteConsumerAfter(invoker, invocation, result);
await pool.Recovery(client);
goodUrls.Add(invoker);
return(new ClusterResult <T>(result, goodUrls, badUrls, exception, false));
}
catch (Exception ex)
{
await pool.DestoryClient(client).ConfigureAwait(false);
_source.WriteConsumerError(invoker, invocation, ex);
throw ex;
}
}
catch (Exception e)
{
exception = e;
badUrls.Add(new BadUrl {
Url = invoker, BadTime = DateTime.Now, CurrentException = exception
});
Logger().LogError(e, $"Failsafe ignore exception: {e.Message}");
result = new RpcResult <T>(e); // ignore
}
return(new ClusterResult <T>(result, goodUrls, badUrls, exception, false));
}
public override async Task <IClusterResult <T> > DoInvoke <T>(IClientPool pool, ILoadBalance loadbalance, URL address, IList <URL> urls, IInvocation invocation)
{
var goodUrls = new List <URL>();
var badUrls = new List <BadUrl>();
IResult <T> result = null;
Exception exception = null;
var isThrow = false;
checkInvokers(urls, invocation, address);
var invoker = base.select(loadbalance, invocation, urls, null);
try
{
var client = await pool.GetClient(invoker);
try
{
var refer = await client.Refer();
_source.WriteConsumerBefore(refer.Instance, invoker, invocation);
result = await refer.Invoke <T>(invocation);
_source.WriteConsumerAfter(invoker, invocation, result);
await pool.Recovery(client);
goodUrls.Add(invoker);
}
catch (Exception ex)
{
await pool.DestoryClient(client);
_source.WriteConsumerError(invoker, invocation, ex);
throw ex;
}
}
catch (Exception e)
{
isThrow = true;
if (e is RpcException eBiz && eBiz.Biz)
{
// biz exception.
exception = e;
//throw (RpcException)e;
}
public override async Task <IClusterResult <T> > DoInvoke <T>(IClientPool pool, ILoadBalance loadbalance, URL address, IList <URL> urls, IInvocation invocation)
{
var goodUrls = new List <URL>();
var badUrls = new List <BadUrl>();
//IResult result = null;
var copyinvokers = urls;
checkInvokers(copyinvokers, invocation, address);
//*getUrl();
int len = address.GetMethodParameter(invocation.MethodInfo.Name, RETRIES_KEY, DEFAULT_RETRIES) + 1;
if (len <= 0)
{
len = 1;
}
// retry loop.
RpcException le = null; // last exception.
var invoked = new List <URL>(copyinvokers.Count); // invoked invokers.
ISet <string> providers = new HashSet <string>(); //*len
for (int i = 0; i < len; i++)
{
//重试时，进行重新选择，避免重试时invoker列表已发生变化.
//注意：如果列表发生了变化，那么invoked判断会失效，因为invoker示例已经改变
if (i > 0)
{
//*checkWhetherDestroyed();
//*copyinvokers = list(invocation);
//重新检查一下
checkInvokers(copyinvokers, invocation, address);
}
var invoker = base.select(loadbalance, invocation, copyinvokers, invoked);
invoked.Add(invoker);
RpcContext.GetContext().SetInvokers(invoked);
try
{
var client = await pool.GetClient(invoker);
try
{
var refer = await client.Refer();
_source.WriteConsumerBefore(refer.Instance, invoker, invocation);
var result = await refer.Invoke <T>(invocation);
_source.WriteConsumerAfter(invoker, invocation, result);
await pool.Recovery(client);
if (le != null)
{
Logger().LogWarning(le, "Although retry the method " + invocation.MethodInfo.Name
+ " in the service " + invocation.TargetType.FullName
+ " was successful by the provider " + invoker.Address
+ ", but there have been failed providers " + string.Join(",", providers)
+ " (" + providers.Count + "/" + copyinvokers.Count
+ ") from the registry " + address.Address
+ " on the consumer " + Local.HostName
+ " using the service version " + invocation.Version
+ ". Last error is: " + le.Message);
}
goodUrls.Add(invoker);
return(new ClusterResult <T>(result, goodUrls, badUrls, le, false));
}
catch (Exception ex)
{
await pool.DestoryClient(client).ConfigureAwait(false);
_source.WriteConsumerError(invoker, invocation, ex);
throw ex;
}
}
catch (RpcException e)
{
if (e.Biz)
{ // biz exception.
throw e;
}
le = e;
}
catch (Exception e)
{
le = new RpcException(e.Message, e);
var badUrl = badUrls.FirstOrDefault(w => w.Url == invoker);
if (badUrl != null)
{
badUrls.Remove(badUrl);
}
badUrls.Add(new BadUrl {
Url = invoker, BadTime = DateTime.Now, CurrentException = le
});
}
finally
{
providers.Add(invoker.Address);
}
}
var re = new RpcException(le != null ? le.Code : 0, "Failed to invoke the method "
+ invocation.MethodInfo.Name + " in the service " + invocation.TargetType.FullName
+ ". Tried " + len + " times of the providers " + string.Join(",", providers)
+ " (" + providers.Count + "/" + copyinvokers.Count
+ ") from the registry " + address
//+ " on the consumer " + NetUtils.getLocalHost()
+ " using the service version " + invocation.Version
+ ". Last error is: "
+ (le != null ? le.Message : ""), le != null && le.InnerException != null ? le.InnerException : le);
return(new ClusterResult <T>(new RpcResult <T>(re), goodUrls, badUrls, re, true));
}
public override async Task <IClusterResult <T> > DoInvoke <T>(IClientPool pool, ILoadBalance loadbalance, URL address, IList <URL> urls, IInvocation invocation)
{
//IResult result = null;
var goodUrls = new List <URL>();
var badUrls = new List <BadUrl>();
checkInvokers(urls, invocation, address);
IList <URL> selected;
int forks = address.GetParameter(FORKS_KEY, DEFAULT_FORKS);
int timeout = address.GetParameter(TIMEOUT_KEY, DEFAULT_TIMEOUT);
if (forks <= 0 || forks >= urls.Count)
{
selected = urls;
}
else
{
selected = new List <URL>();
for (int i = 0; i < forks; i++)
{
//在invoker列表(排除selected)后,如果没有选够,则存在重复循环问题.见select实现.
var invoker = base.select(loadbalance, invocation, urls, selected);
if (!selected.Contains(invoker))
{//防止重复添加invoker
selected.Add(invoker);
}
}
}
RpcContext.GetContext().SetInvokers(selected);
var count = new AtomicInteger();
var taskList = new Task <IResult <T> > [selected.Count];
var index = 0;
foreach (var invoker in selected)
{
var task = Task.Run(async() => {
try
{
var client = await pool.GetClient(invoker);
try
{
var refer = await client.Refer();
_source.WriteConsumerBefore(refer.Instance, invoker, invocation);
var resultInner = await refer.Invoke <T>(invocation);
_source.WriteConsumerAfter(invoker, invocation, resultInner);
await pool.Recovery(client);
goodUrls.Add(invoker);
return(resultInner);
}
catch (Exception ex)
{
await pool.DestoryClient(client).ConfigureAwait(false);
_source.WriteConsumerError(invoker, invocation, ex);
throw ex;
}
}
catch (Exception e)
{
badUrls.Add(new BadUrl {
Url = invoker, BadTime = DateTime.Now, CurrentException = e
});
int value = count.IncrementAndGet();
if (value >= selected.Count)
{
return(new RpcResult <T>(e));
}
}
return(null);
});
taskList[index++] = task;
}
try
{
var retIndex = Task.WaitAny(taskList, timeout);
var ret = await taskList[retIndex];
if (ret.HasException)
{
Exception e = ret.Exception;
throw new RpcException(e is RpcException exception ? exception.Code : 0, "Failed to forking invoke provider " + selected + ", but no luck to perform the invocation. Last error is: " + e.Message, e.InnerException != null ? e.InnerException : e);
}
return(new ClusterResult <T>(ret, goodUrls, badUrls, null, false));
}
catch (Exception e)
{
var exception = new RpcException("Failed to forking invoke provider " + selected + ", but no luck to perform the invocation. Last error is: " + e.Message, e);
return(new ClusterResult <T>(new RpcResult <T>(exception), goodUrls, badUrls, exception, true));
}
}
public override async Task <IClusterResult <T> > DoInvoke <T>(IClientPool pool, ILoadBalance loadbalance, URL address, IList <URL> urls, IInvocation invocation)
{
var goodUrls = new List <URL>();
var badUrls = new List <BadUrl>();
var invokers = urls;
var merger = address.GetMethodParameter(invocation.MethodInfo.Name, MERGER_KEY);
// If a method doesn't have a merger, only invoke one Group
if (string.IsNullOrEmpty(merger))
{
foreach (var invoker in invokers)
{
try
{
var client = await pool.GetClient(invoker);
try
{
var refer = await client.Refer();
_source.WriteConsumerBefore(refer.Instance, invoker, invocation);
var invokeResult = await refer.Invoke <T>(invocation);
_source.WriteConsumerAfter(invoker, invocation, invokeResult);
await pool.Recovery(client);
goodUrls.Add(invoker);
return(new ClusterResult <T>(invokeResult, goodUrls, badUrls, null, false));
}
catch (Exception ex)
{
_source.WriteConsumerError(invoker, invocation, ex);
await pool.DestoryClient(client).ConfigureAwait(false);
throw ex;
}
}
catch (Exception e)
{
badUrls.Add(new BadUrl {
Url = invoker, BadTime = DateTime.Now, CurrentException = e
});
return(new ClusterResult <T>(new RpcResult <T>(e), goodUrls, badUrls, e, true));
}
}
var exMerger = new Exception($"merger: {merger} is null and the invokers is empty");
return(new ClusterResult <T>(new RpcResult <T>(exMerger), goodUrls, badUrls, exMerger, true));
}
Type returnType = invocation.TargetType.GetMethod(invocation.MethodInfo.Name, invocation.ArgumentTypes).ReturnType;
object resultValue = null;
var watch = Stopwatch.StartNew();
try
{
var results = new Dictionary <string, Task <IResult <T> > >();
foreach (var invoker in invokers)
{
var task = Task.Run(async() =>
{
try
{
var client = await pool.GetClient(invoker);
try
{
var refer = await client.Refer();
_source.WriteConsumerBefore(refer.Instance, invoker, invocation);
var invokeResult = await refer.Invoke <T>(invocation);
_source.WriteConsumerAfter(invoker, invocation, invokeResult);
await pool.Recovery(client);
goodUrls.Add(invoker);
return(invokeResult);
}
catch (Exception ex)
{
await pool.DestoryClient(client);
_source.WriteConsumerError(invoker, invocation, ex);
throw ex;
}
}
catch (Exception e)
{
badUrls.Add(new BadUrl {
Url = invoker, BadTime = DateTime.Now, CurrentException = e
});
return(new RpcResult <T>(e));
}
});
results.Add(invoker.ServiceKey, task);
}
var resultList = new List <IResult <T> >(results.Count);
int timeout = address.GetMethodParameter(invocation.MethodInfo.Name, TIMEOUT_KEY, DEFAULT_TIMEOUT);
Task.WaitAll(results.Values.ToArray(), timeout);
foreach (var entry in results)
{
var r = await entry.Value;
if (r.HasException)
{
Logger().LogError(r.Exception, $"Invoke {entry.Key} {getGroupDescFromServiceKey(entry.Key)} failed: {r.Exception.Message}");
return(new ClusterResult <T>(new RpcResult <T>(r.Exception), goodUrls, badUrls, r.Exception, true));
}
else
{
resultList.Add(r);
}
}
watch.Stop();
if (resultList.Count == 0)
{
return(new ClusterResult <T>(new RpcResult <T>(watch.ElapsedMilliseconds), goodUrls, badUrls, null, false));
}
else if (resultList.Count == 1)
{
return(new ClusterResult <T>(resultList[0], goodUrls, badUrls, null, false));
}
if (returnType == typeof(void))
{
return(new ClusterResult <T>(new RpcResult <T>(watch.ElapsedMilliseconds), goodUrls, badUrls, null, false));
}
if (merger.StartsWith("."))
{
merger = merger.Substring(1);
MethodInfo method;
try
{
method = returnType.GetMethod(merger, new[] { returnType });
}
catch (Exception e)
{
var ex = new RpcException($"Can not merge result because missing method [{merger}] in class [{returnType.Name}]{e.Message}", e);
return(new ClusterResult <T>(new RpcResult <T>(ex), goodUrls, badUrls, ex, true));
}
if (method == null)
{
var ex = new RpcException($"Can not merge result because missing method [ {merger} ] in class [ {returnType.Name} ]");
return(new ClusterResult <T>(new RpcResult <T>(ex), goodUrls, badUrls, ex, true));
}
//if (!method.IsPublic)
//{
//method.setAccessible(true);
//}
resultValue = resultList[0].Value;
resultList.RemoveAt(0);
try
{
if (method.ReturnType != typeof(void) &&
method.ReturnType.IsAssignableFrom(resultValue.GetType()))
{
foreach (var r in resultList)
{
resultValue = method.Invoke(resultValue, new object[] { r.Value });
}
}
else
{
foreach (var r in resultList)
{
method.Invoke(resultValue, new object[] { r.Value });
}
}
}
catch (Exception e)
{
var ex = new RpcException($"Can not merge result: {e.Message}", e);
return(new ClusterResult <T>(new RpcResult <T>(ex), goodUrls, badUrls, ex, true));
}
}
else
{
IMerger resultMerger;
if (merger.ToLower() == "true" || merger.ToLower() == "default")
{
resultMerger = MergerFactory.GetMerger(returnType, _defaultMergers);
}
else
{
resultMerger = _mySelfMergers[merger];
}
if (resultMerger != null)
{
var rets = new List <object>(resultList.Count);
foreach (var r in resultList)
{
rets.Add(r.Value);
}
resultValue = resultMerger.GetType().GetMethod("Merge", new Type[] { returnType }).Invoke(resultMerger, new[] { rets });
}
else
{
var ex = new RpcException("There is no merger to merge result.");
return(new ClusterResult <T>(new RpcResult <T>(ex), goodUrls, badUrls, ex, true));
}
}
return(new ClusterResult <T>(new RpcResult <T>((T)resultValue.ChangeType(typeof(T)), watch.ElapsedMilliseconds), goodUrls, badUrls, null, false));
}
catch (Exception ex)
{
Debug.Print(ex.StackTrace);
throw ex;
}
finally
{
if (watch.IsRunning)
{
watch.Stop();
}
}
}
