public ResultQueueDto(string key, IPEndPoint endPoint, string flowControlCfgKey, ServiceConfigureInfo configure, object result)
{
Key = key;
EndPoint = endPoint;
FlowControlCfgKey = flowControlCfgKey;
Configure = configure;
Result = result;
}
async Task TrySetConfigure(ISyncServiceFlowControlConfigureGrain grain, ServiceConfigureInfo newConfigure)
{
try
{
await grain.SetConfigure(newConfigure);
}
catch (Exception)
{
}
}
/// <summary>
/// 强制熔断无法连通的EndPoint
/// </summary>
/// <param name="pathName"></param>
/// <param name="servcieInfo"></param>
/// <param name="breakEndPoint"></param>
public async Task ForcedCircuitBreakEndPoint(string flowControlCfgKey, ServiceConfigureInfo configure, IPEndPoint breakEndPoint)
{
var addr = configure.GetEndPoints().FirstOrDefault(x => x.GetEndPoint().Equals(breakEndPoint));
if (addr != null)
{
addr.BreakerTime = DateTime.Now;
}
await UpdateBreakerConfigure(flowControlCfgKey, configure);
}
/// <summary>
/// 检查令牌是否可用
/// </summary>
/// <param name="serviceName"></param>
/// <returns></returns>
public async Task <bool> Grant(ServiceConfigureInfo configure)
{
configure.UpdateTokens(Capacity, Rate);
if (configure.Tokens < 1)
{
var timeToIntervalEnd = configure.StartTimeStamp - DateTime.UtcNow.Ticks;
if (timeToIntervalEnd < 0)
{
return(await Grant(configure));
}
return(false);
}
configure.Tokens -= 1;
return(true);
}
public async Task SetConfigure(ServiceConfigureInfo newConfigure)
{
configure = newConfigure;
foreach (var observer in observers)
{
try
{
observer.UpdateFlowControlConfigure(newConfigure);
}
catch (Exception)
{
failobservers.Add(observer);
}
}
foreach (var observer in failobservers)
{
observers.Remove(observer);
}
await Task.CompletedTask;
}
/// <summary>
/// 获取服务配置节
/// </summary>
/// <param name="pathName"></param>
/// <returns></returns>
public async Task <ServiceConfigureInfo> GetBreakerConfigure(string flowControlCfgKey, bool readLocalCache = true)
{
var cacheKey = $"{OxygenSetting.BreakerSettingKey}{ flowControlCfgKey}";
ServiceConfigureInfo result = null;
if (readLocalCache)
{
if (!OrleanClientProvider.GetConfigureCache().TryGetValue(cacheKey, out result))
{
result = await _syncConfigureProvider.GetConfigure(cacheKey);
_ = Task.Run(() => RegisterConfigureObserver(cacheKey));
}
}
else
{
result = await _syncConfigureProvider.GetConfigure(cacheKey);
}
return(result);
}
/// <summary>
/// 通过配置组装熔断策略
/// </summary>
/// <param name="config"></param>
/// <returns></returns>
public IAsyncPolicy <T> BuildPolicy <T>(string key, ServiceConfigureInfo configure, IPEndPoint endpoint)
{
if (_lazyDefPolicy.Value.TryGetValue($"{key}{endpoint.ToString()}", out dynamic cachepolicy))
{
return(cachepolicy as IAsyncPolicy <T>);
}
else
{
var address = configure.GetEndPoints().FirstOrDefault(x => x.GetEndPoint().Equals(endpoint));
//定义默认策略
IAsyncPolicy <T> defPolicy = Policy <T> .Handle <Exception>().FallbackAsync(async(CancellationToken cancelToken) =>
{
//请求失败，更新配置节并进行回退
address.ThresholdBreakeTimes += 1;
tmpReqTime.Value.TryGetValue($"{key}{endpoint.ToString()}", out List <DateTime> time);
time = time ?? new List <DateTime>();
//若错误次数超过阈值或者阈值比例则触发熔断
if (address.ThresholdBreakeTimes >= configure.DefThresholdBreakeTimes || time.Count() == 0 || (time.Count() > 0 && (double)configure.DefThresholdBreakeRatePerSec >= (double)address.ThresholdBreakeTimes / (double)time.Count(y => y >= DateTime.Now.AddSeconds(1))))
{
_logger.LogError($"地址{address.GetEndPoint().ToString()}超过熔断阈值，强制熔断。详细信息{{当前IP熔断次数:{address.ThresholdBreakeTimes},成功请求次数:{time.Count()},熔断比率{(double)address.ThresholdBreakeTimes / (double)time.Count(y => y >= DateTime.Now.AddSeconds(1))}}}");
address.BreakerTime = DateTime.Now;
}
_logger.LogError($"地址{address.GetEndPoint().ToString()}请求出错,执行回退");
return(default);
/// <summary>
/// 检查服务断路状态
/// </summary>
/// <param name="pathName"></param>
/// <param name="serviceInfo"></param>
/// <param name="addr"></param>
/// <returns></returns>
public async Task <IPEndPoint> CheckCircuitByEndPoint(ServiceConfigureInfo configure, IPEndPoint clientEndPoint)
{
//根据配置抛弃断路状态地址
var useAddr = configure.GetEndPoints().Where(x => x.BreakerTime == null || (x.BreakerTime != null && x.BreakerTime.Value.AddSeconds(configure.DefBreakerRetryTimeSec) <= DateTime.Now)).ToList();
//若全部熔断
if (!useAddr.Any())
{
_logger.LogError("服务被熔断,无法提供服务");
return(null);
}
else
{
//负载均衡有效地址
var addr = _endPointConfigure.GetServieByLoadBalane(useAddr, clientEndPoint, LoadBalanceType.IPHash, configure);
//初始化令牌桶并判断是否限流
_tokenBucket.InitTokenBucket(configure.DefCapacity, configure.DefRateLimit);
if (await _tokenBucket.Grant(configure))
{
return(addr);
}
return(null);
}
}
/// <summary>
/// 发送消息
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="key"></param>
/// <param name="path"></param>
/// <param name="message"></param>
/// <returns></returns>
public async Task <T> SendMessage <T>(string channelKey, IPEndPoint endPoint, string flowControlCfgKey, ServiceConfigureInfo configure, string key, string path, object message) where T : class
{
T result = default;
if (Channels.TryGetValue(channelKey, out var _channel))
{
try
{
result = await _flowControlCenter.ExcuteAsync(key, endPoint, flowControlCfgKey, configure, async() =>
{
var taskId = Guid.NewGuid();
var sendMessage = new RpcGlobalMessageBase <object>
{
CustomerIp = _customerInfo.Ip,
TaskId = taskId,
Path = path,
Message = message is string?_serialize.Deserializes <object>(_serialize.SerializesJsonString((string)message)) : message
};
sendMessage.Sign(GlobalCommon.SHA256Encrypt(taskId + OxygenSetting.SignKey));
var resultTask = RegisterResultCallbackAsync(taskId);
await _channel.WriteAndFlushAsync(sendMessage);
var resultBt = await resultTask;
if (resultBt != null && resultBt.Any())
{
return(_serialize.Deserializes <T>(resultBt));
}
return(default);
/// <summary>
/// 根据断路策略执行远程调用
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="serviceName"></param>
/// <param name="path"></param>
/// <param name="endPoint"></param>
/// <param name="func"></param>
/// <returns></returns>
public async Task <T> ExcuteAsync <T>(string key, IPEndPoint endPoint, string flowControlCfgKey, ServiceConfigureInfo configure, Func <Task <T> > func) where T : class
{
if (configure == null)
{
//如果当前服务并未配置流控，则直接远程调用
return(await func());
}
else
{
//构造断路策略
var policy = _policyProvider.BuildPolicy <T>(key, configure, endPoint);
//启动polly进行调用检查
try
{
return(await policy.ExecuteAsync(async() =>
{
//远程调用
var result = await func();
//消费结果集
AddQueueResult(new ResultQueueDto(key, endPoint, flowControlCfgKey, configure, result));
return result;
}) as T);
}
catch (Exception e)
{
//ignore
}
}
return(default);
/// <summary>
/// 根据服务名返回IP地址
/// </summary>
/// <param name="serviceName"></param>
/// <param name="flowControlCfgKey"></param>
/// <param name="clientIp"></param>
/// <returns></returns>
public async Task <IPEndPoint> GetFlowControlEndPointByServicePath(string serviceName, ServiceConfigureInfo configure, IPEndPoint clientIp)
{
//根据服务返回健康地址
var healthNode = await _registerCenter.GetServieByName(serviceName);
if (healthNode != null && healthNode.Any())
{
if (configure != null)
{
//更新健康节点和缓存同步
configure.ReflushConfigureEndPoint(healthNode);
//若配置流控，则进行熔断和限流检测
var point = await _breaker.CheckCircuitByEndPoint(configure, clientIp);
if (point != null)
{
//将有效地址的熔断数据清空
configure.CleanBreakTimes();
return(point);
}
}
else
{
//如果当前服务并未配置流控，则直接负载均衡返回节点
return(_endPointConfigure.GetServieByLoadBalane(healthNode, clientIp, LoadBalanceType.IPHash));
}
}
else
{
//删除所有地址并同步
if (configure != null)
{
configure.RemoveAllNode();
}
_logger.LogError($"没有找到健康的服务节点：{serviceName}");
}
return(null);
}
public void UpdateFlowControlConfigure(ServiceConfigureInfo configure)
{
OrleanClientProvider.GetConfigureCache().Set(configure.FlowControlCfgKey, configure);
}
/// <summary>
/// 初始化同步配置节
/// </summary>
/// <param name="key"></param>
/// <param name="newConfigure"></param>
/// <returns></returns>
public async Task InitConfigure(string key, ServiceConfigureInfo newConfigure)
{
await DoSync(key, true, async (grain) => await TrySetConfigure(grain, newConfigure));
}
/// <summary>
/// 服务端初始化配置节
/// </summary>
/// <param name="flowControlCfgKey"></param>
/// <param name="servcieInfo"></param>
/// <returns></returns>
public async Task InitBreakerConfigure(string flowControlCfgKey, ServiceConfigureInfo configure)
{
await _syncConfigureProvider.InitConfigure($"{flowControlCfgKey}", configure);
}
/// <summary>
/// 更新服务配置节
/// </summary>
/// <param name="pathName"></param>
/// <returns></returns>
public async Task UpdateBreakerConfigure(string flowControlCfgKey, ServiceConfigureInfo configure)
{
var cacheKey = $"{OxygenSetting.BreakerSettingKey}{ flowControlCfgKey}";
await _syncConfigureProvider.SetConfigure(cacheKey, configure);
}
public IPEndPoint GetServieByLoadBalane(List <FlowControlEndPoint> lbEndPoints, IPEndPoint clientIp, LoadBalanceType type = LoadBalanceType.IPHash, ServiceConfigureInfo configure = null)
{
var result = default(FlowControlEndPoint);
if (lbEndPoints != null && lbEndPoints.Any())
{
//若没有客户端IP则默认调用随机
if (clientIp == null && type == LoadBalanceType.IPHash)
{
type = LoadBalanceType.Random;
}
switch (type)
{
//随机
case LoadBalanceType.Random:
result = lbEndPoints.OrderBy(x => Guid.NewGuid()).FirstOrDefault();
break;
//轮询
case LoadBalanceType.Polling:
result = TargetIp == null?lbEndPoints.FirstOrDefault() :
lbEndPoints.Any(x => x.HashSort > TargetIpSortInex) ?
lbEndPoints.First(x => x.HashSort > TargetIpSortInex) :
lbEndPoints.First();
TargetIp = result.GetEndPoint();
TargetIpSortInex = result.HashSort;
break;
//IP哈希
case LoadBalanceType.IPHash:
result = lbEndPoints[Math.Abs(clientIp.GetHashCode()) % lbEndPoints.Count];
break;
//最小连接
case LoadBalanceType.MinConnections:
result = lbEndPoints.OrderBy(x => x.ConnectCount).FirstOrDefault();
break;
}
}
if (configure != null)
{
configure.ChangeConnectCount(result.GetEndPoint(), true);
}
return(result.GetEndPoint());
}
