private static bool Alive(BrokerService.BrokerServiceClient client)
{
bool isAlive = false;
try
{
if (client.Ping(request: new Empty(), options: new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(2_000))).Reply)
{
isAlive = true;
}
}
catch
{
isAlive = false;
}
return(isAlive);
}
/// <summary>
/// 从IP列表获取有效IP
/// </summary>
/// <param name="ips"></param>
/// <param name="port"></param>
/// <returns></returns>
private static string GetValidIp(string ips, int port)
{
string[] ipsArr = ips.Split(',');
foreach (var ip in ipsArr)
{
if (ip != "127.0.0.1")
{
try
{
var client = new BrokerService.BrokerServiceClient(new Channel($"{ip}:{port}", ChannelCredentials.Insecure));
if (client.Ping(new Google.Protobuf.WellKnownTypes.Empty()).Reply)
{
return(ip);
}
}
catch
{
//
}
}
}
return(string.Empty);
}
/// <summary>
/// 健康检查，如果连接不上 每秒做一次尝试。
/// 尝试 errorCount 次失败，软删除。
/// 60次失败，永久删除。
/// </summary>
/// <param name="service">被检测主机信息</param>
/// <param name="errorCount">尝试 errorCount 次失败，软删除</param>
/// <returns></returns>
public static void HealthCheck(ServiceInfo service, int errorCount = 3)
{
int hc = 60;//检查次数
var client = new BrokerService.BrokerServiceClient(new Channel($"{service.Ip}:{service.Port}", ChannelCredentials.Insecure));
hCheck: //再次 心跳检测
try
{
service.Checking = true;
if (!client.Ping(new Empty()).Reply)
{
if (hc != 60)
{
Console.ForegroundColor = ConsoleColor.DarkGreen;
Console.WriteLine($"{DateTime.Now:yyyy-MM-dd HH:mm:ss}: ");
Console.WriteLine($"{service.Ip}:{service.Port}");
Console.WriteLine($"{service.Name}");
foreach (var f in service.Name.Split(','))
{
Console.WriteLine($"{f}");
}
Console.WriteLine($"{"w:" + service.Weight}");
Console.WriteLine($"恢复正常！");
Console.ResetColor();
Console.WriteLine($"----------------------------------------------------------------- ");
}
lock (LockHelper) //防止高并发下 影响权重
{
if (!Tc.ServiceInfoList.Exists(s => s.Ip == service.Ip && s.Port == service.Port))
{
//已存在不再添加 不存在则添加
for (int i = 0; i < service.Weight; i++)
{
Tc.ServiceInfoList.Add(service);
}
}
}
if (hc <= (60 - errorCount))
{
CheckNotice?.Invoke(service, NoticeType.RecoverHealth);
}
}
}
catch (Exception ex)
{
Console.WriteLine($"Error Info:{service.Ip}:{service.Port} {ex.Message}");
if (hc == 60)
{
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine($"{DateTime.Now:yyyy-MM-dd HH:mm:ss}: ");
Console.WriteLine($"{service.Ip}:{service.Port}");
foreach (var f in service.Name.Split(','))
{
Console.WriteLine($"{f}");
}
Console.WriteLine($"{"w:" + service.Weight}");
Console.WriteLine($"检测中···{hc}！");
Console.ResetColor();
Console.WriteLine($"----------------------------------------------------------------- ");
}
else if (hc == (60 - errorCount))
{
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine($"{DateTime.Now:yyyy-MM-dd HH:mm:ss}: ");
Console.WriteLine($"{service.Ip}:{service.Port}");
foreach (var f in service.Name.Split(','))
{
Console.WriteLine($"{f}");
}
Console.WriteLine($"{"w:" + service.Weight}");
Console.WriteLine($"故障恢复中···{hc}！");
Console.ResetColor();
Console.WriteLine($"----------------------------------------------------------------- ");
}
else if (hc == 0) //硬删除
{
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine($"{DateTime.Now:yyyy-MM-dd HH:mm:ss}:");
Console.WriteLine($"{service.Ip}:{service.Port}");
foreach (var f in service.Name.Split(','))
{
Console.WriteLine($"{f}");
}
Console.WriteLine($"{"w:" + service.Weight}");
Console.WriteLine($"永久移除！");
Console.ResetColor();
Console.WriteLine($"----------------------------------------------------------------- ");
}
if (hc == (60 - errorCount)) //三次失败之后 临时移除 ，防止更多请求转发给此服务节点
{
//临时移除 并不从配置文件移除
Tc.ServiceInfoList.RemoveAll(i => i.Ip == service.Ip && i.Port == service.Port);
CheckNotice?.Invoke(service, NoticeType.NotHealth);
}
else if (hc == 0) //硬删除
{
Dictionary <string, string> rp = new Dictionary <string, string>
{
{ "ip", service.Ip },
{ "port", service.Port.ToString() }
};
Tc.Remove(rp);
CheckNotice?.Invoke(service, NoticeType.OffLine);
return;
}
Thread.Sleep(1000); //间隔一秒 健康检查
hc--;
goto hCheck;
}
finally
{
service.Checking = false;
}
}
/// <summary>
/// 健康检查，如果连接不上 每秒做一次尝试。
/// 尝试 errorCount 次失败，软删除。
/// 60次失败，永久删除。
/// </summary>
/// <param name="service">被检测主机信息</param>
/// <param name="errorCount">尝试 errorCount 次失败，软删除</param>
/// <returns></returns>
public static void HealthCheck(ServiceInfo service, int errorCount = 3)
{
try
{
int hc = 60;//检查次数
service.Checking = true;
while (hc > 0)
{
var client = new BrokerService.BrokerServiceClient(new Channel($"{service.Ip}:{service.Port}", ChannelCredentials.Insecure));
if (Alive(client))
{
if (hc == 60)
{
break;
}
else
{
if (hc < (60 - errorCount))
{
WriteHealthCheck(service, hc, "恢复正常");
}
if (service.IsTemporaryRemove) //如果服务已被临时移除则找回
{
lock (LockHelper) //防止高并发下 影响权重
{
if (!Tc.ServiceInfoList.Exists(s => s.Ip == service.Ip && s.Port == service.Port))
{
//已存在不再添加 不存在则添加
for (int i = 0; i < service.Weight; i++)
{
Tc.ServiceInfoList.Add(service);
}
}
}
CheckNotice?.Invoke(service, NoticeType.RecoverHealth);
}
}
break;
}
else
{
hc--;
if (hc < (60 - errorCount) && (hc % errorCount == 0))
{
Log.Anno($"Error Info:{service.Ip}:{service.Port} not alive {hc}", typeof(Distribute));
}
if (hc == (60 - errorCount))//三次失败之后 临时移除 ，防止更多请求转发给此服务节点
{
//临时移除 并不从配置文件移除
Tc.ServiceInfoList.RemoveAll(i => i.Ip == service.Ip && i.Port == service.Port);
service.IsTemporaryRemove = true;
CheckNotice?.Invoke(service, NoticeType.NotHealth);
WriteHealthCheck(service, hc, "故障恢复中");
}
else if (hc == 0) //硬删除
{
Dictionary <string, string> rp = new Dictionary <string, string>
{
{ "ip", service.Ip },
{ "port", service.Port.ToString() }
};
Tc.Remove(rp);
CheckNotice?.Invoke(service, NoticeType.OffLine);
WriteHealthCheck(service, hc, "永久移除");
break;
}
Task.Delay(1000).Wait();//间隔一秒 健康检查
}
}
}
finally
{
service.Checking = false;
}
}