static void ReceiveThreadProc(object _host)
{
string host = (string)_host;
byte[] buf = new byte[8 + 4 + 4 + 4]; //!
Entry.LongToBytes(jid, buf, 0);
Entry.ToBytes(heartbeattimeout, buf, 8);
Entry.ToBytes(heartbeatretries, buf, 8 + 4);
Entry.ToBytes(tattletimeout, buf, 8 + 4 + 4);
try
{
System.Net.Sockets.Socket sock = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp);
sock.Connect(host, 55900);
System.Net.Sockets.NetworkStream netsm = new XNetworkStream(sock);
lock (hostToNetsms)
{
hostToNetsms[host] = netsm;
}
netsm.WriteByte((byte)'h'); // start vitals reporter
XContent.SendXContent(netsm, buf);
while (!stop)
{
int ib = netsm.ReadByte();
if (ib == -1)
{
break;
}
else if (ib == 'h') //heartbeat
{
#if FAILOVER_DEBUG
Log("Heartbeat thread received a heartbeat from:" + host);
#endif
lock (heartBeats)
{
heartBeats[host] = DateTime.Now.Ticks;
}
}
else if (ib == (byte)'e') //tattled bad hosts
{
string strbhs = XContent.ReceiveXString(netsm, buf);
#if FAILOVER_DEBUG
Log("Heartbeat thread received tattled host from:" + host + ";tattled host=" + strbhs);
#endif
string[] bhs = strbhs.ToLower().Split(';');
lock (rogueHosts)
{
foreach (string bh in bhs)
{
if (!rogueHosts.ContainsKey(bh))
{
rogueHosts.Add(bh, "Tattled by " + host);
}
}
}
#if FAILOVER_DEBUG
Log("VitalMonitor.rogueHosts:" + string.Join(";", (new List<string>(rogueHosts.Keys).ToArray())));
#endif
}
else
{
#if FAILOVER_DEBUG
Log("VitalMonitor receiving thread obtained unknown heartbeat: " + ib.ToString() + " from host:" + host);
#endif
throw new Exception("VitalMonitor receiving thread obtained unknown heartbeat: " + ib.ToString() + " from host:" + host);
}
}
}
catch (Exception e)
{
LogOutputToFile("VitalMonitor receiver thread exception:" + e.ToString());
}
}
//timeout=frequency of heartbeat
//retries=how many retries slave should do before sending a heartbeat
//expired=when does a heartbeat expired
internal static void Start(string[] _hosts, int timeout, int retries, int expired)
{
rogueHosts = new Dictionary<string, int>(_hosts.Length, new Surrogate.CaseInsensitiveEqualityComparer());
heartBeats = new Dictionary<string, long>(_hosts.Length, new Surrogate.CaseInsensitiveEqualityComparer());
hostToNetsms = new Dictionary<string, System.Net.Sockets.NetworkStream>(_hosts.Length, new Surrogate.CaseInsensitiveEqualityComparer());
string[] hosts = new string[_hosts.Length];
for (int i = 0; i < _hosts.Length; i++)
{
hosts[i] = _hosts[i].ToLower();
}
receivethds = new System.Threading.Thread[hosts.Length];
#if HEARTBEAT_DEBUG
Log("Heartbeat started");
#endif
buf = new byte[4 + 4];
Entry.ToBytes(timeout, buf, 0);
Entry.ToBytes(retries, buf, 4);
for (int i = 0; i < hosts.Length; i++)
{
hosts[i] = hosts[i].ToLower();
string host = hosts[i];
heartBeats[host] = 0;
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
try
{
System.Net.Sockets.Socket sock = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp);
sock.Connect(host, 55900);
System.Net.Sockets.NetworkStream netsm = new XNetworkStream(sock);
lock (hostToNetsms)
{
hostToNetsms[host] = netsm;
}
netsm.WriteByte((byte)'h'); // start heartbeat
XContent.SendXContent(netsm, buf);
while(!stop)
{
int ib = netsm.ReadByte();
if (ib == -1)
{
break;
}
lock (heartBeats)
{
heartBeats[host] = DateTime.Now.Ticks;
}
}
}
catch(Exception e)
{
LogOutputToFile("HeartBeat receive thread exception:" + e.ToString());
}
}));
receivethds[i] = thd;
thd.Priority = System.Threading.ThreadPriority.Highest;
thd.IsBackground = true;
thd.Start();
}
System.Threading.Thread.Sleep(timeout * 2); //give some time for receivethds to start.
//monitor
monitorthd = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
//1 ms = 10000 ticks
long maxdiff = expired * 10000;
while (!stop)
{
#if HEARTBEAT_DEBUG
Log("monitor checking");
#endif
lock (heartBeats)
{
long tsnow = DateTime.Now.Ticks;
for (int i = 0; i < hosts.Length; i++)
{
string host = hosts[i];
if (tsnow - heartBeats[host] > maxdiff) //too many ticks have passed since the last heartbeat
{
#if HEARTBEAT_DEBUG
Log("roguehost found:" + host + "; " + (tsnow - heartBeats[host]).ToString());
#endif
lock (rogueHosts)
{
rogueHosts[host] = 0;
}
}
}
}
System.Threading.Thread.Sleep(timeout); //gives time for heartbeat to catch up.
}
}));
monitorthd.Priority = System.Threading.ThreadPriority.Highest;
monitorthd.IsBackground = true;
monitorthd.Start();
}
