public bool CanBreak(string host, string phase)
{
host = host.ToLower();
phase = phase.ToLower();
if (hosttophase.ContainsKey(host + ":" + phase))
{
return(true);
}
else if (phasetomax.ContainsKey(phase))
{
int max = phasetomax[phase];
int ib = (byte)'-'; //can't throw by default
lock (hosttophase)
{
//Call to surrogate to check if it can throw.
netsm.WriteByte((byte)'1'); // Fault tolerant test
XContent.SendXContent(netsm, jobid.ToString() + ":" + host + ":" + phase + ":" + max.ToString());
ib = netsm.ReadByte();
}
if (ib == (byte)'+')
{
return(true);
}
}
return(false);
}
void _SimpleCmdPinSM(char tagchar, string arg)
{
_StartCmdPinSM(tagchar);
XContent.SendXContent(pinstm, arg);
int ich = pinstm.ReadByte();
if ('+' != ich)
{
string errmsg = null;
try
{
if ('-' == ich)
{
errmsg = XContent.ReceiveXString(pinstm, null);
}
}
catch
{
}
if (null != errmsg)
{
throw new Exception("MemCachePin service error: " + errmsg + " {" + tagchar + "}");
}
throw new Exception("MemCachePin service did not return a success signal {" + tagchar + "}");
}
}
static void _MemCacheRelease(dfs.DfsFile df, bool force)
{
dfs dc = LoadDfsConfig();
string exception = "";
string[] slaves = dc.Slaves.SlaveList.Split(';');
{
//foreach (string slave in slaves)
MySpace.DataMining.Threading.ThreadTools <string> .Parallel(
new Action <string>(
delegate(string slave)
{
System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
nstm.WriteByte((byte)'C');
nstm.WriteByte((byte)'r');
XContent.SendXContent(nstm, df.Name);
XContent.SendXContent(nstm, new byte[1] {
(byte)(force ? 1 : 0)
});
int ich = nstm.ReadByte();
if ('+' != ich)
{
string errmsg = null;
if ('-' == ich)
{
try
{
errmsg = XContent.ReceiveXString(nstm, null);
}
catch
{
}
}
lock (slaves)
{
string newexception;
if (null != errmsg)
{
newexception = ("Error received from DO service during MemCache rollback from " + slave + ": " + errmsg);
}
else
{
newexception = ("Did not receive a success signal from DO service during MemCache rollback from " + slave);
}
if (string.IsNullOrEmpty(exception) ||
-1 != exception.IndexOf("MemCacheWarning"))
{
exception = newexception;
}
}
}
}), slaves, slaves.Length);
}
if (!string.IsNullOrEmpty(exception))
{
throw new Exception(exception);
}
}
static void _MemCacheLoad(dfs.DfsFile df)
{
dfs dc = LoadDfsConfig();
string[] slaves = dc.Slaves.SlaveList.Split(';');
{
//foreach (string slave in slaves)
MySpace.DataMining.Threading.ThreadTools <string> .Parallel(
new Action <string>(
delegate(string slave)
{
System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
nstm.WriteByte((byte)'C');
nstm.WriteByte((byte)'l');
XContent.SendXContent(nstm, df.Name);
int ich = nstm.ReadByte();
if ('+' != ich)
{
string errmsg = null;
if ('-' == ich)
{
try
{
errmsg = XContent.ReceiveXString(nstm, null);
}
catch
{
}
}
if (null != errmsg)
{
throw new Exception("Error received from DO service during MemCache load from " + slave + ": " + errmsg);
}
throw new Exception("Did not receive a success signal from DO service during MemCache load from " + slave);
}
}), slaves, slaves.Length);
}
}
public static void MemCacheFlush(string mcname)
{
if (mcname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
mcname = mcname.Substring(6);
}
dfs dc = LoadDfsConfig();
dfs.DfsFile df = dc.FindAny(mcname);
if (null == df || df.MemCache == null)
{
Console.Error.WriteLine("Error: '{0}' is not a MemCache", (null == df ? mcname : df.Name));
SetFailure();
return;
}
string tempdfsname = mcname + Guid.NewGuid() + dfs.TEMP_FILE_MARKER;
string[] slaves = dc.Slaves.SlaveList.Split(';');
long dfsfilesize = 0;
{
string tempfp = "bp-mcflush" + Guid.NewGuid() + ".tmp";
try
{
using (System.IO.StreamWriter sw = new System.IO.StreamWriter(tempfp))
{
//foreach (string slave in slaves)
MySpace.DataMining.Threading.ThreadTools <string> .Parallel(
new Action <string>(
delegate(string slave)
{
System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
nstm.WriteByte((byte)'C');
nstm.WriteByte((byte)'f');
XContent.SendXContent(nstm, df.Name);
int ich = nstm.ReadByte();
if ('+' != ich)
{
string errmsg = null;
if ('-' == ich)
{
try
{
errmsg = XContent.ReceiveXString(nstm, null);
}
catch
{
}
}
if (null != errmsg)
{
throw new Exception("Error received from DO service during MemCache commit: " + errmsg);
}
throw new Exception("Did not receive a success signal from DO service during MemCache commit");
}
// flushinfos: chunk name, chunk size (without header)
string[] flushinfos = XContent.ReceiveXString(nstm, null).Split(
new char[] { '\r', '\n' },
StringSplitOptions.RemoveEmptyEntries);
foreach (string flushinfo in flushinfos)
{
lock (slaves)
{
sw.WriteLine("{0} {1}", slave, flushinfo);
dfsfilesize += int.Parse(flushinfo.Split(' ')[1]);
}
}
}), slaves, slaves.Length);
}
DfsBulkPut(new string[] { tempfp, tempdfsname, "rbin@" + df.RecordLength });
}
finally
{
try
{
System.IO.File.Delete(tempfp);
}
catch
{
}
}
}
Dictionary <string, bool> newchunknames = new Dictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
using (LockDfsMutex())
{
dc = LoadDfsConfig();
dfs.DfsFile df2 = dc.FindAny(tempdfsname);
if (null == df2)
{
throw new Exception("DEBUG: Temp DFS file not found: " + tempdfsname);
}
for (int i = 0; i < dc.Files.Count; i++)
{
if (0 == string.Compare(dc.Files[i].Name, mcname, true))
{
dc.Files.RemoveAt(i);
break;
}
}
foreach (dfs.DfsFile.FileNode fn in df2.Nodes)
{
newchunknames[fn.Name] = true;
}
df2.MemCache = df.MemCache;
df2.Size = dfsfilesize;
df2.Name = df.Name;
UpdateDfsXml(dc);
}
{
// Just kill the old chunks, not the MemCache stuff.
List <string> delfnodes = new List <string>();
{
//Collect file node paths.
for (int dn = 0; dn < df.Nodes.Count; dn++)
{
if (newchunknames.ContainsKey(df.Nodes[dn].Name))
{
continue;
}
foreach (string chost in df.Nodes[dn].Host.Split(';'))
{
delfnodes.Add(NetworkPathForHost(chost) + @"\" + df.Nodes[dn].Name);
}
}
}
_KillDataFileChunksInternal_unlocked_mt(delfnodes);
}
}
//timeout=frequency of heartbeat
//retries=how many retries slave should do before sending a heartbeat
//expired=when does a heartbeat expired
internal static void Start(string[] _hosts, int timeout, int retries, int expired)
{
rogueHosts = new Dictionary <string, int>(_hosts.Length, new Surrogate.CaseInsensitiveEqualityComparer());
heartBeats = new Dictionary <string, long>(_hosts.Length, new Surrogate.CaseInsensitiveEqualityComparer());
hostToNetsms = new Dictionary <string, System.Net.Sockets.NetworkStream>(_hosts.Length, new Surrogate.CaseInsensitiveEqualityComparer());
string[] hosts = new string[_hosts.Length];
for (int i = 0; i < _hosts.Length; i++)
{
hosts[i] = _hosts[i].ToLower();
}
receivethds = new System.Threading.Thread[hosts.Length];
#if HEARTBEAT_DEBUG
Log("Heartbeat started");
#endif
buf = new byte[4 + 4];
Entry.ToBytes(timeout, buf, 0);
Entry.ToBytes(retries, buf, 4);
for (int i = 0; i < hosts.Length; i++)
{
hosts[i] = hosts[i].ToLower();
string host = hosts[i];
heartBeats[host] = 0;
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
try
{
System.Net.Sockets.Socket sock = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp);
sock.Connect(host, 55900);
System.Net.Sockets.NetworkStream netsm = new XNetworkStream(sock);
lock (hostToNetsms)
{
hostToNetsms[host] = netsm;
}
netsm.WriteByte((byte)'h'); // start heartbeat
XContent.SendXContent(netsm, buf);
while (!stop)
{
int ib = netsm.ReadByte();
if (ib == -1)
{
break;
}
lock (heartBeats)
{
heartBeats[host] = DateTime.Now.Ticks;
}
}
}
catch (Exception e)
{
LogOutputToFile("HeartBeat receive thread exception:" + e.ToString());
}
}));
receivethds[i] = thd;
thd.Priority = System.Threading.ThreadPriority.Highest;
thd.IsBackground = true;
thd.Start();
}
System.Threading.Thread.Sleep(timeout * 2); //give some time for receivethds to start.
//monitor
monitorthd = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
//1 ms = 10000 ticks
long maxdiff = expired * 10000;
while (!stop)
{
#if HEARTBEAT_DEBUG
Log("monitor checking");
#endif
lock (heartBeats)
{
long tsnow = DateTime.Now.Ticks;
for (int i = 0; i < hosts.Length; i++)
{
string host = hosts[i];
if (tsnow - heartBeats[host] > maxdiff) //too many ticks have passed since the last heartbeat
{
#if HEARTBEAT_DEBUG
Log("roguehost found:" + host + "; " + (tsnow - heartBeats[host]).ToString());
#endif
lock (rogueHosts)
{
rogueHosts[host] = 0;
}
}
}
}
System.Threading.Thread.Sleep(timeout); //gives time for heartbeat to catch up.
}
}));
monitorthd.Priority = System.Threading.ThreadPriority.Highest;
monitorthd.IsBackground = true;
monitorthd.Start();
}
static void ReceiveThreadProc(object _host)
{
string host = (string)_host;
byte[] buf = new byte[8 + 4 + 4 + 4]; //!
Entry.LongToBytes(jid, buf, 0);
Entry.ToBytes(heartbeattimeout, buf, 8);
Entry.ToBytes(heartbeatretries, buf, 8 + 4);
Entry.ToBytes(tattletimeout, buf, 8 + 4 + 4);
try
{
System.Net.Sockets.Socket sock = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp);
sock.Connect(host, 55900);
System.Net.Sockets.NetworkStream netsm = new XNetworkStream(sock);
lock (hostToNetsms)
{
hostToNetsms[host] = netsm;
}
netsm.WriteByte((byte)'h'); // start vitals reporter
XContent.SendXContent(netsm, buf);
while (!stop)
{
int ib = netsm.ReadByte();
if (ib == -1)
{
break;
}
else if (ib == 'h') //heartbeat
{
#if FAILOVER_DEBUG
Log("Heartbeat thread received a heartbeat from:" + host);
#endif
lock (heartBeats)
{
heartBeats[host] = DateTime.Now.Ticks;
}
}
else if (ib == (byte)'e') //tattled bad hosts
{
string strbhs = XContent.ReceiveXString(netsm, buf);
#if FAILOVER_DEBUG
Log("Heartbeat thread received tattled host from:" + host + ";tattled host=" + strbhs);
#endif
string[] bhs = strbhs.ToLower().Split(';');
lock (rogueHosts)
{
foreach (string bh in bhs)
{
if (!rogueHosts.ContainsKey(bh))
{
rogueHosts.Add(bh, "Tattled by " + host);
}
}
}
#if FAILOVER_DEBUG
Log("VitalMonitor.rogueHosts:" + string.Join(";", (new List <string>(rogueHosts.Keys).ToArray())));
#endif
}
else
{
#if FAILOVER_DEBUG
Log("VitalMonitor receiving thread obtained unknown heartbeat: " + ib.ToString() + " from host:" + host);
#endif
throw new Exception("VitalMonitor receiving thread obtained unknown heartbeat: " + ib.ToString() + " from host:" + host);
}
}
}
catch (Exception e)
{
LogOutputToFile("VitalMonitor receiver thread exception:" + e.ToString());
}
}