Ejemplo n.º 1
0
        public bool CanBreak(string host, string phase)
        {
            host  = host.ToLower();
            phase = phase.ToLower();

            if (hosttophase.ContainsKey(host + ":" + phase))
            {
                return(true);
            }
            else if (phasetomax.ContainsKey(phase))
            {
                int max = phasetomax[phase];

                int ib = (byte)'-';  //can't throw by default
                lock (hosttophase)
                {
                    //Call to surrogate to check if it can throw.
                    netsm.WriteByte((byte)'1'); // Fault tolerant test
                    XContent.SendXContent(netsm, jobid.ToString() + ":" + host + ":" + phase + ":" + max.ToString());
                    ib = netsm.ReadByte();
                }
                if (ib == (byte)'+')
                {
                    return(true);
                }
            }
            return(false);
        }
Ejemplo n.º 2
0
            void _SimpleCmdPinSM(char tagchar, string arg)
            {
                _StartCmdPinSM(tagchar);
                XContent.SendXContent(pinstm, arg);
                int ich = pinstm.ReadByte();

                if ('+' != ich)
                {
                    string errmsg = null;
                    try
                    {
                        if ('-' == ich)
                        {
                            errmsg = XContent.ReceiveXString(pinstm, null);
                        }
                    }
                    catch
                    {
                    }
                    if (null != errmsg)
                    {
                        throw new Exception("MemCachePin service error: " + errmsg + "  {" + tagchar + "}");
                    }
                    throw new Exception("MemCachePin service did not return a success signal  {" + tagchar + "}");
                }
            }
Ejemplo n.º 3
0
        static void _MemCacheRelease(dfs.DfsFile df, bool force)
        {
            dfs    dc        = LoadDfsConfig();
            string exception = "";

            string[] slaves = dc.Slaves.SlaveList.Split(';');
            {
                //foreach (string slave in slaves)
                MySpace.DataMining.Threading.ThreadTools <string> .Parallel(
                    new Action <string>(
                        delegate(string slave)
                {
                    System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
                    nstm.WriteByte((byte)'C');
                    nstm.WriteByte((byte)'r');
                    XContent.SendXContent(nstm, df.Name);
                    XContent.SendXContent(nstm, new byte[1] {
                        (byte)(force ? 1 : 0)
                    });
                    int ich = nstm.ReadByte();
                    if ('+' != ich)
                    {
                        string errmsg = null;
                        if ('-' == ich)
                        {
                            try
                            {
                                errmsg = XContent.ReceiveXString(nstm, null);
                            }
                            catch
                            {
                            }
                        }
                        lock (slaves)
                        {
                            string newexception;
                            if (null != errmsg)
                            {
                                newexception = ("Error received from DO service during MemCache rollback from " + slave + ": " + errmsg);
                            }
                            else
                            {
                                newexception = ("Did not receive a success signal from DO service during MemCache rollback from " + slave);
                            }
                            if (string.IsNullOrEmpty(exception) ||
                                -1 != exception.IndexOf("MemCacheWarning"))
                            {
                                exception = newexception;
                            }
                        }
                    }
                }), slaves, slaves.Length);
            }
            if (!string.IsNullOrEmpty(exception))
            {
                throw new Exception(exception);
            }
        }
Ejemplo n.º 4
0
        static void _MemCacheLoad(dfs.DfsFile df)
        {
            dfs dc = LoadDfsConfig();

            string[] slaves = dc.Slaves.SlaveList.Split(';');
            {
                //foreach (string slave in slaves)
                MySpace.DataMining.Threading.ThreadTools <string> .Parallel(
                    new Action <string>(
                        delegate(string slave)
                {
                    System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
                    nstm.WriteByte((byte)'C');
                    nstm.WriteByte((byte)'l');
                    XContent.SendXContent(nstm, df.Name);
                    int ich = nstm.ReadByte();
                    if ('+' != ich)
                    {
                        string errmsg = null;
                        if ('-' == ich)
                        {
                            try
                            {
                                errmsg = XContent.ReceiveXString(nstm, null);
                            }
                            catch
                            {
                            }
                        }
                        if (null != errmsg)
                        {
                            throw new Exception("Error received from DO service during MemCache load from " + slave + ": " + errmsg);
                        }
                        throw new Exception("Did not receive a success signal from DO service during MemCache load from " + slave);
                    }
                }), slaves, slaves.Length);
            }
        }
Ejemplo n.º 5
0
        public static void MemCacheFlush(string mcname)
        {
            if (mcname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
            {
                mcname = mcname.Substring(6);
            }
            dfs dc = LoadDfsConfig();

            dfs.DfsFile df = dc.FindAny(mcname);
            if (null == df || df.MemCache == null)
            {
                Console.Error.WriteLine("Error: '{0}' is not a MemCache", (null == df ? mcname : df.Name));
                SetFailure();
                return;
            }

            string tempdfsname = mcname + Guid.NewGuid() + dfs.TEMP_FILE_MARKER;

            string[] slaves      = dc.Slaves.SlaveList.Split(';');
            long     dfsfilesize = 0;
            {
                string tempfp = "bp-mcflush" + Guid.NewGuid() + ".tmp";
                try
                {
                    using (System.IO.StreamWriter sw = new System.IO.StreamWriter(tempfp))
                    {
                        //foreach (string slave in slaves)
                        MySpace.DataMining.Threading.ThreadTools <string> .Parallel(
                            new Action <string>(
                                delegate(string slave)
                        {
                            System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
                            nstm.WriteByte((byte)'C');
                            nstm.WriteByte((byte)'f');
                            XContent.SendXContent(nstm, df.Name);
                            int ich = nstm.ReadByte();
                            if ('+' != ich)
                            {
                                string errmsg = null;
                                if ('-' == ich)
                                {
                                    try
                                    {
                                        errmsg = XContent.ReceiveXString(nstm, null);
                                    }
                                    catch
                                    {
                                    }
                                }
                                if (null != errmsg)
                                {
                                    throw new Exception("Error received from DO service during MemCache commit: " + errmsg);
                                }
                                throw new Exception("Did not receive a success signal from DO service during MemCache commit");
                            }
                            // flushinfos: chunk name, chunk size (without header)
                            string[] flushinfos = XContent.ReceiveXString(nstm, null).Split(
                                new char[] { '\r', '\n' },
                                StringSplitOptions.RemoveEmptyEntries);
                            foreach (string flushinfo in flushinfos)
                            {
                                lock (slaves)
                                {
                                    sw.WriteLine("{0} {1}", slave, flushinfo);
                                    dfsfilesize += int.Parse(flushinfo.Split(' ')[1]);
                                }
                            }
                        }), slaves, slaves.Length);
                    }
                    DfsBulkPut(new string[] { tempfp, tempdfsname, "rbin@" + df.RecordLength });
                }
                finally
                {
                    try
                    {
                        System.IO.File.Delete(tempfp);
                    }
                    catch
                    {
                    }
                }
            }

            Dictionary <string, bool> newchunknames = new Dictionary <string, bool>(StringComparer.OrdinalIgnoreCase);

            using (LockDfsMutex())
            {
                dc = LoadDfsConfig();
                dfs.DfsFile df2 = dc.FindAny(tempdfsname);
                if (null == df2)
                {
                    throw new Exception("DEBUG:  Temp DFS file not found: " + tempdfsname);
                }
                for (int i = 0; i < dc.Files.Count; i++)
                {
                    if (0 == string.Compare(dc.Files[i].Name, mcname, true))
                    {
                        dc.Files.RemoveAt(i);
                        break;
                    }
                }
                foreach (dfs.DfsFile.FileNode fn in df2.Nodes)
                {
                    newchunknames[fn.Name] = true;
                }
                df2.MemCache = df.MemCache;
                df2.Size     = dfsfilesize;
                df2.Name     = df.Name;
                UpdateDfsXml(dc);
            }

            {
                // Just kill the old chunks, not the MemCache stuff.
                List <string> delfnodes = new List <string>();
                {
                    //Collect file node paths.
                    for (int dn = 0; dn < df.Nodes.Count; dn++)
                    {
                        if (newchunknames.ContainsKey(df.Nodes[dn].Name))
                        {
                            continue;
                        }
                        foreach (string chost in df.Nodes[dn].Host.Split(';'))
                        {
                            delfnodes.Add(NetworkPathForHost(chost) + @"\" + df.Nodes[dn].Name);
                        }
                    }
                }
                _KillDataFileChunksInternal_unlocked_mt(delfnodes);
            }
        }
Ejemplo n.º 6
0
            //timeout=frequency of heartbeat
            //retries=how many retries slave should do before sending a heartbeat
            //expired=when does a heartbeat expired
            internal static void Start(string[] _hosts, int timeout, int retries, int expired)
            {
                rogueHosts   = new Dictionary <string, int>(_hosts.Length, new Surrogate.CaseInsensitiveEqualityComparer());
                heartBeats   = new Dictionary <string, long>(_hosts.Length, new Surrogate.CaseInsensitiveEqualityComparer());
                hostToNetsms = new Dictionary <string, System.Net.Sockets.NetworkStream>(_hosts.Length, new Surrogate.CaseInsensitiveEqualityComparer());

                string[] hosts = new string[_hosts.Length];
                for (int i = 0; i < _hosts.Length; i++)
                {
                    hosts[i] = _hosts[i].ToLower();
                }

                receivethds = new System.Threading.Thread[hosts.Length];

#if HEARTBEAT_DEBUG
                Log("Heartbeat started");
#endif

                buf = new byte[4 + 4];
                Entry.ToBytes(timeout, buf, 0);
                Entry.ToBytes(retries, buf, 4);

                for (int i = 0; i < hosts.Length; i++)
                {
                    hosts[i] = hosts[i].ToLower();
                    string host = hosts[i];
                    heartBeats[host] = 0;

                    System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
                    {
                        try
                        {
                            System.Net.Sockets.Socket sock = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp);
                            sock.Connect(host, 55900);
                            System.Net.Sockets.NetworkStream netsm = new XNetworkStream(sock);
                            lock (hostToNetsms)
                            {
                                hostToNetsms[host] = netsm;
                            }

                            netsm.WriteByte((byte)'h'); // start heartbeat
                            XContent.SendXContent(netsm, buf);

                            while (!stop)
                            {
                                int ib = netsm.ReadByte();
                                if (ib == -1)
                                {
                                    break;
                                }
                                lock (heartBeats)
                                {
                                    heartBeats[host] = DateTime.Now.Ticks;
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            LogOutputToFile("HeartBeat receive thread exception:" + e.ToString());
                        }
                    }));
                    receivethds[i]   = thd;
                    thd.Priority     = System.Threading.ThreadPriority.Highest;
                    thd.IsBackground = true;
                    thd.Start();
                }

                System.Threading.Thread.Sleep(timeout * 2); //give some time for receivethds to start.

                //monitor
                monitorthd = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
                {
                    //1 ms = 10000 ticks
                    long maxdiff = expired * 10000;
                    while (!stop)
                    {
#if HEARTBEAT_DEBUG
                        Log("monitor checking");
#endif
                        lock (heartBeats)
                        {
                            long tsnow = DateTime.Now.Ticks;

                            for (int i = 0; i < hosts.Length; i++)
                            {
                                string host = hosts[i];
                                if (tsnow - heartBeats[host] > maxdiff) //too many ticks have passed since the last heartbeat
                                {
#if HEARTBEAT_DEBUG
                                    Log("roguehost found:" + host + "; " + (tsnow - heartBeats[host]).ToString());
#endif

                                    lock (rogueHosts)
                                    {
                                        rogueHosts[host] = 0;
                                    }
                                }
                            }
                        }

                        System.Threading.Thread.Sleep(timeout); //gives time for heartbeat to catch up.
                    }
                }));
                monitorthd.Priority     = System.Threading.ThreadPriority.Highest;
                monitorthd.IsBackground = true;
                monitorthd.Start();
            }
Ejemplo n.º 7
0
            static void ReceiveThreadProc(object _host)
            {
                string host = (string)_host;

                byte[] buf = new byte[8 + 4 + 4 + 4]; //!
                Entry.LongToBytes(jid, buf, 0);
                Entry.ToBytes(heartbeattimeout, buf, 8);
                Entry.ToBytes(heartbeatretries, buf, 8 + 4);
                Entry.ToBytes(tattletimeout, buf, 8 + 4 + 4);

                try
                {
                    System.Net.Sockets.Socket sock = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp);
                    sock.Connect(host, 55900);
                    System.Net.Sockets.NetworkStream netsm = new XNetworkStream(sock);
                    lock (hostToNetsms)
                    {
                        hostToNetsms[host] = netsm;
                    }

                    netsm.WriteByte((byte)'h'); // start vitals reporter
                    XContent.SendXContent(netsm, buf);

                    while (!stop)
                    {
                        int ib = netsm.ReadByte();
                        if (ib == -1)
                        {
                            break;
                        }
                        else if (ib == 'h') //heartbeat
                        {
#if FAILOVER_DEBUG
                            Log("Heartbeat thread received a heartbeat from:" + host);
#endif
                            lock (heartBeats)
                            {
                                heartBeats[host] = DateTime.Now.Ticks;
                            }
                        }
                        else if (ib == (byte)'e') //tattled bad hosts
                        {
                            string strbhs = XContent.ReceiveXString(netsm, buf);
#if FAILOVER_DEBUG
                            Log("Heartbeat thread received tattled host from:" + host + ";tattled host=" + strbhs);
#endif
                            string[] bhs = strbhs.ToLower().Split(';');
                            lock (rogueHosts)
                            {
                                foreach (string bh in bhs)
                                {
                                    if (!rogueHosts.ContainsKey(bh))
                                    {
                                        rogueHosts.Add(bh, "Tattled by " + host);
                                    }
                                }
                            }
#if FAILOVER_DEBUG
                            Log("VitalMonitor.rogueHosts:" + string.Join(";", (new List <string>(rogueHosts.Keys).ToArray())));
#endif
                        }
                        else
                        {
#if FAILOVER_DEBUG
                            Log("VitalMonitor receiving thread obtained unknown heartbeat: " + ib.ToString() + " from host:" + host);
#endif
                            throw new Exception("VitalMonitor receiving thread obtained unknown heartbeat: " + ib.ToString() + " from host:" + host);
                        }
                    }
                }
                catch (Exception e)
                {
                    LogOutputToFile("VitalMonitor receiver thread exception:" + e.ToString());
                }
            }