Пример #1
0
            //static bool isfirst = false;

            // buf must be at least 12 bytes.
            //--F-------------ADDCOOK------------------
            public void CopyInto(List <LongIntComboListPart.B12> cbuffer, byte[] buf)
            {
                ensurefzblock(false, blockbuflen);

                try
                {
                    int flen     = (int)fzblock.Length;
                    int b12count = flen / 12;

                    cbuffer.Clear();
                    int newcap = b12count;

                    /*if (isfirst)
                     * {
                     *  isfirst = false;
                     *  newcap *= 2;
                     * }*/
                    if (newcap > cbuffer.Capacity)
                    {
                        cbuffer.Capacity = newcap;
                    }

                    // Read from existing unsorted zblock file into buffer.
                    fzblock.Seek(0, System.IO.SeekOrigin.Begin);
                    LongIntComboListPart.B12 b12;
                    b12.A = 0;
                    b12.B = 0;
                    for (int i = 0; i != b12count; i++)
                    {
                        int szr = fzblock.Read(buf, 0, 12);
#if DEBUG
                        if (12 != szr)
                        {
                            throw new Exception("Unable to read");
                        }
#endif
                        b12.SetFromArray(buf);
                        cbuffer.Add(b12);
                    }
                }
                catch (Exception e)
                {
                    cbuffer.Clear();
                    XLog.errorlog("LongIntComboList CopyInto failure; zblock skipped: " + e.ToString());
                }

                _justclose();
            }
Пример #2
0
        private LongIntComboListPart(int capacity)
        {
            ZBlock.CleanZBlockFiles("unsorted", "sorted");

            b12buffer = new List <B12>(0);

            this.enums = new LongIntComboListPartEnumerator[32];

            // Pre-set with defaults.
            int numzblocks    = 139;
            int zblockbufsize = 131072; // 128 KB default.

            if (null != DistributedObjectsSlave.xslave)
            {
                System.Xml.XmlNode xzblocks = DistributedObjectsSlave.xslave["zblocks"];
                if (null != xzblocks)
                {
                    {
                        System.Xml.XmlAttribute xnzb = xzblocks.Attributes["count"];
                        if (null != xnzb)
                        {
                            numzblocks = int.Parse(xnzb.Value);
                        }
                    }
                    {
                        System.Xml.XmlAttribute xzbs = xzblocks.Attributes["addbuffersize"];
                        if (null != xzbs)
                        {
                            zblockbufsize = DistributedObjectsSlave.ParseCapacity(xzbs.Value);
                        }
                    }
                }
            }

            if (XLog.logging)
            {
                XLog.log("Creating " + numzblocks.ToString() + " ZBlock`s");
            }

            zblocks = new ZBlock[numzblocks];
            for (int i = 0; i != numzblocks; i++)
            {
                zblocks[i] = new ZBlock(this, i, zblockbufsize);
            }
        }
Пример #3
0
            //static bool isfirst = false;

            // buf must be at least 8 bytes.
            //--D-------------ADDCOOK------------------
            public void CopyInto(List <IntComboListPart.B8> cbuffer, byte[] buf)
            {
                ensurefzblock(false, blockbuflen);

                try
                {
                    int flen    = (int)fzblock.Length;
                    int b8count = flen / 8;

                    cbuffer.Clear();
                    int newcap = b8count;

                    /*if (isfirst)
                     * {
                     *  isfirst = false;
                     *  newcap *= 2;
                     * }*/
                    if (newcap > cbuffer.Capacity)
                    {
                        cbuffer.Capacity = newcap;
                    }

                    // Read from existing unsorted zblock file into buffer.
                    fzblock.Seek(0, System.IO.SeekOrigin.Begin);
                    IntComboListPart.B8 b8;
                    b8.A = 0;
                    b8.B = 0;
                    for (int i = 0; i != b8count; i++)
                    {
                        fzblock.Read(buf, 0, 8);
                        b8.SetFromArray(buf);
                        cbuffer.Add(b8);
                    }
                }
                catch (Exception e)
                {
                    cbuffer.Clear();
                    XLog.errorlog("IntComboList CopyInto failure; zblock skipped: " + e.ToString());
                }

                _justclose();
            }
Пример #4
0
        public void SetError(string msg)
        {
            lasterror = msg;

            if (maxerrors > 0)
            {
                maxerrors--;

                XLog.errorlog(msg);

                try
                {
                    XLog.errorlog(msg, XLog.UserLogFile, XLog.DelimitUserErrors);
                }
                catch
                {
                }
            }

            System.Threading.Thread.Sleep(200);
        }
Пример #5
0
        protected override void ProcessCommand(NetworkStream nstm, char tag)
        {
            //string s;
            int len;

            switch (tag)
            {
            case 'e':     // Batch 'get next' enumeration.
            {
                try
                {
                    int ienumid = nstm.ReadByte();
                    if (ienumid >= 0)
                    {
                        byte enumid = (byte)ienumid;
                        if (enumid >= this.enums.Length)
                        {
                            nstm.WriteByte((byte)'-');
                        }
                        else
                        {
                            if (null == this.enums[enumid])
                            {
                                this.enums[enumid] = new LongIntComboListPartEnumerator(this);
                            }
                            int offset = 0;
                            if (null == buf || buf.Length < 60)
                            {
                                throw new Exception("Enumeration batch buffer too small!");
                            }
                            //if (uniquecompression) // Compressed...
                            {
                                for (; ;)
                                {
                                    if (!this.enums[enumid].MoveNext())
                                    {
                                        break;
                                    }
                                    B12 b12 = this.enums[enumid].Current;
                                    // Using Big Endian!
                                    b12.CopyToArray(buf, offset);
                                    offset += 12;
                                    if (offset + 12 > buf.Length)
                                    {
                                        break;
                                    }
                                }
                            }
                            if (offset > 0)
                            {
                                nstm.WriteByte((byte)'+');
                                XContent.SendXContent(nstm, buf, offset);
                            }
                            else
                            {
                                nstm.WriteByte((byte)'-');
                            }
                        }
                    }
                }
                catch
                {
                    nstm.WriteByte((byte)'-');
                    throw;
                }
            }
            break;

            case 'n':     // Reset next in enumeration..
            {
                int ienumid = nstm.ReadByte();
                if (ienumid >= 0)
                {
                    byte enumid = (byte)ienumid;
                    if (XLog.logging)
                    {
                        XLog.log("Starting enumeration (enumid:" + enumid.ToString() + ")");
                    }
                    if (enumid < this.enums.Length &&
                        null != this.enums[enumid])
                    {
                        //this.enums[enumid].Reset();
                        this.enums[enumid] = null;
                    }
                }
            }
            break;

            case 's':
            {
                try
                {
#if ENABLE_TIMING
                    long start = 0;
                    if (XLog.logging)
                    {
                        QueryPerformanceCounter(out start);
                    }
#endif

                    int readbuflen = 1048576;
                    if (null != DistributedObjectsSlave.xslave)
                    {
                        System.Xml.XmlNode xzblocks = DistributedObjectsSlave.xslave["zblocks"];
                        if (null != xzblocks)
                        {
                            {
                                System.Xml.XmlAttribute xzbs = xzblocks.Attributes["readbuffersize"];
                                if (null != xzbs)
                                {
                                    readbuflen = DistributedObjectsSlave.ParseCapacity(xzbs.Value);
                                }
                            }
                        }
                    }

                    foreach (ZBlock zb in zblocks)
                    {
                        zb.LeaveAddMode(readbuflen);
                    }

                    foreach (ZBlock zb in zblocks)
                    {
                        zb.Sort(b12buffer, this.buf);
                    }

#if ENABLE_TIMING
                    if (XLog.logging)
                    {
                        long stop;
                        QueryPerformanceCounter(out stop);
                        long freq;
                        if (QueryPerformanceFrequency(out freq))
                        {
                            long secs = (stop - start) / freq;
                            if (secs > 10)
                            {
                                XLog.log("IntComboListPart sort seconds: " + secs.ToString());
                            }
                        }
                    }
#endif
                }
                finally
                {
                    nstm.WriteByte((byte)'+');
                }
            }
            break;

            case 'p':     // Batch push/publish...
            {
                buf = XContent.ReceiveXBytes(nstm, out len, buf);
                uint gbfree = (uint)(GetCurrentDiskFreeBytes() / 1073741824);
#if DEBUG
                {
                    string computer_name = System.Environment.GetEnvironmentVariable("COMPUTERNAME");
                    if (computer_name == "MAPDDRULE" || computer_name == "MAPDCMILLER" || computer_name == "computer_name")
                    {
                        gbfree = uint.MaxValue;
                    }
                }
#endif
                if (gbfree > 20)
                {
                    int pcount = len / 12;         // size of B12
                    int y      = 0;
                    for (int i = 0; i != pcount; i++)
                    {
                        TimedAdd(buf, y);
                        y += 12;
                    }
                }
                else
                {
                    if (!nofreedisklog)
                    {
                        nofreedisklog = true;
                        XLog.errorlog("Low free disk space; now dropping entries.");
                    }
                }
            }
            break;

            default:
                base.ProcessCommand(nstm, tag);
                break;
            }
        }
Пример #6
0
        // args: <ipaddr> <portnum> <typechar> <capacity> <logfile> <jid>
        static void Main(string[] args)
        {
            MySpace.DataMining.AELight.Surrogate.LogonMachines();

            try
            {
#if DEBUGnoisy
                XLog.errorlog("DistributedObjectsSlave EntryPoint: " + Environment.CommandLine);
#endif

#if DEBUG
                //System.Threading.Thread.Sleep(1000 * 8);
                {
                    //string computer_name = System.Environment.GetEnvironmentVariable("COMPUTERNAME");
                    //if (computer_name == "MAPDDRULE" || computer_name == "MAPDCMILLER")
                    {
                        if (System.IO.File.Exists("sleep.txt"))
                        {
                            System.Threading.Thread.Sleep(1000 * 8);
                            int i32 = 1 + 32;
                        }
                    }
                }
#endif

#if DEBUG
                SetPriorityClass(GetCurrentProcess(), BELOW_NORMAL_PRIORITY_CLASS);
#endif

                XLog.UserLogFile = args[4];
                if (XLog.UserLogFile.StartsWith("&"))
                {
                    XLog.UserLogFile       = XLog.UserLogFile.Substring(1);
                    XLog.DelimitUserErrors = true;
                }

#if SLAVE_TRACE
                try
                {
                    Thread mainthread             = System.Threading.Thread.CurrentThread;
                    System.Threading.Thread stthd = new System.Threading.Thread(
                        new System.Threading.ThreadStart(
                            delegate
                    {
                        string spid = System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
                        try
                        {
                            string dotracefile          = spid + ".trace";
                            const string tracefiledelim = "{C8683F6C-0655-42e7-ACD9-0DDED6509A7C}";
                            for (; ;)
                            {
                                System.IO.StreamWriter traceout = null;
                                for (System.Threading.Thread.Sleep(1000 * 60)
                                     ; !System.IO.File.Exists(dotracefile)
                                     ; System.Threading.Thread.Sleep(1000 * 60))
                                {
                                }
                                {
                                    string[] tfc;
                                    try
                                    {
                                        tfc = System.IO.File.ReadAllLines(dotracefile);
                                    }
                                    catch
                                    {
                                        continue;
                                    }
                                    if (tfc.Length < 1 || "." != tfc[tfc.Length - 1])
                                    {
                                        continue;
                                    }
                                    try
                                    {
                                        System.IO.File.Delete(dotracefile);
                                    }
                                    catch
                                    {
                                        continue;
                                    }
                                    if ("." != tfc[0])
                                    {
                                        string traceoutfp = tfc[0];
                                        try
                                        {
                                            traceout = System.IO.File.CreateText(traceoutfp);
                                            traceout.Write("BEGIN:");
                                            traceout.WriteLine(tracefiledelim);
                                        }
                                        catch
                                        {
                                            continue;
                                        }
                                    }
                                }
                                if (null == traceout)
                                {
                                    XLog.log("SLAVE_TRACE: " + spid + " Start");
                                }
                                for (; ; System.Threading.Thread.Sleep(1000 * 60))
                                {
                                    {
                                        try
                                        {
#if SLAVE_TRACE_PORT
                                            if (null == traceout)
                                            {
                                                try
                                                {
                                                    StringBuilder sbtp = new StringBuilder();
                                                    sbtp.Append("SLAVE_TRACE_PORT: " + spid + ":");
                                                    for (int i = 0; i < DOSlave_TracePorts.Count; i++)
                                                    {
                                                        sbtp.Append(' ');
                                                        sbtp.Append(DOSlave_TracePorts[i]);
                                                    }
                                                    if (0 == DOSlave_TracePorts.Count)
                                                    {
                                                        sbtp.Append(" None");
                                                    }
                                                    XLog.log(sbtp.ToString());
                                                }
                                                catch
                                                {
                                                }
                                            }
#endif
                                            bool thdsuspended = false;
                                            try
                                            {
                                                mainthread.Suspend();
                                                thdsuspended = true;
                                            }
                                            catch (System.Threading.ThreadStateException)
                                            {
                                            }
                                            try
                                            {
                                                System.Diagnostics.StackTrace st = new System.Diagnostics.StackTrace(mainthread, false);
                                                StringBuilder sbst       = new StringBuilder();
                                                const int maxframesprint = 15;
                                                for (int i = 0, imax = Math.Min(maxframesprint, st.FrameCount); i < imax; i++)
                                                {
                                                    if (0 != sbst.Length)
                                                    {
                                                        sbst.Append(", ");
                                                    }
                                                    string mn = "N/A";
                                                    try
                                                    {
                                                        System.Reflection.MethodBase mb = st.GetFrame(i).GetMethod();
                                                        mn = mb.ReflectedType.Name + "." + mb.Name;
                                                    }
                                                    catch
                                                    {
                                                    }
                                                    sbst.Append(mn);
                                                }
                                                if (st.FrameCount > maxframesprint)
                                                {
                                                    sbst.Append(" ... ");
                                                    sbst.Append(st.FrameCount - maxframesprint);
                                                    sbst.Append(" more");
                                                }
                                                if (null == traceout)
                                                {
                                                    XLog.log("SLAVE_TRACE: " + spid + " Trace: " + sbst.ToString());
                                                }
                                                else
                                                {
                                                    traceout.WriteLine(sbst.ToString());
                                                }
                                            }
                                            catch (Exception e)
                                            {
                                                XLog.log("SLAVE_TRACE: " + spid + " Error: " + e.ToString());
                                            }
                                            finally
                                            {
                                                if (thdsuspended)
                                                {
                                                    mainthread.Resume();
                                                }
                                            }
                                        }
                                        catch (Exception e)
                                        {
                                            XLog.log("SLAVE_TRACE: " + spid + " " + mainthread.Name + " Trace Error: Cannot access thread: " + e.ToString());
                                        }
                                    }

                                    if (null != traceout)
                                    {
                                        traceout.Write(tracefiledelim);
                                        traceout.WriteLine(":END");
                                        traceout.Close();
                                        break;
                                    }
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            XLog.log("SLAVE_TRACE: " + spid + " Trace Failure: " + e.Message);
                        }
                    }));
                    stthd.IsBackground = true;
                    stthd.Start();
                }
                catch (Exception est)
                {
                    XLog.log("SLAVE_TRACE: Thread start error: " + est.ToString());
                }
#endif

                sjid = args[5];
                {
                    int ic = sjid.IndexOf(':');
                    if (-1 != ic)
                    {
                        jobdesc = Encoding.UTF8.GetString(Convert.FromBase64String(sjid.Substring(ic + 1)));
                        sjid    = sjid.Substring(0, ic);
                    }
                }
                jid = long.Parse(sjid);

                try
                {
                    Environment.SetEnvironmentVariable("DOSLAVE", "DO5");
                }
                catch
                {
                }

                System.Xml.XmlDocument xd = new System.Xml.XmlDocument();
#if DEBUGslaveconfigload
                Random _scsr = new Random(DateTime.Now.Millisecond / 2 + System.Threading.Thread.CurrentThread.ManagedThreadId / 2);
                //for (int i = 0; i < 50; i++)
                {
                    System.Threading.Thread.Sleep(_scsr.Next(50, 200));
                    try
                    {
                        xd.Load("slaveconfig.j" + sjid + ".xml");
                        InitXmlConfig(xd);
                    }
                    catch (System.IO.FileNotFoundException e)
                    {
                        //System.Diagnostics.Debugger.Launch();
                    }
                    catch (Exception e)
                    {
                        System.Diagnostics.Debugger.Launch();
                        throw;
                    }
                }
#else
                try
                {
                    xd.Load("slaveconfig.j" + sjid + ".xml");
                    InitXmlConfig(xd);
                }
                catch (System.IO.FileNotFoundException e)
                {
                }
#endif

                if (XLog.logging)
                {
                    XLog.log("New Sub Process: '" + args[0] + "' '" + args[1] + "' '" + args[2] + "' '" + args[3] + "' '" + args[4] + "' '" + args[5] + "'");
                }

                Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
                sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
                string host    = args[0];
                int    portnum = ushort.Parse(args[1]);
#if SLAVE_TRACE_PORT
                DOSlave_TracePorts.Add(portnum);
#endif
                char typechar = args[2][0];
                try
                {
                    Thread.Sleep(100); // Give DLL a chance to start accepting.
                    sock.Connect(host, portnum);
                }
                catch (Exception e)
                {
                    throw new Exception("Sub-process-to-client(dll) connection error: " + e.ToString() + "  [Note: ensure port " + portnum.ToString() + " is free / not blocked]");
                }
                IDistObject dop;
                if ('H' == typechar)
                {
                    int capacity = ParseCapacity(args[3]);
                    if (XLog.logging)
                    {
                        XLog.log("Connected; creating Hashtable with capacity " + args[3]);
                    }
                    dop = HashtableObjectPart.Create(capacity);
                }
                else if ('8' == typechar)
                {
                    int capacity = ParseCapacity(args[3]);
                    if (XLog.logging)
                    {
                        XLog.log("Connected; creating IntComboList with capacity " + args[3]);
                    }
                    dop = IntComboListPart.Create(capacity);
                }
                else if ('C' == typechar) // 12 hex
                {
                    int capacity = ParseCapacity(args[3]);
                    if (XLog.logging)
                    {
                        XLog.log("Connected; creating LongIntComboList with capacity " + args[3]);
                    }
                    dop = LongIntComboListPart.Create(capacity);
                }
                else if ('A' == typechar)
                {
                    //int capacity = ParseCapacity(args[3]);
                    string[] a = args[3].Split(';');
                    int      count_capacity         = ParseCapacity(a[0]);
                    int      estimated_row_capacity = ParseCapacity(a[1]);
                    int      keylen = int.Parse(a[2]);
                    if (XLog.logging)
                    {
                        XLog.log("Connected; creating ArrayComboList(keylength=" + keylen.ToString() + ") with capacity " + args[3]);
                    }
                    dop = ArrayComboListPart.Create(count_capacity, estimated_row_capacity, keylen);
                }
                else if ('R' == typechar)
                {
                    if (XLog.logging)
                    {
                        XLog.log("Connected; creating Remote");
                    }
                    dop = RemotePart.Create();
                }
                else if ('F' == typechar)
                {
                    //int capacity = ParseCapacity(args[3]);
                    string[] a        = args[3].Split(';');
                    int      keylen   = ParseCapacity(a[2]);
                    int      valuelen = ParseCapacity(a[3]);
                    if (XLog.logging)
                    {
                        XLog.log("Connected; creating FixedArrayComboList(keylength=" + keylen.ToString() + ";valuelength=" + valuelen.ToString() + ")");
                    }
                    dop = FixedArrayComboListPart.Create(keylen, valuelen);
                }
                else
                {
                    throw new Exception("Data type not supported");
                }

                NetworkStream nstm = new XNetworkStream(sock);
                {
                    // New handshake with client(dll)...
                    nstm.WriteByte(1);
                    // ... and the SlavePID:
                    int    SlavePID = System.Diagnostics.Process.GetCurrentProcess().Id;
                    byte[] bpid     = DistObjectBase.ToBytes(SlavePID);
                    XContent.SendXContent(nstm, bpid);
                }

                {
                    int          pid         = System.Diagnostics.Process.GetCurrentProcess().Id;
                    string       spid        = pid.ToString();
                    string       pidfilename = spid + ".j" + sjid + ".slave.pid";
                    StreamWriter pidfile     = new StreamWriter(pidfilename);
                    pidfile.WriteLine(pid);
                    pidfile.WriteLine(System.DateTime.Now);
                    pidfile.WriteLine("jid={0}", sjid);
                    pidfile.Flush();
                    for (; ;)
                    {
                        try
                        {
                            dop.ProcessCommands(nstm);
                            break;
                        }
                        catch (DistObjectAbortException e)
                        {
                            XLog.errorlog("ProcessCommands exception: " + e.ToString() + " ((DistObjectAbortException))");
                            break; // !
                        }
                        catch (Exception e)
                        {
                            XLog.errorlog("ProcessCommands exception: " + e.ToString() + " ((recovering from this exception))");
                        }
                    }
                    pidfile.Close();
                    try
                    {
                        System.IO.File.Delete(pidfilename);
                    }
                    catch
                    {
                    }
                }
            }
            catch (Exception e)
            {
                XLog.errorlog("Main exception: " + e.ToString());
            }
        }
Пример #7
0
        public virtual void ProcessCommands(NetworkStream nstm)
        {
            //nstm = new XNetworkStream(sock);
            int x;

            this.nstm = nstm;
            try
            {
                for (; ;)
                {
                    x = nstm.ReadByte();
                    if (x < 0)
                    {
                        //throw new Exception("Unable to read tag for command");
                        break;
                    }
                    buf[0]         = (byte)x;
                    CurrentCommand = (char)x;

                    if ('\\' == (char)buf[0])
                    {
                        if (XLog.logging)
                        {
                            XLog.log("Received close command from service; closing");
                        }
                        break;
                    }
                    else if ('<' == (char)buf[0])
                    {
                        try
                        {
                            string xml = XContent.ReceiveXString(nstm, buf);
                            System.Xml.XmlDocument xd = new System.Xml.XmlDocument();
                            xd.LoadXml(xml);
                            DistributedObjectsSlave.InitXmlConfig(xd);
                            ReloadConfig();
                        }
                        catch
                        {
                            nstm.WriteByte((byte)'-');
                            throw;
                        }
                        nstm.WriteByte((byte)'+');
                        continue;
                    }
                    else if ('?' == (char)buf[0])
                    {
                        if (null != lasterror)
                        {
                            string lerr = lasterror;
                            lasterror = null; // Also resets error!
                            XContent.SendXContent(nstm, lerr);
                        }
                        else
                        {
                            XContent.SendXContent(nstm, buf, 0); // No error!
                        }
                        continue;
                    }

                    /*
                     * if (XLog.logging)
                     * {
                     *  XLog.log("Slave processing command " + ((char)buf[0]).ToString());
                     * }
                     * */
                    ProcessCommand(nstm, (char)buf[0]);
                }
            }
            catch (System.IO.IOException ioex)
            {
                // Drop SocketException (shutdown during tasks), rethrow others.
                if ((ioex.InnerException as SocketException) == null)
                {
                    throw;
                }
#if DEBUG
                XLog.errorlog("DistributedObjectsSlave Warning: IOException+SocketException during task shutdown: " + ioex.ToString());
#endif
            }

            nstm.Close();
            nstm.Dispose();
        }
Пример #8
0
        protected override void ProcessCommand(System.Net.Sockets.NetworkStream nstm, char tag)
        {
            int len;

            switch (tag)
            {
            case 'R':     // Remote!
            {
                string classname = XContent.ReceiveXString(nstm, buf);

                string xlibfn = CreateXlibFileName("remote");
                {
                    buf = XContent.ReceiveXBytes(nstm, out len, buf);
                    if (0 != len)
                    {
                        System.IO.FileStream stm = System.IO.File.Create(xlibfn);
                        stm.Write(buf, 0, len);
                        stm.Close();
                    }
                }

                string dllfn = CreateDllFileName("remote");
                {
                    buf = XContent.ReceiveXBytes(nstm, out len, buf);
                    System.IO.FileStream stm = System.IO.File.Create(dllfn);
                    stm.Write(buf, 0, len);
                    stm.Close();
                }

                if (XLog.logging)
                {
                    string xclassname = classname;
                    if (null == xclassname)
                    {
                        xclassname = "<null>";
                    }
                    XLog.log("Loading IRemote plugin named " + xclassname + " for remote: " + dllfn);
                }

                rem = LoadRemotePlugin(dllfn, classname);
#if DEBUG
                try
                {
                    rem.OnRemote();
                }
                catch (Exception e)
                {
                    throw new UserException(e);
                }
#else
                rem.OnRemote();
#endif
            }
            break;

            case 'O':     //Query DGlobals
            {
                int byteCount = DGlobalsM.ToBytes(ref buf);
                XContent.SendXContent(nstm, buf, byteCount);
            }
            break;

            case 'r':
            {
                buf = XContent.ReceiveXBytes(nstm, out len, buf);
                int n     = Entry.BytesToInt(buf);
                int count = 0;
                if (null != rem)
                {
                    List <long> appendsizes = new List <long>();
                    try
                    {
                        count = rem.GetOutputFileCount(n, appendsizes);
                    }
                    catch (Exception e)
                    {
                        throw new DistributedObjectsSlave.DistObjectAbortException(e);
                    }

                    if (buf.Length < 4 + 8 * appendsizes.Count)
                    {
                        buf = new byte[Entry.Round2Power(4 + 8 * appendsizes.Count)];
                    }
                    Entry.ToBytes(count, buf, 0);
                    int offset = 4;
                    for (int i = 0; i < appendsizes.Count; i++, offset += 8)
                    {
                        Entry.LongToBytes(appendsizes[i], buf, offset);
                    }
                    XContent.SendXContent(nstm, buf, 4 + 8 * appendsizes.Count);
                    break;         // !
                }
                Entry.ToBytes(count, buf, 0);
                XContent.SendXContent(nstm, buf, 4);
            }
            break;

            default:
                base.ProcessCommand(nstm, tag);
                break;
            }
        }
Пример #9
0
        internal void ThreadProc()
        {
            try
            {
                Random rnd = new Random(unchecked (System.DateTime.Now.Millisecond + System.Threading.Thread.CurrentThread.ManagedThreadId));
                List <PullFileInfo> infos = new List <PullFileInfo>(stop - start);
                for (int i = start; i < stop; i++)
                {
                    PullFileInfo info;
                    info.PullFile = allpullfiles[i];
                    string destfn;
                    {
                        int ix;
                        ix = info.PullFile.LastIndexOf('\u0002');
                        if (-1 != ix)
                        {
                            destfn        = info.PullFile.Substring(ix + 1);
                            info.PullFile = info.PullFile.Substring(0, ix);
                        }
                        else
                        {
                            ix = info.PullFile.LastIndexOf('\\');
                            if (-1 != ix)
                            {
                                destfn = info.PullFile.Substring(ix + 1);
                            }
                            else
                            {
                                destfn = info.PullFile;
                            }
                        }
                    }
                    info.DestFile       = destfn;
                    info.DestFileStream = null;
                    info.Position       = 0;
                    info.Retries        = 0;
                    infos.Add(info);
                }

                byte[] fbuf = new byte[MAX_SIZE_PER_RECEIVE];

                for (int i = 0; i < infos.Count; i++)
                {
                    PullFileInfo info = infos[i];
                    for (; ;)  //cooking
                    {
                        bool cooking_is_read = false;

                        try
                        {
                            if (info.DestFileStream == null)
                            {
                                if (info.Retries == 0)
                                {
                                    cooking_is_read = true;

#if TESTFAULTTOLERANT
                                    {
                                        int    dgdel  = info.PullFile.IndexOf(@"\", 2);
                                        string dghost = info.PullFile.Substring(2, dgdel - 2).ToLower();
                                        fttest.BreakPoint(dghost, "replication", new Exception("FAULTTOLERANT replication test exception"));
                                    }
#endif
                                    System.IO.File.Copy(info.PullFile, info.DestFile, true);
                                    cooking_is_read = false;
                                    break; // done with this pullfile
                                }
                                else
                                {
                                    info.DestFileStream = new System.IO.FileStream(info.DestFile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.Read, FILE_BUFFER_SIZE);
                                }
                            }

                            if (info.DestFileStream != null)
                            {
                                cooking_is_read = true;
                                using (System.IO.FileStream pullfs = new System.IO.FileStream(info.PullFile, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, FILE_BUFFER_SIZE))
                                {
#if TESTFAULTTOLERANT
                                    {
                                        int    dgdel  = info.PullFile.IndexOf(@"\", 2);
                                        string dghost = info.PullFile.Substring(2, dgdel - 2).ToLower();
                                        fttest.BreakPoint(dghost, "replication", new Exception("FAULTTOLERANT replication test exception"));
                                    }
#endif
                                    if (info.Position != 0)
                                    {
                                        pullfs.Position = info.Position;
                                    }
                                    for (; ;)
                                    {
                                        //----------------------------COOKING--------------------------------
                                        cooking_is_read = true;
                                        //----------------------------COOKING--------------------------------
                                        int xread = pullfs.Read(fbuf, 0, MAX_SIZE_PER_RECEIVE);
                                        info.Position += xread;
                                        //----------------------------COOKING--------------------------------
                                        cooking_is_read = false;
                                        //----------------------------COOKING--------------------------------
                                        if (xread <= 0)
                                        {
                                            break;
                                        }
                                        info.DestFileStream.Write(fbuf, 0, xread);
                                    }
                                    break;
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            if (!cooking_is_read)
                            {
                                if (info.DestFileStream != null)
                                {
                                    info.DestFileStream.Close();
                                }

                                try
                                {
                                    XLog.errorlog("Non-reading error; retries=" + info.Retries.ToString() +
                                                  ";puller machine=" + System.Net.Dns.GetHostName() +
                                                  ";pulling file=" + info.PullFile +
                                                  ";error=" + e.ToString());
                                }
                                catch
                                {
                                }

                                throw;
                            }

                            if (++info.Retries > ftretries)
                            {
                                if (info.DestFileStream != null)
                                {
                                    info.DestFileStream.Close();
                                }

                                //Report this host with corrupt file
                                int    del     = info.PullFile.IndexOf(@"\", 2);
                                string badhost = info.PullFile.Substring(2, del - 2);
                                RogueHosts.Add(jid, badhost);
                                lock (tattledhosts)
                                {
                                    tattledhosts[badhost] = 1;
                                }

                                try
                                {
                                    XLog.errorlog("Retries exceeded ftretries; retries=" + info.Retries.ToString() +
                                                  ";ftretries=" + ftretries.ToString() +
                                                  ";puller machine=" + System.Net.Dns.GetHostName() +
                                                  ";pulling file=" + info.PullFile +
                                                  ";tattled host=" + badhost);
                                }
                                catch
                                {
                                }

                                goto next_pullfile;
                            }

                            //Random swap with the rest of the list of pullfiles
                            int iswapwith = rnd.Next(i, infos.Count);
                            infos[i]         = infos[iswapwith];
                            infos[iswapwith] = info;
                            info             = infos[i];
                            continue;
                        }
                    }
next_pullfile:
                    int xxx = 10;
                }
            }
            catch (Exception ex)
            {
                exception = ex;
            }
        }
Пример #10
0
        protected override void ProcessCommand(NetworkStream nstm, char tag)
        {
            //string s;
            int len;

            switch (tag)
            {
            case 'N':     // Next in enumeration..
            {
                int ienumid = nstm.ReadByte();
                if (ienumid >= 0)
                {
                    byte enumid = (byte)ienumid;
                    if (enumid >= this.enums.Length)
                    {
                        nstm.WriteByte((byte)'-');
                    }
                    else
                    {
                        if (null == this.enums[enumid])
                        {
                            this.enums[enumid] = table.GetEnumerator();
                        }
                        if (this.enums[enumid].MoveNext())
                        {
                            nstm.WriteByte((byte)'+');
                            System.Collections.DictionaryEntry de = this.enums[enumid].Entry;
                            XContent.SendXContent(nstm, (byte[])de.Key);
                            if (AppendBufferingEnabled)
                            {
                                AppendBuffer abvalue = (AppendBuffer)de.Value;
                                byte[]       result  = abvalue.Get(out len);
                                XContent.SendXContent(nstm, result, len);
                            }
                            else
                            {
                                XContent.SendXContent(nstm, (byte[])de.Value);
                            }
                        }
                        else
                        {
                            nstm.WriteByte((byte)'-');
                        }
                    }
                }
            }
            break;

            case 'n':     // Reset next in enumeration..
            {
                int ienumid = nstm.ReadByte();
                if (ienumid >= 0)
                {
                    byte enumid = (byte)ienumid;
                    if (XLog.logging)
                    {
                        XLog.log("Starting enumeration (enumid:" + enumid.ToString() + ")");
                    }
                    if (enumid < this.enums.Length &&
                        null != this.enums[enumid])
                    {
                        //this.enums[enumid].Reset();
                        this.enums[enumid] = null;
                    }
                }
            }
            break;

            case 'b':     // Enable AppendBuffer
                XContent.ReceiveXBytes(nstm, out len, buf);
                appendbuffersize = BytesToInt(buf);
                if (appendbuffersize < 0 || appendbuffersize >= 16777216)
                {
                    throw new Exception("Invalid AppendBuffer preallocation size (appendbuffersize): " + appendbuffersize.ToString());
                }
                if (XLog.logging)
                {
                    XLog.log("Enabled AppendBuffer with initial value sizes of " + appendbuffersize.ToString());
                }
                break;

            case 'L':     // Set length..
            {
                // Key..
                byte[] putkey = XContent.ReceiveXBytes(nstm, out len, buf);
                putkey = GetSliceCopy(putkey, len);         // !
                // New length..
                byte[] putlength = XContent.ReceiveXBytes(nstm, out len, buf);
                int    newlength = BytesToInt(putlength);

                if (AppendBufferingEnabled)
                {
                    object ovalue;
                    if (UseStringKeys)
                    {
                        ovalue = table[Encoding.ASCII.GetString(putkey)];
                    }
                    else
                    {
                        ovalue = table[putkey];
                    }
                    if (null == ovalue)
                    {
                        int setbufsize = newlength;
                        if (setbufsize < appendbuffersize)
                        {
                            setbufsize = appendbuffersize;         // ?
                        }
                        if (UseStringKeys)
                        {
                            table[Encoding.ASCII.GetString(putkey)] = new AppendBuffer(setbufsize);
                        }
                        else
                        {
                            table[putkey] = new AppendBuffer(setbufsize);
                        }
                    }
                    else
                    {
                        AppendBuffer abvalue = (AppendBuffer)ovalue;
                        abvalue.SetLength(newlength);
                    }
                }
                else
                {
                    byte[] oldvalue = GetValue(putkey, out len);
                    if (newlength < len)
                    {
                        CopyAndSetValue(putkey, oldvalue, newlength);
                    }
                }
            }
            break;

            case 'G':     // Get..
            {
                //Thread.Sleep(15000);

                buf = XContent.ReceiveXBytes(nstm, out len, buf);
                byte[] getkey = GetSliceCopy(buf, len);

                byte[] result = GetValue(getkey, out len);
                if (null == result)
                {
                    nstm.WriteByte((byte)'-');         // Key doesn't exist!
                }
                else
                {
                    nstm.WriteByte((byte)'+');         // It exists!
                    XContent.SendXContent(nstm, result, len);
                }
            }
            break;

            case 'l':     // Get length..
            {
                byte[] getkey = XContent.ReceiveXBytes(nstm, out len, buf);
                getkey = GetSliceCopy(getkey, len);

                byte[] value = GetValue(getkey, out len);
                if (null == value)
                {
                    nstm.WriteByte((byte)'-');         // Key doesn't exist!
                }
                else
                {
                    nstm.WriteByte((byte)'+');         // It exists!
                    XContent.SendXContent(nstm, ToBytes(len));
                }
            }
            break;

            case 'P':     // Put..
            {
                // Key..
                buf = XContent.ReceiveXBytes(nstm, out len, buf);
                byte[] putkey = GetSliceCopy(buf, len);
                // Value..
                buf = XContent.ReceiveXBytes(nstm, out len, buf);
                CopyAndSetValue(putkey, buf, len);
            }
            break;

            case 'a':     // Math add..
            {
                // Key..
                buf = XContent.ReceiveXBytes(nstm, out len, buf);
                byte[] putkey = GetSliceCopy(buf, len);         // !
                // Operand..
                buf = XContent.ReceiveXBytes(nstm, out len, buf);
                if (buf.Length >= 4)
                {
                    int operand = BytesToInt(buf);
                    // Do add...
                    byte[] val = GetValue(putkey, out len);
                    if (null == val || val.Length < 4)         // Key doesn't exist, or too small, so assume it was 0.
                    {
                        ToBytes(operand, buf, 0);
                        CopyAndSetValue(putkey, buf, 4);
                    }
                    else
                    {
                        int existingint = BytesToInt(val);
                        int result      = existingint + operand;
                        //XLog.log("blarg", "adding " + existingint.ToString() + " and " + operand.ToString() + ", got result " + result.ToString());
                        ToBytes(result, buf, 0);
                        CopyAndSetValue(putkey, buf, 4);
                    }
                }
            }
            break;

            case 'A':     // Append..
            {
                // Key..
                buf = XContent.ReceiveXBytes(nstm, out len, buf);
                byte[] putkey = GetSliceCopy(buf, len);
                // Value..
                buf = XContent.ReceiveXBytes(nstm, out len, buf);
                AppendValue(putkey, buf, len);
            }
            break;

            default:
                base.ProcessCommand(nstm, tag);
                break;
            }
        }