// Note: appendsizes can be null, but the return will still be valid. // Sizes exclude header data. protected virtual int GetNumberOfRemoteOutputFilesCreated(int n, IList <long> appendsizes) { int result = 0; foreach (SlaveInfo slave in dslaves) { slave.nstm.WriteByte((byte)'r'); Entry.ToBytes(n, buf, 0); XContent.SendXContent(slave.nstm, buf, 4); int len; buf = XContent.ReceiveXBytes(slave.nstm, out len, buf); if (len >= 4) { result += Entry.BytesToInt(buf); if (null != appendsizes) { for (int offset = 4; offset + 8 <= len; offset += 8) { appendsizes.Add(Entry.BytesToLong(buf, offset)); } } } } return(result); }
internal bool cangetnext() { lock (slave) { if (first) { first = false; internalsendreset(); } // 'N' for next.. slave.nstm.WriteByte((byte)'N'); slave.nstm.WriteByte(enumid); // Replies with tag '+' and xcontent key and xcontent value; or tag '-' if not exists. int len; int x = slave.nstm.ReadByte(); if ('+' == x) { byte[] key = XContent.ReceiveXBytes(slave.nstm, out len, null); // Null buffer for copy. byte[] value = XContent.ReceiveXBytes(slave.nstm, out len, null); // Null buffer for copy. cur = new System.Collections.DictionaryEntry(key, value); return(true); } else if ('-' == x) { return(false); } else { throw new Exception("Server returned invalid response for enumeration"); } } }
protected virtual void GetDGlobals() { foreach (SlaveInfo slave in dslaves) { slave.nstm.WriteByte((byte)'O'); int len = 0; buf = XContent.ReceiveXBytes(slave.nstm, out len, buf); DGlobalsM.FromBytes(buf, len, true); } }
protected string GetLastErrorReset() { #if SLAVE_GET_LAST_ERROR nstm.WriteByte((byte)'?'); int len; errbuf = XContent.ReceiveXBytes(nstm, out len, errbuf); if (0 != len) { return(Encoding.UTF8.GetString(errbuf, 0, len)); } #endif return(null); // No error. }
public int nextbufend = 0; // Where in nextbuf the valid elements end. internal bool cangetnext() { lock (slave) { if (first) { first = false; internalsendreset(); } if (nextbufpos < nextbufend) { fillcurfromnextbuf(); return(true); } else { // 'e for batch next.. slave.nstm.WriteByte((byte)'e'); // Replies with tag '+' and xcontent values; or tag '-' if not exists. //int len; int x = slave.nstm.ReadByte(); if ('+' == x) { nextbufpos = 0; nextbufend = 0; nextbuf = XContent.ReceiveXBytes(slave.nstm, out nextbufend, nextbuf); slave.SlaveErrorCheck(); fillcurfromnextbuf(); return(true); } else if ('-' == x) { return(false); } else { throw new Exception("Server returned invalid response for enumeration"); } } } }
// Returns -1 if no such key. public int Length(byte[] key) { if (!didopen) { throw new Exception("Must Open before writing to Hashtable"); } int slaveID = DetermineSlave(key); if (slaveID < 0 || slaveID >= dslaves.Count) { throw new Exception("Sub process missing: subProcessID needed: " + slaveID.ToString()); } SlaveInfo slave = dslaves[slaveID]; lock (slave) { // 'l' for get length slave.nstm.WriteByte((byte)'l'); XContent.SendXContent(slave.nstm, key); // Replies with tag '+' and xcontent length; or tag '-' if not exists. byte[] result = null; int len; int x = slave.nstm.ReadByte(); if ('+' == x) { result = XContent.ReceiveXBytes(slave.nstm, out len, buf); return(DistObjectBase.BytesToInt(result)); // Only reads first one. } else if ('-' == x) { //result = null; return(-1); } else { throw new Exception("Server returned invalid response for l (Length)"); } } }
// Gives null if no such key. public byte[] Get(byte[] key) { if (!didopen) { throw new Exception("Must Open before reading from Hashtable"); } int slaveID = DetermineSlave(key); if (slaveID < 0 || slaveID >= dslaves.Count) { throw new Exception("Sub process missing: subProcessID needed: " + slaveID.ToString()); } SlaveInfo slave = dslaves[slaveID]; lock (slave) { // 'G' for get slave.nstm.WriteByte((byte)'G'); XContent.SendXContent(slave.nstm, key); // Replies with tag '+' and xcontent value; or tag '-' if not exists. byte[] result = null; int len; int x = slave.nstm.ReadByte(); if ('+' == x) { result = XContent.ReceiveXBytes(slave.nstm, out len, null); // Null buffer for copy. } else if ('-' == x) { //result = null; } else { throw new Exception("Server returned invalid response for G (get)"); } return(result); } }
protected override void ProcessCommand(NetworkStream nstm, char tag) { //string s; int len; switch (tag) { case 'e': // Batch 'get next' enumeration. { try { int ienumid = nstm.ReadByte(); if (ienumid >= 0) { byte enumid = (byte)ienumid; if (enumid >= this.enums.Length) { nstm.WriteByte((byte)'-'); } else { if (null == this.enums[enumid]) { this.enums[enumid] = new LongIntComboListPartEnumerator(this); } int offset = 0; if (null == buf || buf.Length < 60) { throw new Exception("Enumeration batch buffer too small!"); } //if (uniquecompression) // Compressed... { for (; ;) { if (!this.enums[enumid].MoveNext()) { break; } B12 b12 = this.enums[enumid].Current; // Using Big Endian! b12.CopyToArray(buf, offset); offset += 12; if (offset + 12 > buf.Length) { break; } } } if (offset > 0) { nstm.WriteByte((byte)'+'); XContent.SendXContent(nstm, buf, offset); } else { nstm.WriteByte((byte)'-'); } } } } catch { nstm.WriteByte((byte)'-'); throw; } } break; case 'n': // Reset next in enumeration.. { int ienumid = nstm.ReadByte(); if (ienumid >= 0) { byte enumid = (byte)ienumid; if (XLog.logging) { XLog.log("Starting enumeration (enumid:" + enumid.ToString() + ")"); } if (enumid < this.enums.Length && null != this.enums[enumid]) { //this.enums[enumid].Reset(); this.enums[enumid] = null; } } } break; case 's': { try { #if ENABLE_TIMING long start = 0; if (XLog.logging) { QueryPerformanceCounter(out start); } #endif int readbuflen = 1048576; if (null != DistributedObjectsSlave.xslave) { System.Xml.XmlNode xzblocks = DistributedObjectsSlave.xslave["zblocks"]; if (null != xzblocks) { { System.Xml.XmlAttribute xzbs = xzblocks.Attributes["readbuffersize"]; if (null != xzbs) { readbuflen = DistributedObjectsSlave.ParseCapacity(xzbs.Value); } } } } foreach (ZBlock zb in zblocks) { zb.LeaveAddMode(readbuflen); } foreach (ZBlock zb in zblocks) { zb.Sort(b12buffer, this.buf); } #if ENABLE_TIMING if (XLog.logging) { long stop; QueryPerformanceCounter(out stop); long freq; if (QueryPerformanceFrequency(out freq)) { long secs = (stop - start) / freq; if (secs > 10) { XLog.log("IntComboListPart sort seconds: " + secs.ToString()); } } } #endif } finally { nstm.WriteByte((byte)'+'); } } break; case 'p': // Batch push/publish... { buf = XContent.ReceiveXBytes(nstm, out len, buf); uint gbfree = (uint)(GetCurrentDiskFreeBytes() / 1073741824); #if DEBUG { string computer_name = System.Environment.GetEnvironmentVariable("COMPUTERNAME"); if (computer_name == "MAPDDRULE" || computer_name == "MAPDCMILLER" || computer_name == "computer_name") { gbfree = uint.MaxValue; } } #endif if (gbfree > 20) { int pcount = len / 12; // size of B12 int y = 0; for (int i = 0; i != pcount; i++) { TimedAdd(buf, y); y += 12; } } else { if (!nofreedisklog) { nofreedisklog = true; XLog.errorlog("Low free disk space; now dropping entries."); } } } break; default: base.ProcessCommand(nstm, tag); break; } }
protected override void ProcessCommand(System.Net.Sockets.NetworkStream nstm, char tag) { int len; switch (tag) { case 'R': // Remote! { string classname = XContent.ReceiveXString(nstm, buf); string xlibfn = CreateXlibFileName("remote"); { buf = XContent.ReceiveXBytes(nstm, out len, buf); if (0 != len) { System.IO.FileStream stm = System.IO.File.Create(xlibfn); stm.Write(buf, 0, len); stm.Close(); } } string dllfn = CreateDllFileName("remote"); { buf = XContent.ReceiveXBytes(nstm, out len, buf); System.IO.FileStream stm = System.IO.File.Create(dllfn); stm.Write(buf, 0, len); stm.Close(); } if (XLog.logging) { string xclassname = classname; if (null == xclassname) { xclassname = "<null>"; } XLog.log("Loading IRemote plugin named " + xclassname + " for remote: " + dllfn); } rem = LoadRemotePlugin(dllfn, classname); #if DEBUG try { rem.OnRemote(); } catch (Exception e) { throw new UserException(e); } #else rem.OnRemote(); #endif } break; case 'O': //Query DGlobals { int byteCount = DGlobalsM.ToBytes(ref buf); XContent.SendXContent(nstm, buf, byteCount); } break; case 'r': { buf = XContent.ReceiveXBytes(nstm, out len, buf); int n = Entry.BytesToInt(buf); int count = 0; if (null != rem) { List <long> appendsizes = new List <long>(); try { count = rem.GetOutputFileCount(n, appendsizes); } catch (Exception e) { throw new DistributedObjectsSlave.DistObjectAbortException(e); } if (buf.Length < 4 + 8 * appendsizes.Count) { buf = new byte[Entry.Round2Power(4 + 8 * appendsizes.Count)]; } Entry.ToBytes(count, buf, 0); int offset = 4; for (int i = 0; i < appendsizes.Count; i++, offset += 8) { Entry.LongToBytes(appendsizes[i], buf, offset); } XContent.SendXContent(nstm, buf, 4 + 8 * appendsizes.Count); break; // ! } Entry.ToBytes(count, buf, 0); XContent.SendXContent(nstm, buf, 4); } break; default: base.ProcessCommand(nstm, tag); break; } }
public virtual void Open() { if (didopen) { throw new Exception("Attempted to Open after already Open"); } didopen = true; string portfilename = null; System.IO.FileStream portfile = null; try { Socket lsock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); IPEndPoint lipep; //int lptests = 0; for (; ;) { // Try a few ports... lipep = new IPEndPoint(IPAddress.Any, GetSlavePort()); try { System.Threading.Mutex lm = new System.Threading.Mutex(false, "DODLL_Listen{2FDD7284-14A8-4f4d-8448-0CE229C1E596}"); lm.WaitOne(); try { string spf = lipep.Port.ToString() + ".pf"; if (System.IO.File.Exists(spf)) { continue; } lsock.Bind(lipep); lsock.Listen(2); portfile = new System.IO.FileStream(spf, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.Delete, 8, System.IO.FileOptions.DeleteOnClose); portfilename = spf; } finally { lm.ReleaseMutex(); } } catch (SocketException e) { //if (++lptests < 4) { continue; } //throw new Exception(e.ToString() + " [Note: ensure port " + lipep.Port.ToString() + " is free / not blocked (ports >= " + SlavePortMin.ToString() + ")]"); } break; } #if DEBUG this.OpenPort = lipep.Port; lock (AllOpenPorts) { AllOpenPorts.Add(this.OpenPort); } #endif foreach (SlaveInfo slave in dslaves) { NetworkStream servStm = null; #if DEBUGfailtest bool failtest = true; #else const bool failtest = false; #endif for (; ;) { Socket servSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); servSock.Connect(slave.blockinfo[3], 55900); servStm = new XNetworkStream(servSock); servStm.WriteByte((byte)'B'); // AddBlock. XContent.SendXContent(servStm, slave.sblockinfo); XContent.SendXContent(servStm, lipep.Port.ToString()); XContent.SendXContent(servStm, sjid + ":" + Convert.ToBase64String(Encoding.UTF8.GetBytes(jobdesc))); { int rB = servStm.ReadByte(); if ('+' != rB) { if ('_' == rB) { throw new Exception("Error returned from AddBlock: " + XContent.ReceiveXString(servStm, null)); } else { throw new Exception("AddBlock failure"); } } } // 30000000 microseconds == 30 seconds if (failtest || !lsock.Poll(30000000, SelectMode.SelectRead)) { #if DEBUGfailtest failtest = false; #endif servStm.Close(); continue; } Socket slaveSock = lsock.Accept(); slave.nstm = new XNetworkStream(slaveSock); break; } if (1 != slave.nstm.ReadByte()) { throw new Exception("Sub process connection error: invalid handshake [Non-sub-process connection?]"); } { int len; byte[] bpid = XContent.ReceiveXBytes(slave.nstm, out len, null); if (len < 4) { throw new Exception("Sub process connection error: invalid SlavePID handshake"); } slave.pid = BytesToInt(bpid); } servStm.WriteByte((byte)'\\'); servStm.Close(1000); } lsock.Close(); } catch (FormatException e) { throw new FormatException("Format error in Open: " + e.ToString()); } catch (Exception e) { throw new Exception("Error in Open: " + e.ToString() + " [Note: ensure the Windows service is running]"); } finally { #if DEBUG if (0 != this.OpenPort) { lock (AllOpenPorts) { AllOpenPorts.Remove(this.OpenPort); } } #endif if (null != portfile) { System.Threading.Mutex lm = new System.Threading.Mutex(false, "DODLL_Listen{2FDD7284-14A8-4f4d-8448-0CE229C1E596}"); lm.WaitOne(); try { if (null != portfile) { portfile.Close(); } System.IO.File.Delete(portfilename); } catch { } finally { lm.ReleaseMutex(); } } } }
protected override void ProcessCommand(NetworkStream nstm, char tag) { //string s; int len; switch (tag) { case 'N': // Next in enumeration.. { int ienumid = nstm.ReadByte(); if (ienumid >= 0) { byte enumid = (byte)ienumid; if (enumid >= this.enums.Length) { nstm.WriteByte((byte)'-'); } else { if (null == this.enums[enumid]) { this.enums[enumid] = table.GetEnumerator(); } if (this.enums[enumid].MoveNext()) { nstm.WriteByte((byte)'+'); System.Collections.DictionaryEntry de = this.enums[enumid].Entry; XContent.SendXContent(nstm, (byte[])de.Key); if (AppendBufferingEnabled) { AppendBuffer abvalue = (AppendBuffer)de.Value; byte[] result = abvalue.Get(out len); XContent.SendXContent(nstm, result, len); } else { XContent.SendXContent(nstm, (byte[])de.Value); } } else { nstm.WriteByte((byte)'-'); } } } } break; case 'n': // Reset next in enumeration.. { int ienumid = nstm.ReadByte(); if (ienumid >= 0) { byte enumid = (byte)ienumid; if (XLog.logging) { XLog.log("Starting enumeration (enumid:" + enumid.ToString() + ")"); } if (enumid < this.enums.Length && null != this.enums[enumid]) { //this.enums[enumid].Reset(); this.enums[enumid] = null; } } } break; case 'b': // Enable AppendBuffer XContent.ReceiveXBytes(nstm, out len, buf); appendbuffersize = BytesToInt(buf); if (appendbuffersize < 0 || appendbuffersize >= 16777216) { throw new Exception("Invalid AppendBuffer preallocation size (appendbuffersize): " + appendbuffersize.ToString()); } if (XLog.logging) { XLog.log("Enabled AppendBuffer with initial value sizes of " + appendbuffersize.ToString()); } break; case 'L': // Set length.. { // Key.. byte[] putkey = XContent.ReceiveXBytes(nstm, out len, buf); putkey = GetSliceCopy(putkey, len); // ! // New length.. byte[] putlength = XContent.ReceiveXBytes(nstm, out len, buf); int newlength = BytesToInt(putlength); if (AppendBufferingEnabled) { object ovalue; if (UseStringKeys) { ovalue = table[Encoding.ASCII.GetString(putkey)]; } else { ovalue = table[putkey]; } if (null == ovalue) { int setbufsize = newlength; if (setbufsize < appendbuffersize) { setbufsize = appendbuffersize; // ? } if (UseStringKeys) { table[Encoding.ASCII.GetString(putkey)] = new AppendBuffer(setbufsize); } else { table[putkey] = new AppendBuffer(setbufsize); } } else { AppendBuffer abvalue = (AppendBuffer)ovalue; abvalue.SetLength(newlength); } } else { byte[] oldvalue = GetValue(putkey, out len); if (newlength < len) { CopyAndSetValue(putkey, oldvalue, newlength); } } } break; case 'G': // Get.. { //Thread.Sleep(15000); buf = XContent.ReceiveXBytes(nstm, out len, buf); byte[] getkey = GetSliceCopy(buf, len); byte[] result = GetValue(getkey, out len); if (null == result) { nstm.WriteByte((byte)'-'); // Key doesn't exist! } else { nstm.WriteByte((byte)'+'); // It exists! XContent.SendXContent(nstm, result, len); } } break; case 'l': // Get length.. { byte[] getkey = XContent.ReceiveXBytes(nstm, out len, buf); getkey = GetSliceCopy(getkey, len); byte[] value = GetValue(getkey, out len); if (null == value) { nstm.WriteByte((byte)'-'); // Key doesn't exist! } else { nstm.WriteByte((byte)'+'); // It exists! XContent.SendXContent(nstm, ToBytes(len)); } } break; case 'P': // Put.. { // Key.. buf = XContent.ReceiveXBytes(nstm, out len, buf); byte[] putkey = GetSliceCopy(buf, len); // Value.. buf = XContent.ReceiveXBytes(nstm, out len, buf); CopyAndSetValue(putkey, buf, len); } break; case 'a': // Math add.. { // Key.. buf = XContent.ReceiveXBytes(nstm, out len, buf); byte[] putkey = GetSliceCopy(buf, len); // ! // Operand.. buf = XContent.ReceiveXBytes(nstm, out len, buf); if (buf.Length >= 4) { int operand = BytesToInt(buf); // Do add... byte[] val = GetValue(putkey, out len); if (null == val || val.Length < 4) // Key doesn't exist, or too small, so assume it was 0. { ToBytes(operand, buf, 0); CopyAndSetValue(putkey, buf, 4); } else { int existingint = BytesToInt(val); int result = existingint + operand; //XLog.log("blarg", "adding " + existingint.ToString() + " and " + operand.ToString() + ", got result " + result.ToString()); ToBytes(result, buf, 0); CopyAndSetValue(putkey, buf, 4); } } } break; case 'A': // Append.. { // Key.. buf = XContent.ReceiveXBytes(nstm, out len, buf); byte[] putkey = GetSliceCopy(buf, len); // Value.. buf = XContent.ReceiveXBytes(nstm, out len, buf); AppendValue(putkey, buf, len); } break; default: base.ProcessCommand(nstm, tag); break; } }