// Note: appendsizes can be null, but the return will still be valid.
// Sizes exclude header data.
protected virtual int GetNumberOfRemoteOutputFilesCreated(int n, IList <long> appendsizes)
{
int result = 0;
foreach (SlaveInfo slave in dslaves)
{
slave.nstm.WriteByte((byte)'r');
Entry.ToBytes(n, buf, 0);
XContent.SendXContent(slave.nstm, buf, 4);
int len;
buf = XContent.ReceiveXBytes(slave.nstm, out len, buf);
if (len >= 4)
{
result += Entry.BytesToInt(buf);
if (null != appendsizes)
{
for (int offset = 4; offset + 8 <= len; offset += 8)
{
appendsizes.Add(Entry.BytesToLong(buf, offset));
}
}
}
}
return(result);
}
internal bool cangetnext()
{
lock (slave)
{
if (first)
{
first = false;
internalsendreset();
}
// 'N' for next..
slave.nstm.WriteByte((byte)'N');
slave.nstm.WriteByte(enumid);
// Replies with tag '+' and xcontent key and xcontent value; or tag '-' if not exists.
int len;
int x = slave.nstm.ReadByte();
if ('+' == x)
{
byte[] key = XContent.ReceiveXBytes(slave.nstm, out len, null); // Null buffer for copy.
byte[] value = XContent.ReceiveXBytes(slave.nstm, out len, null); // Null buffer for copy.
cur = new System.Collections.DictionaryEntry(key, value);
return(true);
}
else if ('-' == x)
{
return(false);
}
else
{
throw new Exception("Server returned invalid response for enumeration");
}
}
}
protected virtual void GetDGlobals()
{
foreach (SlaveInfo slave in dslaves)
{
slave.nstm.WriteByte((byte)'O');
int len = 0;
buf = XContent.ReceiveXBytes(slave.nstm, out len, buf);
DGlobalsM.FromBytes(buf, len, true);
}
}
protected string GetLastErrorReset()
{
#if SLAVE_GET_LAST_ERROR
nstm.WriteByte((byte)'?');
int len;
errbuf = XContent.ReceiveXBytes(nstm, out len, errbuf);
if (0 != len)
{
return(Encoding.UTF8.GetString(errbuf, 0, len));
}
#endif
return(null); // No error.
}
public int nextbufend = 0; // Where in nextbuf the valid elements end.
internal bool cangetnext()
{
lock (slave)
{
if (first)
{
first = false;
internalsendreset();
}
if (nextbufpos < nextbufend)
{
fillcurfromnextbuf();
return(true);
}
else
{
// 'e for batch next..
slave.nstm.WriteByte((byte)'e');
// Replies with tag '+' and xcontent values; or tag '-' if not exists.
//int len;
int x = slave.nstm.ReadByte();
if ('+' == x)
{
nextbufpos = 0;
nextbufend = 0;
nextbuf = XContent.ReceiveXBytes(slave.nstm, out nextbufend, nextbuf);
slave.SlaveErrorCheck();
fillcurfromnextbuf();
return(true);
}
else if ('-' == x)
{
return(false);
}
else
{
throw new Exception("Server returned invalid response for enumeration");
}
}
}
}
// Returns -1 if no such key.
public int Length(byte[] key)
{
if (!didopen)
{
throw new Exception("Must Open before writing to Hashtable");
}
int slaveID = DetermineSlave(key);
if (slaveID < 0 || slaveID >= dslaves.Count)
{
throw new Exception("Sub process missing: subProcessID needed: " + slaveID.ToString());
}
SlaveInfo slave = dslaves[slaveID];
lock (slave)
{
// 'l' for get length
slave.nstm.WriteByte((byte)'l');
XContent.SendXContent(slave.nstm, key);
// Replies with tag '+' and xcontent length; or tag '-' if not exists.
byte[] result = null;
int len;
int x = slave.nstm.ReadByte();
if ('+' == x)
{
result = XContent.ReceiveXBytes(slave.nstm, out len, buf);
return(DistObjectBase.BytesToInt(result)); // Only reads first one.
}
else if ('-' == x)
{
//result = null;
return(-1);
}
else
{
throw new Exception("Server returned invalid response for l (Length)");
}
}
}
// Gives null if no such key.
public byte[] Get(byte[] key)
{
if (!didopen)
{
throw new Exception("Must Open before reading from Hashtable");
}
int slaveID = DetermineSlave(key);
if (slaveID < 0 || slaveID >= dslaves.Count)
{
throw new Exception("Sub process missing: subProcessID needed: " + slaveID.ToString());
}
SlaveInfo slave = dslaves[slaveID];
lock (slave)
{
// 'G' for get
slave.nstm.WriteByte((byte)'G');
XContent.SendXContent(slave.nstm, key);
// Replies with tag '+' and xcontent value; or tag '-' if not exists.
byte[] result = null;
int len;
int x = slave.nstm.ReadByte();
if ('+' == x)
{
result = XContent.ReceiveXBytes(slave.nstm, out len, null); // Null buffer for copy.
}
else if ('-' == x)
{
//result = null;
}
else
{
throw new Exception("Server returned invalid response for G (get)");
}
return(result);
}
}
protected override void ProcessCommand(NetworkStream nstm, char tag)
{
//string s;
int len;
switch (tag)
{
case 'e': // Batch 'get next' enumeration.
{
try
{
int ienumid = nstm.ReadByte();
if (ienumid >= 0)
{
byte enumid = (byte)ienumid;
if (enumid >= this.enums.Length)
{
nstm.WriteByte((byte)'-');
}
else
{
if (null == this.enums[enumid])
{
this.enums[enumid] = new LongIntComboListPartEnumerator(this);
}
int offset = 0;
if (null == buf || buf.Length < 60)
{
throw new Exception("Enumeration batch buffer too small!");
}
//if (uniquecompression) // Compressed...
{
for (; ;)
{
if (!this.enums[enumid].MoveNext())
{
break;
}
B12 b12 = this.enums[enumid].Current;
// Using Big Endian!
b12.CopyToArray(buf, offset);
offset += 12;
if (offset + 12 > buf.Length)
{
break;
}
}
}
if (offset > 0)
{
nstm.WriteByte((byte)'+');
XContent.SendXContent(nstm, buf, offset);
}
else
{
nstm.WriteByte((byte)'-');
}
}
}
}
catch
{
nstm.WriteByte((byte)'-');
throw;
}
}
break;
case 'n': // Reset next in enumeration..
{
int ienumid = nstm.ReadByte();
if (ienumid >= 0)
{
byte enumid = (byte)ienumid;
if (XLog.logging)
{
XLog.log("Starting enumeration (enumid:" + enumid.ToString() + ")");
}
if (enumid < this.enums.Length &&
null != this.enums[enumid])
{
//this.enums[enumid].Reset();
this.enums[enumid] = null;
}
}
}
break;
case 's':
{
try
{
#if ENABLE_TIMING
long start = 0;
if (XLog.logging)
{
QueryPerformanceCounter(out start);
}
#endif
int readbuflen = 1048576;
if (null != DistributedObjectsSlave.xslave)
{
System.Xml.XmlNode xzblocks = DistributedObjectsSlave.xslave["zblocks"];
if (null != xzblocks)
{
{
System.Xml.XmlAttribute xzbs = xzblocks.Attributes["readbuffersize"];
if (null != xzbs)
{
readbuflen = DistributedObjectsSlave.ParseCapacity(xzbs.Value);
}
}
}
}
foreach (ZBlock zb in zblocks)
{
zb.LeaveAddMode(readbuflen);
}
foreach (ZBlock zb in zblocks)
{
zb.Sort(b12buffer, this.buf);
}
#if ENABLE_TIMING
if (XLog.logging)
{
long stop;
QueryPerformanceCounter(out stop);
long freq;
if (QueryPerformanceFrequency(out freq))
{
long secs = (stop - start) / freq;
if (secs > 10)
{
XLog.log("IntComboListPart sort seconds: " + secs.ToString());
}
}
}
#endif
}
finally
{
nstm.WriteByte((byte)'+');
}
}
break;
case 'p': // Batch push/publish...
{
buf = XContent.ReceiveXBytes(nstm, out len, buf);
uint gbfree = (uint)(GetCurrentDiskFreeBytes() / 1073741824);
#if DEBUG
{
string computer_name = System.Environment.GetEnvironmentVariable("COMPUTERNAME");
if (computer_name == "MAPDDRULE" || computer_name == "MAPDCMILLER" || computer_name == "computer_name")
{
gbfree = uint.MaxValue;
}
}
#endif
if (gbfree > 20)
{
int pcount = len / 12; // size of B12
int y = 0;
for (int i = 0; i != pcount; i++)
{
TimedAdd(buf, y);
y += 12;
}
}
else
{
if (!nofreedisklog)
{
nofreedisklog = true;
XLog.errorlog("Low free disk space; now dropping entries.");
}
}
}
break;
default:
base.ProcessCommand(nstm, tag);
break;
}
}
protected override void ProcessCommand(System.Net.Sockets.NetworkStream nstm, char tag)
{
int len;
switch (tag)
{
case 'R': // Remote!
{
string classname = XContent.ReceiveXString(nstm, buf);
string xlibfn = CreateXlibFileName("remote");
{
buf = XContent.ReceiveXBytes(nstm, out len, buf);
if (0 != len)
{
System.IO.FileStream stm = System.IO.File.Create(xlibfn);
stm.Write(buf, 0, len);
stm.Close();
}
}
string dllfn = CreateDllFileName("remote");
{
buf = XContent.ReceiveXBytes(nstm, out len, buf);
System.IO.FileStream stm = System.IO.File.Create(dllfn);
stm.Write(buf, 0, len);
stm.Close();
}
if (XLog.logging)
{
string xclassname = classname;
if (null == xclassname)
{
xclassname = "<null>";
}
XLog.log("Loading IRemote plugin named " + xclassname + " for remote: " + dllfn);
}
rem = LoadRemotePlugin(dllfn, classname);
#if DEBUG
try
{
rem.OnRemote();
}
catch (Exception e)
{
throw new UserException(e);
}
#else
rem.OnRemote();
#endif
}
break;
case 'O': //Query DGlobals
{
int byteCount = DGlobalsM.ToBytes(ref buf);
XContent.SendXContent(nstm, buf, byteCount);
}
break;
case 'r':
{
buf = XContent.ReceiveXBytes(nstm, out len, buf);
int n = Entry.BytesToInt(buf);
int count = 0;
if (null != rem)
{
List <long> appendsizes = new List <long>();
try
{
count = rem.GetOutputFileCount(n, appendsizes);
}
catch (Exception e)
{
throw new DistributedObjectsSlave.DistObjectAbortException(e);
}
if (buf.Length < 4 + 8 * appendsizes.Count)
{
buf = new byte[Entry.Round2Power(4 + 8 * appendsizes.Count)];
}
Entry.ToBytes(count, buf, 0);
int offset = 4;
for (int i = 0; i < appendsizes.Count; i++, offset += 8)
{
Entry.LongToBytes(appendsizes[i], buf, offset);
}
XContent.SendXContent(nstm, buf, 4 + 8 * appendsizes.Count);
break; // !
}
Entry.ToBytes(count, buf, 0);
XContent.SendXContent(nstm, buf, 4);
}
break;
default:
base.ProcessCommand(nstm, tag);
break;
}
}
public virtual void Open()
{
if (didopen)
{
throw new Exception("Attempted to Open after already Open");
}
didopen = true;
string portfilename = null;
System.IO.FileStream portfile = null;
try
{
Socket lsock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
IPEndPoint lipep;
//int lptests = 0;
for (; ;)
{
// Try a few ports...
lipep = new IPEndPoint(IPAddress.Any, GetSlavePort());
try
{
System.Threading.Mutex lm = new System.Threading.Mutex(false, "DODLL_Listen{2FDD7284-14A8-4f4d-8448-0CE229C1E596}");
lm.WaitOne();
try
{
string spf = lipep.Port.ToString() + ".pf";
if (System.IO.File.Exists(spf))
{
continue;
}
lsock.Bind(lipep);
lsock.Listen(2);
portfile = new System.IO.FileStream(spf, System.IO.FileMode.Create, System.IO.FileAccess.Write,
System.IO.FileShare.Delete, 8, System.IO.FileOptions.DeleteOnClose);
portfilename = spf;
}
finally
{
lm.ReleaseMutex();
}
}
catch (SocketException e)
{
//if (++lptests < 4)
{
continue;
}
//throw new Exception(e.ToString() + " [Note: ensure port " + lipep.Port.ToString() + " is free / not blocked (ports >= " + SlavePortMin.ToString() + ")]");
}
break;
}
#if DEBUG
this.OpenPort = lipep.Port;
lock (AllOpenPorts)
{
AllOpenPorts.Add(this.OpenPort);
}
#endif
foreach (SlaveInfo slave in dslaves)
{
NetworkStream servStm = null;
#if DEBUGfailtest
bool failtest = true;
#else
const bool failtest = false;
#endif
for (; ;)
{
Socket servSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
servSock.Connect(slave.blockinfo[3], 55900);
servStm = new XNetworkStream(servSock);
servStm.WriteByte((byte)'B'); // AddBlock.
XContent.SendXContent(servStm, slave.sblockinfo);
XContent.SendXContent(servStm, lipep.Port.ToString());
XContent.SendXContent(servStm, sjid + ":" + Convert.ToBase64String(Encoding.UTF8.GetBytes(jobdesc)));
{
int rB = servStm.ReadByte();
if ('+' != rB)
{
if ('_' == rB)
{
throw new Exception("Error returned from AddBlock: " + XContent.ReceiveXString(servStm, null));
}
else
{
throw new Exception("AddBlock failure");
}
}
}
// 30000000 microseconds == 30 seconds
if (failtest || !lsock.Poll(30000000, SelectMode.SelectRead))
{
#if DEBUGfailtest
failtest = false;
#endif
servStm.Close();
continue;
}
Socket slaveSock = lsock.Accept();
slave.nstm = new XNetworkStream(slaveSock);
break;
}
if (1 != slave.nstm.ReadByte())
{
throw new Exception("Sub process connection error: invalid handshake [Non-sub-process connection?]");
}
{
int len;
byte[] bpid = XContent.ReceiveXBytes(slave.nstm, out len, null);
if (len < 4)
{
throw new Exception("Sub process connection error: invalid SlavePID handshake");
}
slave.pid = BytesToInt(bpid);
}
servStm.WriteByte((byte)'\\');
servStm.Close(1000);
}
lsock.Close();
}
catch (FormatException e)
{
throw new FormatException("Format error in Open: " + e.ToString());
}
catch (Exception e)
{
throw new Exception("Error in Open: " + e.ToString() + " [Note: ensure the Windows service is running]");
}
finally
{
#if DEBUG
if (0 != this.OpenPort)
{
lock (AllOpenPorts)
{
AllOpenPorts.Remove(this.OpenPort);
}
}
#endif
if (null != portfile)
{
System.Threading.Mutex lm = new System.Threading.Mutex(false, "DODLL_Listen{2FDD7284-14A8-4f4d-8448-0CE229C1E596}");
lm.WaitOne();
try
{
if (null != portfile)
{
portfile.Close();
}
System.IO.File.Delete(portfilename);
}
catch
{
}
finally
{
lm.ReleaseMutex();
}
}
}
}
protected override void ProcessCommand(NetworkStream nstm, char tag)
{
//string s;
int len;
switch (tag)
{
case 'N': // Next in enumeration..
{
int ienumid = nstm.ReadByte();
if (ienumid >= 0)
{
byte enumid = (byte)ienumid;
if (enumid >= this.enums.Length)
{
nstm.WriteByte((byte)'-');
}
else
{
if (null == this.enums[enumid])
{
this.enums[enumid] = table.GetEnumerator();
}
if (this.enums[enumid].MoveNext())
{
nstm.WriteByte((byte)'+');
System.Collections.DictionaryEntry de = this.enums[enumid].Entry;
XContent.SendXContent(nstm, (byte[])de.Key);
if (AppendBufferingEnabled)
{
AppendBuffer abvalue = (AppendBuffer)de.Value;
byte[] result = abvalue.Get(out len);
XContent.SendXContent(nstm, result, len);
}
else
{
XContent.SendXContent(nstm, (byte[])de.Value);
}
}
else
{
nstm.WriteByte((byte)'-');
}
}
}
}
break;
case 'n': // Reset next in enumeration..
{
int ienumid = nstm.ReadByte();
if (ienumid >= 0)
{
byte enumid = (byte)ienumid;
if (XLog.logging)
{
XLog.log("Starting enumeration (enumid:" + enumid.ToString() + ")");
}
if (enumid < this.enums.Length &&
null != this.enums[enumid])
{
//this.enums[enumid].Reset();
this.enums[enumid] = null;
}
}
}
break;
case 'b': // Enable AppendBuffer
XContent.ReceiveXBytes(nstm, out len, buf);
appendbuffersize = BytesToInt(buf);
if (appendbuffersize < 0 || appendbuffersize >= 16777216)
{
throw new Exception("Invalid AppendBuffer preallocation size (appendbuffersize): " + appendbuffersize.ToString());
}
if (XLog.logging)
{
XLog.log("Enabled AppendBuffer with initial value sizes of " + appendbuffersize.ToString());
}
break;
case 'L': // Set length..
{
// Key..
byte[] putkey = XContent.ReceiveXBytes(nstm, out len, buf);
putkey = GetSliceCopy(putkey, len); // !
// New length..
byte[] putlength = XContent.ReceiveXBytes(nstm, out len, buf);
int newlength = BytesToInt(putlength);
if (AppendBufferingEnabled)
{
object ovalue;
if (UseStringKeys)
{
ovalue = table[Encoding.ASCII.GetString(putkey)];
}
else
{
ovalue = table[putkey];
}
if (null == ovalue)
{
int setbufsize = newlength;
if (setbufsize < appendbuffersize)
{
setbufsize = appendbuffersize; // ?
}
if (UseStringKeys)
{
table[Encoding.ASCII.GetString(putkey)] = new AppendBuffer(setbufsize);
}
else
{
table[putkey] = new AppendBuffer(setbufsize);
}
}
else
{
AppendBuffer abvalue = (AppendBuffer)ovalue;
abvalue.SetLength(newlength);
}
}
else
{
byte[] oldvalue = GetValue(putkey, out len);
if (newlength < len)
{
CopyAndSetValue(putkey, oldvalue, newlength);
}
}
}
break;
case 'G': // Get..
{
//Thread.Sleep(15000);
buf = XContent.ReceiveXBytes(nstm, out len, buf);
byte[] getkey = GetSliceCopy(buf, len);
byte[] result = GetValue(getkey, out len);
if (null == result)
{
nstm.WriteByte((byte)'-'); // Key doesn't exist!
}
else
{
nstm.WriteByte((byte)'+'); // It exists!
XContent.SendXContent(nstm, result, len);
}
}
break;
case 'l': // Get length..
{
byte[] getkey = XContent.ReceiveXBytes(nstm, out len, buf);
getkey = GetSliceCopy(getkey, len);
byte[] value = GetValue(getkey, out len);
if (null == value)
{
nstm.WriteByte((byte)'-'); // Key doesn't exist!
}
else
{
nstm.WriteByte((byte)'+'); // It exists!
XContent.SendXContent(nstm, ToBytes(len));
}
}
break;
case 'P': // Put..
{
// Key..
buf = XContent.ReceiveXBytes(nstm, out len, buf);
byte[] putkey = GetSliceCopy(buf, len);
// Value..
buf = XContent.ReceiveXBytes(nstm, out len, buf);
CopyAndSetValue(putkey, buf, len);
}
break;
case 'a': // Math add..
{
// Key..
buf = XContent.ReceiveXBytes(nstm, out len, buf);
byte[] putkey = GetSliceCopy(buf, len); // !
// Operand..
buf = XContent.ReceiveXBytes(nstm, out len, buf);
if (buf.Length >= 4)
{
int operand = BytesToInt(buf);
// Do add...
byte[] val = GetValue(putkey, out len);
if (null == val || val.Length < 4) // Key doesn't exist, or too small, so assume it was 0.
{
ToBytes(operand, buf, 0);
CopyAndSetValue(putkey, buf, 4);
}
else
{
int existingint = BytesToInt(val);
int result = existingint + operand;
//XLog.log("blarg", "adding " + existingint.ToString() + " and " + operand.ToString() + ", got result " + result.ToString());
ToBytes(result, buf, 0);
CopyAndSetValue(putkey, buf, 4);
}
}
}
break;
case 'A': // Append..
{
// Key..
buf = XContent.ReceiveXBytes(nstm, out len, buf);
byte[] putkey = GetSliceCopy(buf, len);
// Value..
buf = XContent.ReceiveXBytes(nstm, out len, buf);
AppendValue(putkey, buf, len);
}
break;
default:
base.ProcessCommand(nstm, tag);
break;
}
}
