//static bool isfirst = false;
// buf must be at least 12 bytes.
//--F-------------ADDCOOK------------------
public void CopyInto(List <LongIntComboListPart.B12> cbuffer, byte[] buf)
{
ensurefzblock(false, blockbuflen);
try
{
int flen = (int)fzblock.Length;
int b12count = flen / 12;
cbuffer.Clear();
int newcap = b12count;
/*if (isfirst)
* {
* isfirst = false;
* newcap *= 2;
* }*/
if (newcap > cbuffer.Capacity)
{
cbuffer.Capacity = newcap;
}
// Read from existing unsorted zblock file into buffer.
fzblock.Seek(0, System.IO.SeekOrigin.Begin);
LongIntComboListPart.B12 b12;
b12.A = 0;
b12.B = 0;
for (int i = 0; i != b12count; i++)
{
int szr = fzblock.Read(buf, 0, 12);
#if DEBUG
if (12 != szr)
{
throw new Exception("Unable to read");
}
#endif
b12.SetFromArray(buf);
cbuffer.Add(b12);
}
}
catch (Exception e)
{
cbuffer.Clear();
XLog.errorlog("LongIntComboList CopyInto failure; zblock skipped: " + e.ToString());
}
_justclose();
}
//static bool isfirst = false;
// buf must be at least 8 bytes.
//--D-------------ADDCOOK------------------
public void CopyInto(List <IntComboListPart.B8> cbuffer, byte[] buf)
{
ensurefzblock(false, blockbuflen);
try
{
int flen = (int)fzblock.Length;
int b8count = flen / 8;
cbuffer.Clear();
int newcap = b8count;
/*if (isfirst)
* {
* isfirst = false;
* newcap *= 2;
* }*/
if (newcap > cbuffer.Capacity)
{
cbuffer.Capacity = newcap;
}
// Read from existing unsorted zblock file into buffer.
fzblock.Seek(0, System.IO.SeekOrigin.Begin);
IntComboListPart.B8 b8;
b8.A = 0;
b8.B = 0;
for (int i = 0; i != b8count; i++)
{
fzblock.Read(buf, 0, 8);
b8.SetFromArray(buf);
cbuffer.Add(b8);
}
}
catch (Exception e)
{
cbuffer.Clear();
XLog.errorlog("IntComboList CopyInto failure; zblock skipped: " + e.ToString());
}
_justclose();
}
public void SetError(string msg)
{
lasterror = msg;
if (maxerrors > 0)
{
maxerrors--;
XLog.errorlog(msg);
try
{
XLog.errorlog(msg, XLog.UserLogFile, XLog.DelimitUserErrors);
}
catch
{
}
}
System.Threading.Thread.Sleep(200);
}
protected override void ProcessCommand(NetworkStream nstm, char tag)
{
//string s;
int len;
switch (tag)
{
case 'e': // Batch 'get next' enumeration.
{
try
{
int ienumid = nstm.ReadByte();
if (ienumid >= 0)
{
byte enumid = (byte)ienumid;
if (enumid >= this.enums.Length)
{
nstm.WriteByte((byte)'-');
}
else
{
if (null == this.enums[enumid])
{
this.enums[enumid] = new LongIntComboListPartEnumerator(this);
}
int offset = 0;
if (null == buf || buf.Length < 60)
{
throw new Exception("Enumeration batch buffer too small!");
}
//if (uniquecompression) // Compressed...
{
for (; ;)
{
if (!this.enums[enumid].MoveNext())
{
break;
}
B12 b12 = this.enums[enumid].Current;
// Using Big Endian!
b12.CopyToArray(buf, offset);
offset += 12;
if (offset + 12 > buf.Length)
{
break;
}
}
}
if (offset > 0)
{
nstm.WriteByte((byte)'+');
XContent.SendXContent(nstm, buf, offset);
}
else
{
nstm.WriteByte((byte)'-');
}
}
}
}
catch
{
nstm.WriteByte((byte)'-');
throw;
}
}
break;
case 'n': // Reset next in enumeration..
{
int ienumid = nstm.ReadByte();
if (ienumid >= 0)
{
byte enumid = (byte)ienumid;
if (XLog.logging)
{
XLog.log("Starting enumeration (enumid:" + enumid.ToString() + ")");
}
if (enumid < this.enums.Length &&
null != this.enums[enumid])
{
//this.enums[enumid].Reset();
this.enums[enumid] = null;
}
}
}
break;
case 's':
{
try
{
#if ENABLE_TIMING
long start = 0;
if (XLog.logging)
{
QueryPerformanceCounter(out start);
}
#endif
int readbuflen = 1048576;
if (null != DistributedObjectsSlave.xslave)
{
System.Xml.XmlNode xzblocks = DistributedObjectsSlave.xslave["zblocks"];
if (null != xzblocks)
{
{
System.Xml.XmlAttribute xzbs = xzblocks.Attributes["readbuffersize"];
if (null != xzbs)
{
readbuflen = DistributedObjectsSlave.ParseCapacity(xzbs.Value);
}
}
}
}
foreach (ZBlock zb in zblocks)
{
zb.LeaveAddMode(readbuflen);
}
foreach (ZBlock zb in zblocks)
{
zb.Sort(b12buffer, this.buf);
}
#if ENABLE_TIMING
if (XLog.logging)
{
long stop;
QueryPerformanceCounter(out stop);
long freq;
if (QueryPerformanceFrequency(out freq))
{
long secs = (stop - start) / freq;
if (secs > 10)
{
XLog.log("IntComboListPart sort seconds: " + secs.ToString());
}
}
}
#endif
}
finally
{
nstm.WriteByte((byte)'+');
}
}
break;
case 'p': // Batch push/publish...
{
buf = XContent.ReceiveXBytes(nstm, out len, buf);
uint gbfree = (uint)(GetCurrentDiskFreeBytes() / 1073741824);
#if DEBUG
{
string computer_name = System.Environment.GetEnvironmentVariable("COMPUTERNAME");
if (computer_name == "MAPDDRULE" || computer_name == "MAPDCMILLER" || computer_name == "computer_name")
{
gbfree = uint.MaxValue;
}
}
#endif
if (gbfree > 20)
{
int pcount = len / 12; // size of B12
int y = 0;
for (int i = 0; i != pcount; i++)
{
TimedAdd(buf, y);
y += 12;
}
}
else
{
if (!nofreedisklog)
{
nofreedisklog = true;
XLog.errorlog("Low free disk space; now dropping entries.");
}
}
}
break;
default:
base.ProcessCommand(nstm, tag);
break;
}
}
// args: <ipaddr> <portnum> <typechar> <capacity> <logfile> <jid>
static void Main(string[] args)
{
MySpace.DataMining.AELight.Surrogate.LogonMachines();
try
{
#if DEBUGnoisy
XLog.errorlog("DistributedObjectsSlave EntryPoint: " + Environment.CommandLine);
#endif
#if DEBUG
//System.Threading.Thread.Sleep(1000 * 8);
{
//string computer_name = System.Environment.GetEnvironmentVariable("COMPUTERNAME");
//if (computer_name == "MAPDDRULE" || computer_name == "MAPDCMILLER")
{
if (System.IO.File.Exists("sleep.txt"))
{
System.Threading.Thread.Sleep(1000 * 8);
int i32 = 1 + 32;
}
}
}
#endif
#if DEBUG
SetPriorityClass(GetCurrentProcess(), BELOW_NORMAL_PRIORITY_CLASS);
#endif
XLog.UserLogFile = args[4];
if (XLog.UserLogFile.StartsWith("&"))
{
XLog.UserLogFile = XLog.UserLogFile.Substring(1);
XLog.DelimitUserErrors = true;
}
#if SLAVE_TRACE
try
{
Thread mainthread = System.Threading.Thread.CurrentThread;
System.Threading.Thread stthd = new System.Threading.Thread(
new System.Threading.ThreadStart(
delegate
{
string spid = System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
try
{
string dotracefile = spid + ".trace";
const string tracefiledelim = "{C8683F6C-0655-42e7-ACD9-0DDED6509A7C}";
for (; ;)
{
System.IO.StreamWriter traceout = null;
for (System.Threading.Thread.Sleep(1000 * 60)
; !System.IO.File.Exists(dotracefile)
; System.Threading.Thread.Sleep(1000 * 60))
{
}
{
string[] tfc;
try
{
tfc = System.IO.File.ReadAllLines(dotracefile);
}
catch
{
continue;
}
if (tfc.Length < 1 || "." != tfc[tfc.Length - 1])
{
continue;
}
try
{
System.IO.File.Delete(dotracefile);
}
catch
{
continue;
}
if ("." != tfc[0])
{
string traceoutfp = tfc[0];
try
{
traceout = System.IO.File.CreateText(traceoutfp);
traceout.Write("BEGIN:");
traceout.WriteLine(tracefiledelim);
}
catch
{
continue;
}
}
}
if (null == traceout)
{
XLog.log("SLAVE_TRACE: " + spid + " Start");
}
for (; ; System.Threading.Thread.Sleep(1000 * 60))
{
{
try
{
#if SLAVE_TRACE_PORT
if (null == traceout)
{
try
{
StringBuilder sbtp = new StringBuilder();
sbtp.Append("SLAVE_TRACE_PORT: " + spid + ":");
for (int i = 0; i < DOSlave_TracePorts.Count; i++)
{
sbtp.Append(' ');
sbtp.Append(DOSlave_TracePorts[i]);
}
if (0 == DOSlave_TracePorts.Count)
{
sbtp.Append(" None");
}
XLog.log(sbtp.ToString());
}
catch
{
}
}
#endif
bool thdsuspended = false;
try
{
mainthread.Suspend();
thdsuspended = true;
}
catch (System.Threading.ThreadStateException)
{
}
try
{
System.Diagnostics.StackTrace st = new System.Diagnostics.StackTrace(mainthread, false);
StringBuilder sbst = new StringBuilder();
const int maxframesprint = 15;
for (int i = 0, imax = Math.Min(maxframesprint, st.FrameCount); i < imax; i++)
{
if (0 != sbst.Length)
{
sbst.Append(", ");
}
string mn = "N/A";
try
{
System.Reflection.MethodBase mb = st.GetFrame(i).GetMethod();
mn = mb.ReflectedType.Name + "." + mb.Name;
}
catch
{
}
sbst.Append(mn);
}
if (st.FrameCount > maxframesprint)
{
sbst.Append(" ... ");
sbst.Append(st.FrameCount - maxframesprint);
sbst.Append(" more");
}
if (null == traceout)
{
XLog.log("SLAVE_TRACE: " + spid + " Trace: " + sbst.ToString());
}
else
{
traceout.WriteLine(sbst.ToString());
}
}
catch (Exception e)
{
XLog.log("SLAVE_TRACE: " + spid + " Error: " + e.ToString());
}
finally
{
if (thdsuspended)
{
mainthread.Resume();
}
}
}
catch (Exception e)
{
XLog.log("SLAVE_TRACE: " + spid + " " + mainthread.Name + " Trace Error: Cannot access thread: " + e.ToString());
}
}
if (null != traceout)
{
traceout.Write(tracefiledelim);
traceout.WriteLine(":END");
traceout.Close();
break;
}
}
}
}
catch (Exception e)
{
XLog.log("SLAVE_TRACE: " + spid + " Trace Failure: " + e.Message);
}
}));
stthd.IsBackground = true;
stthd.Start();
}
catch (Exception est)
{
XLog.log("SLAVE_TRACE: Thread start error: " + est.ToString());
}
#endif
sjid = args[5];
{
int ic = sjid.IndexOf(':');
if (-1 != ic)
{
jobdesc = Encoding.UTF8.GetString(Convert.FromBase64String(sjid.Substring(ic + 1)));
sjid = sjid.Substring(0, ic);
}
}
jid = long.Parse(sjid);
try
{
Environment.SetEnvironmentVariable("DOSLAVE", "DO5");
}
catch
{
}
System.Xml.XmlDocument xd = new System.Xml.XmlDocument();
#if DEBUGslaveconfigload
Random _scsr = new Random(DateTime.Now.Millisecond / 2 + System.Threading.Thread.CurrentThread.ManagedThreadId / 2);
//for (int i = 0; i < 50; i++)
{
System.Threading.Thread.Sleep(_scsr.Next(50, 200));
try
{
xd.Load("slaveconfig.j" + sjid + ".xml");
InitXmlConfig(xd);
}
catch (System.IO.FileNotFoundException e)
{
//System.Diagnostics.Debugger.Launch();
}
catch (Exception e)
{
System.Diagnostics.Debugger.Launch();
throw;
}
}
#else
try
{
xd.Load("slaveconfig.j" + sjid + ".xml");
InitXmlConfig(xd);
}
catch (System.IO.FileNotFoundException e)
{
}
#endif
if (XLog.logging)
{
XLog.log("New Sub Process: '" + args[0] + "' '" + args[1] + "' '" + args[2] + "' '" + args[3] + "' '" + args[4] + "' '" + args[5] + "'");
}
Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
string host = args[0];
int portnum = ushort.Parse(args[1]);
#if SLAVE_TRACE_PORT
DOSlave_TracePorts.Add(portnum);
#endif
char typechar = args[2][0];
try
{
Thread.Sleep(100); // Give DLL a chance to start accepting.
sock.Connect(host, portnum);
}
catch (Exception e)
{
throw new Exception("Sub-process-to-client(dll) connection error: " + e.ToString() + " [Note: ensure port " + portnum.ToString() + " is free / not blocked]");
}
IDistObject dop;
if ('H' == typechar)
{
int capacity = ParseCapacity(args[3]);
if (XLog.logging)
{
XLog.log("Connected; creating Hashtable with capacity " + args[3]);
}
dop = HashtableObjectPart.Create(capacity);
}
else if ('8' == typechar)
{
int capacity = ParseCapacity(args[3]);
if (XLog.logging)
{
XLog.log("Connected; creating IntComboList with capacity " + args[3]);
}
dop = IntComboListPart.Create(capacity);
}
else if ('C' == typechar) // 12 hex
{
int capacity = ParseCapacity(args[3]);
if (XLog.logging)
{
XLog.log("Connected; creating LongIntComboList with capacity " + args[3]);
}
dop = LongIntComboListPart.Create(capacity);
}
else if ('A' == typechar)
{
//int capacity = ParseCapacity(args[3]);
string[] a = args[3].Split(';');
int count_capacity = ParseCapacity(a[0]);
int estimated_row_capacity = ParseCapacity(a[1]);
int keylen = int.Parse(a[2]);
if (XLog.logging)
{
XLog.log("Connected; creating ArrayComboList(keylength=" + keylen.ToString() + ") with capacity " + args[3]);
}
dop = ArrayComboListPart.Create(count_capacity, estimated_row_capacity, keylen);
}
else if ('R' == typechar)
{
if (XLog.logging)
{
XLog.log("Connected; creating Remote");
}
dop = RemotePart.Create();
}
else if ('F' == typechar)
{
//int capacity = ParseCapacity(args[3]);
string[] a = args[3].Split(';');
int keylen = ParseCapacity(a[2]);
int valuelen = ParseCapacity(a[3]);
if (XLog.logging)
{
XLog.log("Connected; creating FixedArrayComboList(keylength=" + keylen.ToString() + ";valuelength=" + valuelen.ToString() + ")");
}
dop = FixedArrayComboListPart.Create(keylen, valuelen);
}
else
{
throw new Exception("Data type not supported");
}
NetworkStream nstm = new XNetworkStream(sock);
{
// New handshake with client(dll)...
nstm.WriteByte(1);
// ... and the SlavePID:
int SlavePID = System.Diagnostics.Process.GetCurrentProcess().Id;
byte[] bpid = DistObjectBase.ToBytes(SlavePID);
XContent.SendXContent(nstm, bpid);
}
{
int pid = System.Diagnostics.Process.GetCurrentProcess().Id;
string spid = pid.ToString();
string pidfilename = spid + ".j" + sjid + ".slave.pid";
StreamWriter pidfile = new StreamWriter(pidfilename);
pidfile.WriteLine(pid);
pidfile.WriteLine(System.DateTime.Now);
pidfile.WriteLine("jid={0}", sjid);
pidfile.Flush();
for (; ;)
{
try
{
dop.ProcessCommands(nstm);
break;
}
catch (DistObjectAbortException e)
{
XLog.errorlog("ProcessCommands exception: " + e.ToString() + " ((DistObjectAbortException))");
break; // !
}
catch (Exception e)
{
XLog.errorlog("ProcessCommands exception: " + e.ToString() + " ((recovering from this exception))");
}
}
pidfile.Close();
try
{
System.IO.File.Delete(pidfilename);
}
catch
{
}
}
}
catch (Exception e)
{
XLog.errorlog("Main exception: " + e.ToString());
}
}
public virtual void ProcessCommands(NetworkStream nstm)
{
//nstm = new XNetworkStream(sock);
int x;
this.nstm = nstm;
try
{
for (; ;)
{
x = nstm.ReadByte();
if (x < 0)
{
//throw new Exception("Unable to read tag for command");
break;
}
buf[0] = (byte)x;
CurrentCommand = (char)x;
if ('\\' == (char)buf[0])
{
if (XLog.logging)
{
XLog.log("Received close command from service; closing");
}
break;
}
else if ('<' == (char)buf[0])
{
try
{
string xml = XContent.ReceiveXString(nstm, buf);
System.Xml.XmlDocument xd = new System.Xml.XmlDocument();
xd.LoadXml(xml);
DistributedObjectsSlave.InitXmlConfig(xd);
ReloadConfig();
}
catch
{
nstm.WriteByte((byte)'-');
throw;
}
nstm.WriteByte((byte)'+');
continue;
}
else if ('?' == (char)buf[0])
{
if (null != lasterror)
{
string lerr = lasterror;
lasterror = null; // Also resets error!
XContent.SendXContent(nstm, lerr);
}
else
{
XContent.SendXContent(nstm, buf, 0); // No error!
}
continue;
}
/*
* if (XLog.logging)
* {
* XLog.log("Slave processing command " + ((char)buf[0]).ToString());
* }
* */
ProcessCommand(nstm, (char)buf[0]);
}
}
catch (System.IO.IOException ioex)
{
// Drop SocketException (shutdown during tasks), rethrow others.
if ((ioex.InnerException as SocketException) == null)
{
throw;
}
#if DEBUG
XLog.errorlog("DistributedObjectsSlave Warning: IOException+SocketException during task shutdown: " + ioex.ToString());
#endif
}
nstm.Close();
nstm.Dispose();
}
internal void ThreadProc()
{
try
{
Random rnd = new Random(unchecked (System.DateTime.Now.Millisecond + System.Threading.Thread.CurrentThread.ManagedThreadId));
List <PullFileInfo> infos = new List <PullFileInfo>(stop - start);
for (int i = start; i < stop; i++)
{
PullFileInfo info;
info.PullFile = allpullfiles[i];
string destfn;
{
int ix;
ix = info.PullFile.LastIndexOf('\u0002');
if (-1 != ix)
{
destfn = info.PullFile.Substring(ix + 1);
info.PullFile = info.PullFile.Substring(0, ix);
}
else
{
ix = info.PullFile.LastIndexOf('\\');
if (-1 != ix)
{
destfn = info.PullFile.Substring(ix + 1);
}
else
{
destfn = info.PullFile;
}
}
}
info.DestFile = destfn;
info.DestFileStream = null;
info.Position = 0;
info.Retries = 0;
infos.Add(info);
}
byte[] fbuf = new byte[MAX_SIZE_PER_RECEIVE];
for (int i = 0; i < infos.Count; i++)
{
PullFileInfo info = infos[i];
for (; ;) //cooking
{
bool cooking_is_read = false;
try
{
if (info.DestFileStream == null)
{
if (info.Retries == 0)
{
cooking_is_read = true;
#if TESTFAULTTOLERANT
{
int dgdel = info.PullFile.IndexOf(@"\", 2);
string dghost = info.PullFile.Substring(2, dgdel - 2).ToLower();
fttest.BreakPoint(dghost, "replication", new Exception("FAULTTOLERANT replication test exception"));
}
#endif
System.IO.File.Copy(info.PullFile, info.DestFile, true);
cooking_is_read = false;
break; // done with this pullfile
}
else
{
info.DestFileStream = new System.IO.FileStream(info.DestFile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.Read, FILE_BUFFER_SIZE);
}
}
if (info.DestFileStream != null)
{
cooking_is_read = true;
using (System.IO.FileStream pullfs = new System.IO.FileStream(info.PullFile, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, FILE_BUFFER_SIZE))
{
#if TESTFAULTTOLERANT
{
int dgdel = info.PullFile.IndexOf(@"\", 2);
string dghost = info.PullFile.Substring(2, dgdel - 2).ToLower();
fttest.BreakPoint(dghost, "replication", new Exception("FAULTTOLERANT replication test exception"));
}
#endif
if (info.Position != 0)
{
pullfs.Position = info.Position;
}
for (; ;)
{
//----------------------------COOKING--------------------------------
cooking_is_read = true;
//----------------------------COOKING--------------------------------
int xread = pullfs.Read(fbuf, 0, MAX_SIZE_PER_RECEIVE);
info.Position += xread;
//----------------------------COOKING--------------------------------
cooking_is_read = false;
//----------------------------COOKING--------------------------------
if (xread <= 0)
{
break;
}
info.DestFileStream.Write(fbuf, 0, xread);
}
break;
}
}
}
catch (Exception e)
{
if (!cooking_is_read)
{
if (info.DestFileStream != null)
{
info.DestFileStream.Close();
}
try
{
XLog.errorlog("Non-reading error; retries=" + info.Retries.ToString() +
";puller machine=" + System.Net.Dns.GetHostName() +
";pulling file=" + info.PullFile +
";error=" + e.ToString());
}
catch
{
}
throw;
}
if (++info.Retries > ftretries)
{
if (info.DestFileStream != null)
{
info.DestFileStream.Close();
}
//Report this host with corrupt file
int del = info.PullFile.IndexOf(@"\", 2);
string badhost = info.PullFile.Substring(2, del - 2);
RogueHosts.Add(jid, badhost);
lock (tattledhosts)
{
tattledhosts[badhost] = 1;
}
try
{
XLog.errorlog("Retries exceeded ftretries; retries=" + info.Retries.ToString() +
";ftretries=" + ftretries.ToString() +
";puller machine=" + System.Net.Dns.GetHostName() +
";pulling file=" + info.PullFile +
";tattled host=" + badhost);
}
catch
{
}
goto next_pullfile;
}
//Random swap with the rest of the list of pullfiles
int iswapwith = rnd.Next(i, infos.Count);
infos[i] = infos[iswapwith];
infos[iswapwith] = info;
info = infos[i];
continue;
}
}
next_pullfile:
int xxx = 10;
}
}
catch (Exception ex)
{
exception = ex;
}
}