internal void start()
{
kname = parent.kbasename + "_thread" + threadnum.ToString();
hmap = SlaveMemory.CreateFileMapping(SlaveMemory.INVALID_HANDLE_VALUE, IntPtr.Zero, SlaveMemory.PAGE_READWRITE, 0, 1 + 8 + 4 + parent.packetsize, kname + "_block" + parent.slavenum.ToString());
if (IntPtr.Zero == hmap)
{
throw new SlaveMemoryException("Unable to opeb shared memory communication buffer");
}
pview = SlaveMemory.MapViewOfFile(hmap, SlaveMemory.FILE_MAP_ALL_ACCESS, 0, 0, 0);
if (IntPtr.Zero == pview)
{
SlaveMemory.CloseHandle(hmap);
hmap = IntPtr.Zero;
throw new SlaveMemoryException("Unable to map shared memory communication buffer");
}
this.ewh = System.Threading.EventWaitHandle.OpenExisting(kname + "_event" + parent.slavenum.ToString());
if (parent.usereturnevent)
{
this.ewhreturn = System.Threading.EventWaitHandle.OpenExisting(kname + "_returnevent" + parent.slavenum.ToString());
}
this.thread = new System.Threading.Thread(new System.Threading.ThreadStart(slavethreadproc));
this.thread.Start();
}
void _clean()
{
if (IntPtr.Zero != hmap)
{
SlaveMemory.UnmapViewOfFile(pview);
SlaveMemory.CloseHandle(hmap);
hmap = IntPtr.Zero;
}
}
internal ThreadView(SlaveMemory parent, int threadnum, int maxpacketsize)
{
this.parent = parent;
this.threadnum = threadnum;
this.maxpacketsize = maxpacketsize;
this.kname = parent.kbasename + "_thread" + threadnum.ToString();
_batspill = new List <Batch>();
}
internal void slavethreadproc()
{
try
{
unsafe
{
byte *com = (byte *)pview;
LongByteArray block = parent.block;
long offset = 0;
int length = 0;
for (; ;)
{
if (!ewh.WaitOne(1000, true))
{
if (parent.mainproc.WaitForExit(1))
{
//throw new SlaveMemoryException("Slave process exiting because main process exited");
break;
}
continue;
}
byte action = com[0];
switch (action)
{
case 1: // Read
case 3: // Batch read
{
byte *comnext = com;
for (; ;)
{
offset = SlaveMemory.BytesToInt64(comnext, 1);
length = SlaveMemory.BytesToInt32(comnext, 1 + 8);
int pos = 1 + 8 + 4;
long stop = offset + length;
if (stop > block.LongLength)
{
throw new SlaveMemoryException("Read out of bounds (offset=" + offset.ToString() + ";length=" + length.ToString() + ")");
}
if (length > parent.packetsize)
{
throw new SlaveMemoryException("Read out of bounds (length>packetsize; length=" + length.ToString() + ")");
}
while (offset < stop)
{
comnext[pos++] = block[offset++];
}
if (3 == action)
{
comnext += 1 + 8 + 4 + length;
if (action == comnext[0])
{
continue;
}
}
break;
}
com[0] = 255; // Done!
if (parent.usereturnevent)
{
signalreturnevent();
}
}
break;
case 2: // Write
case 4: // Batch write
{
byte *comnext = com;
for (; ;)
{
offset = SlaveMemory.BytesToInt64(comnext, 1);
length = SlaveMemory.BytesToInt32(comnext, 1 + 8);
int pos = 1 + 8 + 4;
long stop = offset + length;
if (stop > block.LongLength)
{
throw new SlaveMemoryException("Write out of bounds (offset=" + offset.ToString() + ";length=" + length.ToString() + ")");
}
if (length > parent.packetsize)
{
throw new SlaveMemoryException("Write out of bounds (length>packetsize; length=" + length.ToString() + ")");
}
while (offset < stop)
{
block[offset++] = comnext[pos++];
}
if (4 == action)
{
comnext += 1 + 8 + 4 + length;
if (action == comnext[0])
{
continue;
}
}
break;
}
com[0] = 255; // Done!
if (parent.usereturnevent)
{
signalreturnevent();
}
}
break;
case 0: // Exit
com[0] = 255; // Done.
if (parent.usereturnevent)
{
signalreturnevent();
}
return; // !
}
}
}
}
catch (Exception e)
{
if (null != parent.slavethreadprocexception)
{
//e.InnerException = parent.slavethreadprocexception;
//parent.slavethreadprocexception = e;
}
else
{
parent.slavethreadprocexception = e;
}
}
}
protected internal Slave(SlaveMemory parent, int slavenum)
{
this.parent = parent;
this.slavenum = slavenum;
}
public void Load(int filebuffersize, string file)
{
LogLine("---- RectangularFriendsListsHugeMemory.Load ----");
//usercount = GetUniqueUserCount(file);
int blockusercount = usercount / nslaves;
if (0 != (usercount % nslaves))
{
blockusercount++;
}
long blocksize = ((long)blockusercount * (1 + (long)MaxFriends)) << 2;
int packetsize = nbatchedrows * (((1 + MaxFriends) << 2) + 1 + 8 + 4) + 1;
smtable = new SlaveMemory(objname, "CollaborativeFilteringObjectsSlave.exe", blocksize, packetsize, nthreads, nslaves);
numskippedlines = 0;
if (0 == usertotableranges.Count)
{
AddUserIDRange(0, 400000000 - 1);
}
using (MySpace.DataMining.SeqFileStream.SeqFileStreamReader stm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, filebuffersize))
{
SharedLoadContext slc = new SharedLoadContext();
slc.streamqueue = stm;
List <System.Threading.Thread> threads = new List <System.Threading.Thread>(nthreads);
smtable.Open(); // !
for (int i = 0; i != nthreads; i++)
{
ThreadLoadData tld = new ThreadLoadData();
tld.slc = slc;
tld.tv = smtable.ThreadViews[i];
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(loadthreadproc));
thd.Name = "CFO_Load_" + i.ToString();
thd.Start(tld);
threads.Add(thd);
}
for (int i = 0; i != nthreads; i++)
{
threads[i].Join();
}
}
if (null != goterr)
{
Exception e = goterr;
goterr = null;
throw e;
}
#if DEBUG
if (smtable._batchspilled)
{
LogLine("RectangularFriendsListsHugeMemory.Load: SlaveMemory batching spilled to another slave");
smtable._batchspilled = false;
}
#endif
if (realusercount >= usercount)
{
//GotException(new Exception("row limit reached"));
LogLine("row limit reached");
}
LogLine("Load finished: number of rows: " + realusercount.ToString() + " (max " + usercount.ToString() + ")");
}
