internal void start() { kname = parent.kbasename + "_thread" + threadnum.ToString(); hmap = SlaveMemory.CreateFileMapping(SlaveMemory.INVALID_HANDLE_VALUE, IntPtr.Zero, SlaveMemory.PAGE_READWRITE, 0, 1 + 8 + 4 + parent.packetsize, kname + "_block" + parent.slavenum.ToString()); if (IntPtr.Zero == hmap) { throw new SlaveMemoryException("Unable to opeb shared memory communication buffer"); } pview = SlaveMemory.MapViewOfFile(hmap, SlaveMemory.FILE_MAP_ALL_ACCESS, 0, 0, 0); if (IntPtr.Zero == pview) { SlaveMemory.CloseHandle(hmap); hmap = IntPtr.Zero; throw new SlaveMemoryException("Unable to map shared memory communication buffer"); } this.ewh = System.Threading.EventWaitHandle.OpenExisting(kname + "_event" + parent.slavenum.ToString()); if (parent.usereturnevent) { this.ewhreturn = System.Threading.EventWaitHandle.OpenExisting(kname + "_returnevent" + parent.slavenum.ToString()); } this.thread = new System.Threading.Thread(new System.Threading.ThreadStart(slavethreadproc)); this.thread.Start(); }
void _clean() { if (IntPtr.Zero != hmap) { SlaveMemory.UnmapViewOfFile(pview); SlaveMemory.CloseHandle(hmap); hmap = IntPtr.Zero; } }
internal ThreadView(SlaveMemory parent, int threadnum, int maxpacketsize) { this.parent = parent; this.threadnum = threadnum; this.maxpacketsize = maxpacketsize; this.kname = parent.kbasename + "_thread" + threadnum.ToString(); _batspill = new List <Batch>(); }
internal void slavethreadproc() { try { unsafe { byte *com = (byte *)pview; LongByteArray block = parent.block; long offset = 0; int length = 0; for (; ;) { if (!ewh.WaitOne(1000, true)) { if (parent.mainproc.WaitForExit(1)) { //throw new SlaveMemoryException("Slave process exiting because main process exited"); break; } continue; } byte action = com[0]; switch (action) { case 1: // Read case 3: // Batch read { byte *comnext = com; for (; ;) { offset = SlaveMemory.BytesToInt64(comnext, 1); length = SlaveMemory.BytesToInt32(comnext, 1 + 8); int pos = 1 + 8 + 4; long stop = offset + length; if (stop > block.LongLength) { throw new SlaveMemoryException("Read out of bounds (offset=" + offset.ToString() + ";length=" + length.ToString() + ")"); } if (length > parent.packetsize) { throw new SlaveMemoryException("Read out of bounds (length>packetsize; length=" + length.ToString() + ")"); } while (offset < stop) { comnext[pos++] = block[offset++]; } if (3 == action) { comnext += 1 + 8 + 4 + length; if (action == comnext[0]) { continue; } } break; } com[0] = 255; // Done! if (parent.usereturnevent) { signalreturnevent(); } } break; case 2: // Write case 4: // Batch write { byte *comnext = com; for (; ;) { offset = SlaveMemory.BytesToInt64(comnext, 1); length = SlaveMemory.BytesToInt32(comnext, 1 + 8); int pos = 1 + 8 + 4; long stop = offset + length; if (stop > block.LongLength) { throw new SlaveMemoryException("Write out of bounds (offset=" + offset.ToString() + ";length=" + length.ToString() + ")"); } if (length > parent.packetsize) { throw new SlaveMemoryException("Write out of bounds (length>packetsize; length=" + length.ToString() + ")"); } while (offset < stop) { block[offset++] = comnext[pos++]; } if (4 == action) { comnext += 1 + 8 + 4 + length; if (action == comnext[0]) { continue; } } break; } com[0] = 255; // Done! if (parent.usereturnevent) { signalreturnevent(); } } break; case 0: // Exit com[0] = 255; // Done. if (parent.usereturnevent) { signalreturnevent(); } return; // ! } } } } catch (Exception e) { if (null != parent.slavethreadprocexception) { //e.InnerException = parent.slavethreadprocexception; //parent.slavethreadprocexception = e; } else { parent.slavethreadprocexception = e; } } }
protected internal Slave(SlaveMemory parent, int slavenum) { this.parent = parent; this.slavenum = slavenum; }
public void Load(int filebuffersize, string file) { LogLine("---- RectangularFriendsListsHugeMemory.Load ----"); //usercount = GetUniqueUserCount(file); int blockusercount = usercount / nslaves; if (0 != (usercount % nslaves)) { blockusercount++; } long blocksize = ((long)blockusercount * (1 + (long)MaxFriends)) << 2; int packetsize = nbatchedrows * (((1 + MaxFriends) << 2) + 1 + 8 + 4) + 1; smtable = new SlaveMemory(objname, "CollaborativeFilteringObjectsSlave.exe", blocksize, packetsize, nthreads, nslaves); numskippedlines = 0; if (0 == usertotableranges.Count) { AddUserIDRange(0, 400000000 - 1); } using (MySpace.DataMining.SeqFileStream.SeqFileStreamReader stm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, filebuffersize)) { SharedLoadContext slc = new SharedLoadContext(); slc.streamqueue = stm; List <System.Threading.Thread> threads = new List <System.Threading.Thread>(nthreads); smtable.Open(); // ! for (int i = 0; i != nthreads; i++) { ThreadLoadData tld = new ThreadLoadData(); tld.slc = slc; tld.tv = smtable.ThreadViews[i]; System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(loadthreadproc)); thd.Name = "CFO_Load_" + i.ToString(); thd.Start(tld); threads.Add(thd); } for (int i = 0; i != nthreads; i++) { threads[i].Join(); } } if (null != goterr) { Exception e = goterr; goterr = null; throw e; } #if DEBUG if (smtable._batchspilled) { LogLine("RectangularFriendsListsHugeMemory.Load: SlaveMemory batching spilled to another slave"); smtable._batchspilled = false; } #endif if (realusercount >= usercount) { //GotException(new Exception("row limit reached")); LogLine("row limit reached"); } LogLine("Load finished: number of rows: " + realusercount.ToString() + " (max " + usercount.ToString() + ")"); }