public void Load(int filebuffersize, string file) { LogLine("---- RectangularFriendsListsHugeMemory.Load ----"); //usercount = GetUniqueUserCount(file); int blockusercount = usercount / nslaves; if (0 != (usercount % nslaves)) { blockusercount++; } long blocksize = ((long)blockusercount * (1 + (long)MaxFriends)) << 2; int packetsize = nbatchedrows * (((1 + MaxFriends) << 2) + 1 + 8 + 4) + 1; smtable = new SlaveMemory(objname, "CollaborativeFilteringObjectsSlave.exe", blocksize, packetsize, nthreads, nslaves); numskippedlines = 0; if (0 == usertotableranges.Count) { AddUserIDRange(0, 400000000 - 1); } using (MySpace.DataMining.SeqFileStream.SeqFileStreamReader stm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, filebuffersize)) { SharedLoadContext slc = new SharedLoadContext(); slc.streamqueue = stm; List<System.Threading.Thread> threads = new List<System.Threading.Thread>(nthreads); smtable.Open(); // ! for (int i = 0; i != nthreads; i++) { ThreadLoadData tld = new ThreadLoadData(); tld.slc = slc; tld.tv = smtable.ThreadViews[i]; System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(loadthreadproc)); thd.Name = "CFO_Load_" + i.ToString(); thd.Start(tld); threads.Add(thd); } for (int i = 0; i != nthreads; i++) { threads[i].Join(); } } if (null != goterr) { Exception e = goterr; goterr = null; throw e; } #if DEBUG if (smtable._batchspilled) { LogLine("RectangularFriendsListsHugeMemory.Load: SlaveMemory batching spilled to another slave"); smtable._batchspilled = false; } #endif if (realusercount >= usercount) { //GotException(new Exception("row limit reached")); LogLine("row limit reached"); } LogLine("Load finished: number of rows: " + realusercount.ToString() + " (max " + usercount.ToString() + ")"); }
internal void BatchGet(SlaveMemory.ThreadView tv) { tv.Get(batch); batch.Clear(); }