void loadthreadproc(object obj)
{
try
{
#if DEBUG
#else
unchecked
#endif
{
ThreadBatchData batchdata = new ThreadBatchData(this, nbatchedrows);
int curbatch = 0; // Current row from nbatchedrows in batchdata.
ThreadLoadData tld = (ThreadLoadData)obj;
SharedLoadContext slc = tld.slc;
SlaveMemory.ThreadView tv = tld.tv;
SeqFileStream.SeqFileStreamReader streamqueue = slc.streamqueue;
Int32[] friendsbuf = new Int32[maxfriends];
byte[] linebuf = new byte[32];
#if DEBUG
for (int i = 0; i < maxfriends; i++)
{
friendsbuf[i] = -929292;
}
#endif
// Assumes trailing friend-IDs are 0.
while (slc.nexttableindex < usercount)
{
long linnum = 0;
string curfilename = null;
System.IO.Stream stm = null;
try
{
lock (streamqueue)
{
stm = streamqueue.GetNextStream(out curfilename);
}
if (null == stm)
{
break;
}
using (System.IO.StreamReader rstm = new System.IO.StreamReader(stm))
{
linnum = 0;
int friendindex = 0;
Int32 previd = 0;
for (; ;)
{
linnum++;
Int32 uid, fid;
if (GetNextLine(rstm, out uid, out fid, linebuf))
{
if (uid <= 0)
{
//throw new Exception("bad line: user ID invalid");
numskippedlines++;
continue;
}
if (fid <= 0)
{
//throw new Exception("bad line: friend ID invalid");
numskippedlines++;
continue;
}
}
if (uid != previd)
{
if (0 != previd)
{
if (slc.nexttableindex >= usercount)
{
break;
}
for (int i = friendindex; i < maxfriends; i++)
{
friendsbuf[i] = 0;
}
friendindex = 0;
lock (slc)
{
SetRowIDForUserID(previd, slc.nexttableindex);
batchdata.SetRowData(curbatch, previd, friendsbuf);
batchdata.BatchRow(curbatch, slc.nexttableindex);
curbatch++;
if (curbatch >= nbatchedrows)
{
curbatch = 0;
batchdata.BatchSet(tv);
}
slc.nexttableindex++;
}
#if DEBUG
for (int i = 0; i < maxfriends; i++)
{
friendsbuf[i] = -929292;
}
#endif
}
}
if (0 == uid)
{
break;
}
if (friendindex < maxfriends)
{
friendsbuf[friendindex] = fid;
friendindex++;
}
previd = uid;
}
}
}
catch (Exception e)
{
if (null == goterr)
{
goterr = new Exception("Load error: " + e.ToString() + " file: " + (null == curfilename ? "<null>" : "'" + curfilename + "'") + "; line: " + linnum.ToString() + "; table index: " + slc.nexttableindex.ToString(), e);
}
try
{
LogLine("\r\nLoad error: " + e.ToString() + " file: " + (null == curfilename ? "<null>" : "'" + curfilename + "'") + "; line: " + linnum.ToString() + "; table index: " + slc.nexttableindex.ToString() + "; exception: " + e.ToString() + "\r\n(Skipping the rest of this file)\r\n");
}
catch (Exception e2)
{
LogLine("\r\nError error: " + e2.ToString() + "\r\n");
}
}
finally
{
if (null != stm)
{
stm.Close();
}
}
}
if (curbatch > 0)
{
try
{
batchdata.BatchSet(tv);
}
catch (Exception e33)
{
int i33 = 33 + 33;
throw e33;
}
}
realusercount = slc.nexttableindex;
}
}
catch (Exception e)
{
if (null == goterr)
{
goterr = new Exception("Load error: loadthreadproc catch-all exception", e);
}
LogLine("\r\nLoad error: loadthreadproc catch-all exception: " + e.ToString() + "\r\n");
}
}
void tofofscorespartthreadproc(object obj)
{
try
{
ThreadBatchData fbatchdata = new ThreadBatchData(this, nbatchedrows);
ThreadBatchData fofbatchdata = new ThreadBatchData(this, nbatchedrows);
DoScores ds = (DoScores)obj;
SlaveMemory.ThreadView tv = ds.tv;
Int32[] friendsbuf = new Int32[maxfriends];
Int32[] afriends = new Int32[maxfriends];
DHT htfof = new DHT(maxfics); // Friends of my friends.
DHT htf = new DHT(); // My friends, excluding from fof.
// Start at row 1 instead of 0...
int rowA = 1 + ds.iMyThread;
int rowAstart = rowA;
int ifbatch = 0;
for ( ; ; rowA += ds.iTotalThreads)
{
if (ifbatch >= nbatchedrows ||
rowA >= realusercount)
{
fbatchdata.BatchGet(tv);
for (int br = 0; br < ifbatch; br++)
{
#if DEBUG
for (int i = 0; i < maxfriends; i++)
{
afriends[i] = -929292;
}
#endif
Int32 uidA = fbatchdata.GetRowData(br, afriends);
for (int ifriend = 0; ifriend < maxfriends; ifriend++)
{
Int32 fid = afriends[ifriend];
if (0 == fid)
{
break;
}
#if DEBUG
if (-929292 == fid)
{
throw new Exception("fid -929292");
}
#endif
htf.Add(fid);
}
{
int ibatch = 0;
int ifriend = 0;
//int ifriendstart = 0;
for (; ; ifriend++)
{
if (ibatch >= nbatchedrows ||
ifriend >= maxfriends ||
0 == afriends[ifriend])
{
fofbatchdata.BatchGet(tv);
for (int bfriend = 0; bfriend < ibatch; bfriend++)
{
#if DEBUG
for (int i = 0; i < maxfriends; i++)
{
friendsbuf[i] = -929292;
}
#endif
int xfid = fofbatchdata.GetRowData(bfriend, friendsbuf);
//int brealifriend = ifriendstart + bfriend;
for (int ifof = 0; ifof < maxfriends; ifof++)
{
Int32 fofid = friendsbuf[ifof];
if (0 == fofid)
{
break;
}
#if DEBUG
if (-929292 == fofid)
{
throw new Exception("fofid -929292");
}
#endif
if (fofid != uidA && !htf.ContainsKey(fofid))
{
#if DEBUG
if (uidA == 17 && fofid == 10)
{
int i33 = 33 + 33;
}
#endif
// Fof only if not self and not one of my direct friends.
htfof.Add(fofid, xfid);
}
}
}
if (ifriend >= maxfriends ||
0 == afriends[ifriend])
{
break;
}
ibatch = 0;
//ifriendstart = ifriend + 1;
}
Int32 fid = afriends[ifriend];
if (0 == fid)
{
break;
}
#if DEBUG
if (-929292 == fid)
{
throw new Exception("fid -929292");
}
#endif
//int frow = GetRowIDFromUserID(fid); // Row index of fid. -1 if not found (0 is valid).
int frow = GetRowIDFromUserIDSameRange(fid, uidA); // Row index of fid in same id range as uidA. -1 if not found (0 is valid).
if (-1 != frow)
{
fofbatchdata.BatchRow(ibatch++, frow);
}
}
}
htf.DeepClean(); // !
// Count up friend dupes.
DHTSlot[] htslots = htfof.slots;
foreach (int i in htfof.dirtyslots)
{
int score = htslots[i].score; // 1 is base score.
if (score >= minscores)
{
if (null == ds.sw)
{
int ior = FindOutputRange(uidA);
if (-1 == ior)
{
noranges++;
}
else
{
lock (outranges[ior].sw)
{
int nfics = score;
if (score > maxfics)
{
nfics = maxfics;
}
WriteOutputLine(outranges[ior].sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
}
}
}
else
{
int nfics = score;
if (score > maxfics)
{
nfics = maxfics;
}
WriteOutputLine(ds.sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
}
}
htfof.UnsetSlot(i); // Deep clean pt 1/2.
}
htfof.ZeroDirty(); // Deep clean pt 2/2.
}
if (rowA >= realusercount)
{
break;
}
ifbatch = 0;
rowAstart = rowA + ds.iTotalThreads;
}
fbatchdata.BatchRow(ifbatch++, rowA);
}
if (null != ds.sw)
{
ds.sw.Flush();
}
}
catch (Exception e)
{
if (null == goterr)
{
goterr = new Exception("Processing error: tofofscorespartthreadproc catch-all exception", e);
}
LogLine("\r\nProcessing error: tofofscorespartthreadproc catch-all exception: " + e.ToString() + "\r\n");
}
}
internal void BatchGet(SlaveMemory.ThreadView tv)
{
tv.Get(batch);
batch.Clear();
}
