// fofmax is max number of friends-of-friends.
// fofhashsize should be prime near (fofmax * 10)
// fhashsize should be prime near (MaxFriends * 10)
public void ToFofScores(int iTotalThreads, IList <int> iMyThreads, string base_output_filename, int fofmax, int fofhashsize, int fhashsize)
{
LogLine("---- JaggedFriendsLists.ToFofScores ----");
List <DoScores> threads = new List <DoScores>(iMyThreads.Count);
for (int i = 0; i != iMyThreads.Count; i++)
{
DoScores ds = new DoScores();
ds.iTotalThreads = iTotalThreads;
ds.iMyThread = iMyThreads[i];
ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
ds.fofmax = fofmax;
ds.fofhashsize = fofhashsize;
ds.fhashsize = fhashsize;
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
thd.Name = "CFO_ToFofScores_" + i.ToString();
ds.thd = thd;
thd.Start(ds);
threads.Add(ds);
}
for (int i = 0; i != iMyThreads.Count; i++)
{
threads[i].thd.Join();
threads[i].sw.Close();
}
//LogLine("Total exceptions caught: " + exceptioncount.ToString());
LogLine("---- JaggedFriendsLists processing done ----");
}
public void ToFofScores(int iTotalThreads, IList <int> iMyThreads, string base_output_filename)
{
LogLine("---- RectangularFriendsLists.ToFofScores ----");
if (null == base_output_filename)
{
if (null == outranges)
{
throw new Exception("No output file specified");
}
}
List <DoScores> threads = new List <DoScores>(iMyThreads.Count);
for (int i = 0; i != iMyThreads.Count; i++)
{
DoScores ds = new DoScores();
ds.iTotalThreads = iTotalThreads;
ds.iMyThread = iMyThreads[i];
if (null != base_output_filename)
{
ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
}
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
thd.Name = "CFO_ToFofScores_" + i.ToString();
ds.thd = thd;
thd.Start(ds);
threads.Add(ds);
}
for (int i = 0; i != iMyThreads.Count; i++)
{
threads[i].thd.Join();
if (null != threads[i].sw)
{
threads[i].sw.Close();
}
}
if (null != outranges)
{
foreach (OutRange or in outranges)
{
or.sw.Close();
}
}
//LogLine("Total exceptions caught: " + exceptioncount.ToString());
LogLine("---- RectangularFriendsLists processing done ----");
}
// Note: iTotalThreads is >= ThreadCount; it is considering multiple passes.
public void ToFofScores(int iTotalThreads, IList<int> iMyThreads, string base_output_filename)
{
LogLine("---- RectangularFriendsListsHugeMemory.ToFofScores ----");
if (null == base_output_filename)
{
if (null == outranges)
{
throw new Exception("No output file specified");
}
}
if (iMyThreads.Count != nthreads)
{
throw new Exception("ToFofScores: iMyThreads.Count is not the same as ThreadCount");
}
List<DoScores> threads = new List<DoScores>(nthreads);
for (int i = 0; i != iMyThreads.Count; i++)
{
DoScores ds = new DoScores();
ds.tv = smtable.ThreadViews[i];
ds.iTotalThreads = iTotalThreads;
ds.iMyThread = iMyThreads[i];
if (null != base_output_filename)
{
ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
}
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
thd.Name = "CFO_ToFofScores_" + i.ToString();
ds.thd = thd;
thd.Start(ds);
threads.Add(ds);
}
for (int i = 0; i != iMyThreads.Count; i++)
{
threads[i].thd.Join();
if (null != threads[i].sw)
{
threads[i].sw.Close();
}
}
if (null != outranges)
{
foreach (OutRange or in outranges)
{
or.sw.Close();
}
}
if (null != goterr)
{
Exception e = goterr;
goterr = null;
throw e;
}
#if DEBUG
if (smtable._batchspilled)
{
LogLine("RectangularFriendsListsHugeMemory.ToFofScores: SlaveMemory batching spilled to another slave");
smtable._batchspilled = false;
}
#endif
//LogLine("Total exceptions caught: " + exceptioncount.ToString());
LogLine("---- RectangularFriendsListsHugeMemory processing done ----");
}
void tofofscorespartthreadproc(object obj)
{
try
{
DoScores ds = (DoScores)obj;
System.IO.StreamWriter sw = ds.sw;
int fofmax = ds.fofmax;
DHT htfof = new DHT(ds.fofmax, ds.fofhashsize); // Friends of my friends.
DHT htf = new DHT(maxfriends, ds.fhashsize); // My friends, excluding from fof.
long rowA = 0;
int curusercount = 0;
for (int i = 0; i < ds.iMyThread; i++)
{
if (curusercount >= realusercount)
{
break;
}
UserData ud = UserData.Create(this.table, rowA);
rowA = ud.GetNextRowID();
curusercount++;
}
while (curusercount < realusercount)
{
Int32 uidA = GetUserIDFromRowID(rowA);
UserData udA = UserData.Create(this.table, rowA);
for (int ifriend = 0; ifriend < udA.NumberOfFriends; ifriend++)
{
Int32 fid = udA.GetFriendAt(ifriend);
htf.Add(fid);
}
int fofcount = 0;
for (int ifriend = 0; ifriend < udA.NumberOfFriends; ifriend++)
{
#if DEBUG
if (_sleep)
{
System.Threading.Thread.Sleep(DateTime.Now.Millisecond % 128);
}
#endif
Int32 fid = udA.GetFriendAt(ifriend);
//int frow = GetRowIDFromUserID(fid); // Row index of fid. -1 if not found (0 is valid).
long frowid = GetRowIDFromUserIDSameRange(fid, uidA); // Row index of fid in same id range as uidA. -1 if not found (0 is valid).
if (-1 != frowid)
{
UserData udf = UserData.Create(this.table, frowid);
for (int ifof = 0; ifof < udf.NumberOfFriends; ifof++) // fof index.
{
Int32 fofid = udf.GetFriendAt(ifof);
if (fofid != uidA && !htf.ContainsKey(fofid))
{
// Fof only if not self and not one of my direct friends.
htfof.Add(fofid);
if (++fofcount >= fofmax)
{
break;
}
}
}
}
if (fofcount >= fofmax)
{
break;
}
}
htf.DeepClean(); // !
// Count up friend dupes.
DHTSlot[] htslots = htfof.slots;
foreach (int i in htfof.dirtyslots)
{
int score = htslots[i].score; // 1 is base score.
if (score >= minscores)
{
sw.Write(uidA);
sw.Write(',');
sw.Write(htslots[i].key);
sw.Write(',');
sw.Write(score);
sw.Write(ds.sw.NewLine);
}
htfof.UnsetSlot(i); // Deep clean pt 1/2.
}
htfof.ZeroDirty(); // Deep clean pt 2/2.
for (int i = 0; i < ds.iTotalThreads; i++)
{
if (curusercount >= realusercount)
{
break;
}
UserData ud = UserData.Create(this.table, rowA);
rowA = ud.GetNextRowID();
curusercount++;
}
}
sw.Flush();
}
catch (Exception e)
{
LogLine("\r\nProcessing error: tofofscorespartthreadproc catch-all exception: " + e.ToString() + "\r\n");
}
}
public void ToFofScores(int iTotalThreads, IList<int> iMyThreads, string base_output_filename)
{
LogLine("---- RectangularFriendsLists.ToFofScores ----");
if (null == base_output_filename)
{
if (null == outranges)
{
throw new Exception("No output file specified");
}
}
List<DoScores> threads = new List<DoScores>(iMyThreads.Count);
for (int i = 0; i != iMyThreads.Count; i++)
{
DoScores ds = new DoScores();
ds.iTotalThreads = iTotalThreads;
ds.iMyThread = iMyThreads[i];
if (null != base_output_filename)
{
ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
}
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
thd.Name = "CFO_ToFofScores_" + i.ToString();
ds.thd = thd;
thd.Start(ds);
threads.Add(ds);
}
for (int i = 0; i != iMyThreads.Count; i++)
{
threads[i].thd.Join();
if (null != threads[i].sw)
{
threads[i].sw.Close();
}
}
if (null != outranges)
{
foreach (OutRange or in outranges)
{
or.sw.Close();
}
}
//LogLine("Total exceptions caught: " + exceptioncount.ToString());
LogLine("---- RectangularFriendsLists processing done ----");
}
void tofofscorespartthreadproc(object obj)
{
try
{
DoScores ds = (DoScores)obj;
DHT htfof = new DHT(maxfics); // Friends of my friends.
DHT htf = new DHT(); // My friends, excluding from fof.
// Start at row 1 instead of 0...
for (int rowA = 1 + ds.iMyThread; rowA < realusercount; rowA += ds.iTotalThreads)
{
Int32 uidA = GetUserIDFromRowID(rowA);
for (int colA = 1; colA < 1 + maxfriends; colA++)
{
Int32 fid = table[rowA, colA];
if (0 == fid)
{
break;
}
htf.Add(fid);
}
for (int colA = 1; colA < 1 + maxfriends; colA++)
{
Int32 fid = table[rowA, colA];
if (0 == fid)
{
break;
}
//int frow = GetRowIDFromUserID(fid); // Row index of fid. -1 if not found (0 is valid).
int frow = GetRowIDFromUserIDSameRange(fid, uidA); // Row index of fid in same id range as uidA. -1 if not found (0 is valid).
if (-1 != frow)
{
for (int fofcol = 1; fofcol != 1 + maxfriends; fofcol++) // fof column.
{
Int32 fofid = table[frow, fofcol]; // fof user ID.
if (0 == fofid)
{
break;
}
if (fofid != uidA && !htf.ContainsKey(fofid))
{
// Fof only if not self and not one of my direct friends.
htfof.Add(fofid, fid);
}
}
}
}
htf.DeepClean(); // !
// Count up friend dupes.
DHTSlot[] htslots = htfof.slots;
foreach (int i in htfof.dirtyslots)
{
int score = htslots[i].score; // 1 is base score.
if (score >= minscores)
{
if (null == ds.sw)
{
int ior = FindOutputRange(uidA);
if (-1 == ior)
{
noranges++;
}
else
{
lock (outranges[ior].sw)
{
int nfics = score;
if (score > maxfics)
{
nfics = maxfics;
}
WriteOutputLine(outranges[ior].sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
}
}
}
else
{
int nfics = score;
if (score > maxfics)
{
nfics = maxfics;
}
WriteOutputLine(ds.sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
}
}
htfof.UnsetSlot(i); // Deep clean pt 1/2.
}
htfof.ZeroDirty(); // Deep clean pt 2/2.
}
if (null != ds.sw)
{
ds.sw.Flush();
}
}
catch (Exception e)
{
LogLine("\r\nProcessing error: tofofscorespartthreadproc catch-all exception: " + e.ToString() + "\r\n");
}
}
// fofmax is max number of friends-of-friends.
// fofhashsize should be prime near (fofmax * 10)
// fhashsize should be prime near (MaxFriends * 10)
public void ToFofScores(int iTotalThreads, IList<int> iMyThreads, string base_output_filename, int fofmax, int fofhashsize, int fhashsize)
{
LogLine("---- JaggedFriendsLists.ToFofScores ----");
List<DoScores> threads = new List<DoScores>(iMyThreads.Count);
for (int i = 0; i != iMyThreads.Count; i++)
{
DoScores ds = new DoScores();
ds.iTotalThreads = iTotalThreads;
ds.iMyThread = iMyThreads[i];
ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
ds.fofmax = fofmax;
ds.fofhashsize = fofhashsize;
ds.fhashsize = fhashsize;
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
thd.Name = "CFO_ToFofScores_" + i.ToString();
ds.thd = thd;
thd.Start(ds);
threads.Add(ds);
}
for (int i = 0; i != iMyThreads.Count; i++)
{
threads[i].thd.Join();
threads[i].sw.Close();
}
//LogLine("Total exceptions caught: " + exceptioncount.ToString());
LogLine("---- JaggedFriendsLists processing done ----");
}
// Note: iTotalThreads is >= ThreadCount; it is considering multiple passes.
public void ToFofScores(int iTotalThreads, IList <int> iMyThreads, string base_output_filename)
{
LogLine("---- RectangularFriendsListsHugeMemory.ToFofScores ----");
if (null == base_output_filename)
{
if (null == outranges)
{
throw new Exception("No output file specified");
}
}
if (iMyThreads.Count != nthreads)
{
throw new Exception("ToFofScores: iMyThreads.Count is not the same as ThreadCount");
}
List <DoScores> threads = new List <DoScores>(nthreads);
for (int i = 0; i != iMyThreads.Count; i++)
{
DoScores ds = new DoScores();
ds.tv = smtable.ThreadViews[i];
ds.iTotalThreads = iTotalThreads;
ds.iMyThread = iMyThreads[i];
if (null != base_output_filename)
{
ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
}
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
thd.Name = "CFO_ToFofScores_" + i.ToString();
ds.thd = thd;
thd.Start(ds);
threads.Add(ds);
}
for (int i = 0; i != iMyThreads.Count; i++)
{
threads[i].thd.Join();
if (null != threads[i].sw)
{
threads[i].sw.Close();
}
}
if (null != outranges)
{
foreach (OutRange or in outranges)
{
or.sw.Close();
}
}
if (null != goterr)
{
Exception e = goterr;
goterr = null;
throw e;
}
#if DEBUG
if (smtable._batchspilled)
{
LogLine("RectangularFriendsListsHugeMemory.ToFofScores: SlaveMemory batching spilled to another slave");
smtable._batchspilled = false;
}
#endif
//LogLine("Total exceptions caught: " + exceptioncount.ToString());
LogLine("---- RectangularFriendsListsHugeMemory processing done ----");
}
void tofofscorespartthreadproc(object obj)
{
try
{
ThreadBatchData fbatchdata = new ThreadBatchData(this, nbatchedrows);
ThreadBatchData fofbatchdata = new ThreadBatchData(this, nbatchedrows);
DoScores ds = (DoScores)obj;
SlaveMemory.ThreadView tv = ds.tv;
Int32[] friendsbuf = new Int32[maxfriends];
Int32[] afriends = new Int32[maxfriends];
DHT htfof = new DHT(maxfics); // Friends of my friends.
DHT htf = new DHT(); // My friends, excluding from fof.
// Start at row 1 instead of 0...
int rowA = 1 + ds.iMyThread;
int rowAstart = rowA;
int ifbatch = 0;
for ( ; ; rowA += ds.iTotalThreads)
{
if (ifbatch >= nbatchedrows ||
rowA >= realusercount)
{
fbatchdata.BatchGet(tv);
for (int br = 0; br < ifbatch; br++)
{
#if DEBUG
for (int i = 0; i < maxfriends; i++)
{
afriends[i] = -929292;
}
#endif
Int32 uidA = fbatchdata.GetRowData(br, afriends);
for (int ifriend = 0; ifriend < maxfriends; ifriend++)
{
Int32 fid = afriends[ifriend];
if (0 == fid)
{
break;
}
#if DEBUG
if (-929292 == fid)
{
throw new Exception("fid -929292");
}
#endif
htf.Add(fid);
}
{
int ibatch = 0;
int ifriend = 0;
//int ifriendstart = 0;
for (; ; ifriend++)
{
if (ibatch >= nbatchedrows ||
ifriend >= maxfriends ||
0 == afriends[ifriend])
{
fofbatchdata.BatchGet(tv);
for (int bfriend = 0; bfriend < ibatch; bfriend++)
{
#if DEBUG
for (int i = 0; i < maxfriends; i++)
{
friendsbuf[i] = -929292;
}
#endif
int xfid = fofbatchdata.GetRowData(bfriend, friendsbuf);
//int brealifriend = ifriendstart + bfriend;
for (int ifof = 0; ifof < maxfriends; ifof++)
{
Int32 fofid = friendsbuf[ifof];
if (0 == fofid)
{
break;
}
#if DEBUG
if (-929292 == fofid)
{
throw new Exception("fofid -929292");
}
#endif
if (fofid != uidA && !htf.ContainsKey(fofid))
{
#if DEBUG
if (uidA == 17 && fofid == 10)
{
int i33 = 33 + 33;
}
#endif
// Fof only if not self and not one of my direct friends.
htfof.Add(fofid, xfid);
}
}
}
if (ifriend >= maxfriends ||
0 == afriends[ifriend])
{
break;
}
ibatch = 0;
//ifriendstart = ifriend + 1;
}
Int32 fid = afriends[ifriend];
if (0 == fid)
{
break;
}
#if DEBUG
if (-929292 == fid)
{
throw new Exception("fid -929292");
}
#endif
//int frow = GetRowIDFromUserID(fid); // Row index of fid. -1 if not found (0 is valid).
int frow = GetRowIDFromUserIDSameRange(fid, uidA); // Row index of fid in same id range as uidA. -1 if not found (0 is valid).
if (-1 != frow)
{
fofbatchdata.BatchRow(ibatch++, frow);
}
}
}
htf.DeepClean(); // !
// Count up friend dupes.
DHTSlot[] htslots = htfof.slots;
foreach (int i in htfof.dirtyslots)
{
int score = htslots[i].score; // 1 is base score.
if (score >= minscores)
{
if (null == ds.sw)
{
int ior = FindOutputRange(uidA);
if (-1 == ior)
{
noranges++;
}
else
{
lock (outranges[ior].sw)
{
int nfics = score;
if (score > maxfics)
{
nfics = maxfics;
}
WriteOutputLine(outranges[ior].sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
}
}
}
else
{
int nfics = score;
if (score > maxfics)
{
nfics = maxfics;
}
WriteOutputLine(ds.sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
}
}
htfof.UnsetSlot(i); // Deep clean pt 1/2.
}
htfof.ZeroDirty(); // Deep clean pt 2/2.
}
if (rowA >= realusercount)
{
break;
}
ifbatch = 0;
rowAstart = rowA + ds.iTotalThreads;
}
fbatchdata.BatchRow(ifbatch++, rowA);
}
if (null != ds.sw)
{
ds.sw.Flush();
}
}
catch (Exception e)
{
if (null == goterr)
{
goterr = new Exception("Processing error: tofofscorespartthreadproc catch-all exception", e);
}
LogLine("\r\nProcessing error: tofofscorespartthreadproc catch-all exception: " + e.ToString() + "\r\n");
}
}