public void Load(int filebuffersize, string file, int loadthreadcount)
{
LogLine("---- JaggedFriendsLists.Load ----");
if (loadthreadcount < 1)
{
throw new Exception("Bad load thread count");
}
//usercount = GetUniqueUserCount(file);
if (table.IsNull)
{
//table = new OneDInt32(1024 * 1024 * 1024 / 2); // ...
throw new Exception("Table is null; must call SetUserTableSize");
}
numskippedlines = 0;
if (0 == usertotableranges.Count)
{
AddUserIDRange(0, 400000000 - 1);
}
using (MySpace.DataMining.SeqFileStream.SeqFileStreamReader stm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, filebuffersize))
{
SharedLoadContext slc = new SharedLoadContext();
slc.streamqueue = stm;
List <System.Threading.Thread> threads = new List <System.Threading.Thread>(loadthreadcount);
for (int i = 0; i != loadthreadcount; i++)
{
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(loadthreadproc));
thd.Name = "CFO_Load_" + i.ToString();
thd.Start(slc);
threads.Add(thd);
}
for (int i = 0; i != loadthreadcount; i++)
{
threads[i].Join();
}
}
if (hittablelimit)
{
//GotException(new Exception("table limit reached"));
LogLine("table limit reached");
}
LogLine("Load finished: number of rows: " + realusercount.ToString());
}
public void Load(int filebuffersize, string file, int loadthreadcount)
{
LogLine("---- RectangularFriendsLists.Load ----");
if (loadthreadcount < 1)
{
throw new Exception("Bad load thread count");
}
//usercount = GetUniqueUserCount(file);
table = new TwoDInt32(usercount, 1 + maxfriends);
numskippedlines = 0;
if (0 == usertotableranges.Count)
{
AddUserIDRange(0, 400000000 - 1);
}
using (MySpace.DataMining.SeqFileStream.SeqFileStreamReader stm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, filebuffersize))
{
SharedLoadContext slc = new SharedLoadContext();
slc.streamqueue = stm;
List <System.Threading.Thread> threads = new List <System.Threading.Thread>(loadthreadcount);
for (int i = 0; i != loadthreadcount; i++)
{
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(loadthreadproc));
thd.Name = "CFO_Load_" + i.ToString();
thd.Start(slc);
threads.Add(thd);
}
for (int i = 0; i != loadthreadcount; i++)
{
threads[i].Join();
}
}
if (realusercount >= usercount)
{
//GotException(new Exception("row limit reached"));
LogLine("row limit reached");
}
LogLine("Load finished: number of rows: " + realusercount.ToString() + " (max " + usercount.ToString() + ")");
}
public void OpenFile(string filename)
{
try
{
ButtonDown.Enabled = false;
sstm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Delete, 0x100000 * 20);
this.Text = (new System.IO.FileInfo(filename)).Name + " - SeqFileView";
LoadFileTop();
ButtonDown.Enabled = true;
}
catch (Exception e)
{
MessageBox.Show("Problem loading file:\r\n" + e.ToString());
}
}
public void OpenFile(string filename)
{
try
{
ButtonDown.Enabled = false;
sstm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Delete, 0x100000 * 20);
this.Text = (new System.IO.FileInfo(filename)).Name + " - SeqFileView";
LoadFileTop();
ButtonDown.Enabled = true;
}
catch (Exception e)
{
MessageBox.Show("Problem loading file:\r\n" + e.ToString());
}
}
public void Load(int filebuffersize, string file)
{
LogLine("---- RectangularFriendsListsHugeMemory.Load ----");
//usercount = GetUniqueUserCount(file);
int blockusercount = usercount / nslaves;
if (0 != (usercount % nslaves))
{
blockusercount++;
}
long blocksize = ((long)blockusercount * (1 + (long)MaxFriends)) << 2;
int packetsize = nbatchedrows * (((1 + MaxFriends) << 2) + 1 + 8 + 4) + 1;
smtable = new SlaveMemory(objname, "CollaborativeFilteringObjectsSlave.exe", blocksize, packetsize, nthreads, nslaves);
numskippedlines = 0;
if (0 == usertotableranges.Count)
{
AddUserIDRange(0, 400000000 - 1);
}
using (MySpace.DataMining.SeqFileStream.SeqFileStreamReader stm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, filebuffersize))
{
SharedLoadContext slc = new SharedLoadContext();
slc.streamqueue = stm;
List<System.Threading.Thread> threads = new List<System.Threading.Thread>(nthreads);
smtable.Open(); // !
for (int i = 0; i != nthreads; i++)
{
ThreadLoadData tld = new ThreadLoadData();
tld.slc = slc;
tld.tv = smtable.ThreadViews[i];
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(loadthreadproc));
thd.Name = "CFO_Load_" + i.ToString();
thd.Start(tld);
threads.Add(thd);
}
for (int i = 0; i != nthreads; i++)
{
threads[i].Join();
}
}
if (null != goterr)
{
Exception e = goterr;
goterr = null;
throw e;
}
#if DEBUG
if (smtable._batchspilled)
{
LogLine("RectangularFriendsListsHugeMemory.Load: SlaveMemory batching spilled to another slave");
smtable._batchspilled = false;
}
#endif
if (realusercount >= usercount)
{
//GotException(new Exception("row limit reached"));
LogLine("row limit reached");
}
LogLine("Load finished: number of rows: " + realusercount.ToString() + " (max " + usercount.ToString() + ")");
}
public void Load(int filebuffersize, string file, int loadthreadcount)
{
LogLine("---- RectangularFriendsLists.Load ----");
if (loadthreadcount < 1)
{
throw new Exception("Bad load thread count");
}
//usercount = GetUniqueUserCount(file);
table = new TwoDInt32(usercount, 1 + maxfriends);
numskippedlines = 0;
if (0 == usertotableranges.Count)
{
AddUserIDRange(0, 400000000 - 1);
}
using (MySpace.DataMining.SeqFileStream.SeqFileStreamReader stm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, filebuffersize))
{
SharedLoadContext slc = new SharedLoadContext();
slc.streamqueue = stm;
List<System.Threading.Thread> threads = new List<System.Threading.Thread>(loadthreadcount);
for (int i = 0; i != loadthreadcount; i++)
{
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(loadthreadproc));
thd.Name = "CFO_Load_" + i.ToString();
thd.Start(slc);
threads.Add(thd);
}
for (int i = 0; i != loadthreadcount; i++)
{
threads[i].Join();
}
}
if (realusercount >= usercount)
{
//GotException(new Exception("row limit reached"));
LogLine("row limit reached");
}
LogLine("Load finished: number of rows: " + realusercount.ToString() + " (max " + usercount.ToString() + ")");
}
public void Load(int filebuffersize, string file, int loadthreadcount)
{
LogLine("---- JaggedFriendsLists.Load ----");
if (loadthreadcount < 1)
{
throw new Exception("Bad load thread count");
}
//usercount = GetUniqueUserCount(file);
if (table.IsNull)
{
//table = new OneDInt32(1024 * 1024 * 1024 / 2); // ...
throw new Exception("Table is null; must call SetUserTableSize");
}
numskippedlines = 0;
if (0 == usertotableranges.Count)
{
AddUserIDRange(0, 400000000 - 1);
}
using (MySpace.DataMining.SeqFileStream.SeqFileStreamReader stm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, filebuffersize))
{
SharedLoadContext slc = new SharedLoadContext();
slc.streamqueue = stm;
List<System.Threading.Thread> threads = new List<System.Threading.Thread>(loadthreadcount);
for (int i = 0; i != loadthreadcount; i++)
{
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(loadthreadproc));
thd.Name = "CFO_Load_" + i.ToString();
thd.Start(slc);
threads.Add(thd);
}
for (int i = 0; i != loadthreadcount; i++)
{
threads[i].Join();
}
}
if (hittablelimit)
{
//GotException(new Exception("table limit reached"));
LogLine("table limit reached");
}
LogLine("Load finished: number of rows: " + realusercount.ToString());
}
public void Load(int filebuffersize, string file)
{
LogLine("---- RectangularFriendsListsHugeMemory.Load ----");
//usercount = GetUniqueUserCount(file);
int blockusercount = usercount / nslaves;
if (0 != (usercount % nslaves))
{
blockusercount++;
}
long blocksize = ((long)blockusercount * (1 + (long)MaxFriends)) << 2;
int packetsize = nbatchedrows * (((1 + MaxFriends) << 2) + 1 + 8 + 4) + 1;
smtable = new SlaveMemory(objname, "CollaborativeFilteringObjectsSlave.exe", blocksize, packetsize, nthreads, nslaves);
numskippedlines = 0;
if (0 == usertotableranges.Count)
{
AddUserIDRange(0, 400000000 - 1);
}
using (MySpace.DataMining.SeqFileStream.SeqFileStreamReader stm = new MySpace.DataMining.SeqFileStream.SeqFileStreamReader(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, filebuffersize))
{
SharedLoadContext slc = new SharedLoadContext();
slc.streamqueue = stm;
List <System.Threading.Thread> threads = new List <System.Threading.Thread>(nthreads);
smtable.Open(); // !
for (int i = 0; i != nthreads; i++)
{
ThreadLoadData tld = new ThreadLoadData();
tld.slc = slc;
tld.tv = smtable.ThreadViews[i];
System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(loadthreadproc));
thd.Name = "CFO_Load_" + i.ToString();
thd.Start(tld);
threads.Add(thd);
}
for (int i = 0; i != nthreads; i++)
{
threads[i].Join();
}
}
if (null != goterr)
{
Exception e = goterr;
goterr = null;
throw e;
}
#if DEBUG
if (smtable._batchspilled)
{
LogLine("RectangularFriendsListsHugeMemory.Load: SlaveMemory batching spilled to another slave");
smtable._batchspilled = false;
}
#endif
if (realusercount >= usercount)
{
//GotException(new Exception("row limit reached"));
LogLine("row limit reached");
}
LogLine("Load finished: number of rows: " + realusercount.ToString() + " (max " + usercount.ToString() + ")");
}
