Esempio n. 1
0
        // fofmax is max number of friends-of-friends.
        // fofhashsize should be prime near (fofmax * 10)
        // fhashsize should be prime near (MaxFriends * 10)
        public void ToFofScores(int iTotalThreads, IList <int> iMyThreads, string base_output_filename, int fofmax, int fofhashsize, int fhashsize)
        {
            LogLine("---- JaggedFriendsLists.ToFofScores ----");

            List <DoScores> threads = new List <DoScores>(iMyThreads.Count);

            for (int i = 0; i != iMyThreads.Count; i++)
            {
                DoScores ds = new DoScores();
                ds.iTotalThreads = iTotalThreads;
                ds.iMyThread     = iMyThreads[i];
                ds.sw            = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
                ds.fofmax        = fofmax;
                ds.fofhashsize   = fofhashsize;
                ds.fhashsize     = fhashsize;
                System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
                thd.Name = "CFO_ToFofScores_" + i.ToString();
                ds.thd   = thd;
                thd.Start(ds);
                threads.Add(ds);
            }
            for (int i = 0; i != iMyThreads.Count; i++)
            {
                threads[i].thd.Join();
                threads[i].sw.Close();
            }

            //LogLine("Total exceptions caught: " + exceptioncount.ToString());

            LogLine("---- JaggedFriendsLists processing done ----");
        }
Esempio n. 2
0
        public void ToFofScores(int iTotalThreads, IList <int> iMyThreads, string base_output_filename)
        {
            LogLine("---- RectangularFriendsLists.ToFofScores ----");

            if (null == base_output_filename)
            {
                if (null == outranges)
                {
                    throw new Exception("No output file specified");
                }
            }

            List <DoScores> threads = new List <DoScores>(iMyThreads.Count);

            for (int i = 0; i != iMyThreads.Count; i++)
            {
                DoScores ds = new DoScores();
                ds.iTotalThreads = iTotalThreads;
                ds.iMyThread     = iMyThreads[i];
                if (null != base_output_filename)
                {
                    ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
                }
                System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
                thd.Name = "CFO_ToFofScores_" + i.ToString();
                ds.thd   = thd;
                thd.Start(ds);
                threads.Add(ds);
            }
            for (int i = 0; i != iMyThreads.Count; i++)
            {
                threads[i].thd.Join();
                if (null != threads[i].sw)
                {
                    threads[i].sw.Close();
                }
            }
            if (null != outranges)
            {
                foreach (OutRange or in outranges)
                {
                    or.sw.Close();
                }
            }

            //LogLine("Total exceptions caught: " + exceptioncount.ToString());

            LogLine("---- RectangularFriendsLists processing done ----");
        }
        // Note: iTotalThreads is >= ThreadCount; it is considering multiple passes.
        public void ToFofScores(int iTotalThreads, IList<int> iMyThreads, string base_output_filename)
        {
            LogLine("---- RectangularFriendsListsHugeMemory.ToFofScores ----");

            if (null == base_output_filename)
            {
                if (null == outranges)
                {
                    throw new Exception("No output file specified");
                }
            }

            if (iMyThreads.Count != nthreads)
            {
                throw new Exception("ToFofScores: iMyThreads.Count is not the same as ThreadCount");
            }

            List<DoScores> threads = new List<DoScores>(nthreads);
            for (int i = 0; i != iMyThreads.Count; i++)
            {
                DoScores ds = new DoScores();
                ds.tv = smtable.ThreadViews[i];
                ds.iTotalThreads = iTotalThreads;
                ds.iMyThread = iMyThreads[i];
                if (null != base_output_filename)
                {
                    ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
                }
                System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
                thd.Name = "CFO_ToFofScores_" + i.ToString();
                ds.thd = thd;
                thd.Start(ds);
                threads.Add(ds);
            }
            for (int i = 0; i != iMyThreads.Count; i++)
            {
                threads[i].thd.Join();
                if (null != threads[i].sw)
                {
                    threads[i].sw.Close();
                }
            }
            if (null != outranges)
            {
                foreach (OutRange or in outranges)
                {
                    or.sw.Close();
                }
            }
            if (null != goterr)
            {
                Exception e = goterr;
                goterr = null;
                throw e;
            }

#if DEBUG
            if (smtable._batchspilled)
            {
                LogLine("RectangularFriendsListsHugeMemory.ToFofScores: SlaveMemory batching spilled to another slave");
                smtable._batchspilled = false;
            }
#endif

            //LogLine("Total exceptions caught: " + exceptioncount.ToString());

            LogLine("---- RectangularFriendsListsHugeMemory processing done ----");
        }
Esempio n. 4
0
        void tofofscorespartthreadproc(object obj)
        {
            try
            {
                DoScores ds = (DoScores)obj;
                System.IO.StreamWriter sw = ds.sw;
                int fofmax = ds.fofmax;

                DHT htfof = new DHT(ds.fofmax, ds.fofhashsize); // Friends of my friends.
                DHT htf   = new DHT(maxfriends, ds.fhashsize);  // My friends, excluding from fof.

                long rowA         = 0;
                int  curusercount = 0;
                for (int i = 0; i < ds.iMyThread; i++)
                {
                    if (curusercount >= realusercount)
                    {
                        break;
                    }
                    UserData ud = UserData.Create(this.table, rowA);
                    rowA = ud.GetNextRowID();
                    curusercount++;
                }

                while (curusercount < realusercount)
                {
                    Int32    uidA = GetUserIDFromRowID(rowA);
                    UserData udA  = UserData.Create(this.table, rowA);
                    for (int ifriend = 0; ifriend < udA.NumberOfFriends; ifriend++)
                    {
                        Int32 fid = udA.GetFriendAt(ifriend);
                        htf.Add(fid);
                    }
                    int fofcount = 0;
                    for (int ifriend = 0; ifriend < udA.NumberOfFriends; ifriend++)
                    {
#if DEBUG
                        if (_sleep)
                        {
                            System.Threading.Thread.Sleep(DateTime.Now.Millisecond % 128);
                        }
#endif
                        Int32 fid = udA.GetFriendAt(ifriend);
                        //int frow = GetRowIDFromUserID(fid); // Row index of fid. -1 if not found (0 is valid).
                        long frowid = GetRowIDFromUserIDSameRange(fid, uidA); // Row index of fid in same id range as uidA. -1 if not found (0 is valid).
                        if (-1 != frowid)
                        {
                            UserData udf = UserData.Create(this.table, frowid);
                            for (int ifof = 0; ifof < udf.NumberOfFriends; ifof++) // fof index.
                            {
                                Int32 fofid = udf.GetFriendAt(ifof);
                                if (fofid != uidA && !htf.ContainsKey(fofid))
                                {
                                    // Fof only if not self and not one of my direct friends.
                                    htfof.Add(fofid);
                                    if (++fofcount >= fofmax)
                                    {
                                        break;
                                    }
                                }
                            }
                        }
                        if (fofcount >= fofmax)
                        {
                            break;
                        }
                    }
                    htf.DeepClean(); // !
                    // Count up friend dupes.
                    DHTSlot[] htslots = htfof.slots;
                    foreach (int i in htfof.dirtyslots)
                    {
                        int score = htslots[i].score; // 1 is base score.
                        if (score >= minscores)
                        {
                            sw.Write(uidA);
                            sw.Write(',');
                            sw.Write(htslots[i].key);
                            sw.Write(',');
                            sw.Write(score);
                            sw.Write(ds.sw.NewLine);
                        }
                        htfof.UnsetSlot(i); // Deep clean pt 1/2.
                    }
                    htfof.ZeroDirty();      // Deep clean pt 2/2.

                    for (int i = 0; i < ds.iTotalThreads; i++)
                    {
                        if (curusercount >= realusercount)
                        {
                            break;
                        }
                        UserData ud = UserData.Create(this.table, rowA);
                        rowA = ud.GetNextRowID();
                        curusercount++;
                    }
                }

                sw.Flush();
            }
            catch (Exception e)
            {
                LogLine("\r\nProcessing error: tofofscorespartthreadproc catch-all exception: " + e.ToString() + "\r\n");
            }
        }
        public void ToFofScores(int iTotalThreads, IList<int> iMyThreads, string base_output_filename)
        {
            LogLine("---- RectangularFriendsLists.ToFofScores ----");

            if (null == base_output_filename)
            {
                if (null == outranges)
                {
                    throw new Exception("No output file specified");
                }
            }

            List<DoScores> threads = new List<DoScores>(iMyThreads.Count);
            for (int i = 0; i != iMyThreads.Count; i++)
            {
                DoScores ds = new DoScores();
                ds.iTotalThreads = iTotalThreads;
                ds.iMyThread = iMyThreads[i];
                if (null != base_output_filename)
                {
                    ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
                }
                System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
                thd.Name = "CFO_ToFofScores_" + i.ToString();
                ds.thd = thd;
                thd.Start(ds);
                threads.Add(ds);
            }
            for (int i = 0; i != iMyThreads.Count; i++)
            {
                threads[i].thd.Join();
                if (null != threads[i].sw)
                {
                    threads[i].sw.Close();
                }
            }
            if (null != outranges)
            {
                foreach (OutRange or in outranges)
                {
                    or.sw.Close();
                }
            }

            //LogLine("Total exceptions caught: " + exceptioncount.ToString());

            LogLine("---- RectangularFriendsLists processing done ----");
        }
Esempio n. 6
0
        void tofofscorespartthreadproc(object obj)
        {
            try
            {
                DoScores ds = (DoScores)obj;

                DHT htfof = new DHT(maxfics); // Friends of my friends.
                DHT htf   = new DHT();        // My friends, excluding from fof.

                // Start at row 1 instead of 0...
                for (int rowA = 1 + ds.iMyThread; rowA < realusercount; rowA += ds.iTotalThreads)
                {
                    Int32 uidA = GetUserIDFromRowID(rowA);
                    for (int colA = 1; colA < 1 + maxfriends; colA++)
                    {
                        Int32 fid = table[rowA, colA];
                        if (0 == fid)
                        {
                            break;
                        }
                        htf.Add(fid);
                    }
                    for (int colA = 1; colA < 1 + maxfriends; colA++)
                    {
                        Int32 fid = table[rowA, colA];
                        if (0 == fid)
                        {
                            break;
                        }
                        //int frow = GetRowIDFromUserID(fid); // Row index of fid. -1 if not found (0 is valid).
                        int frow = GetRowIDFromUserIDSameRange(fid, uidA); // Row index of fid in same id range as uidA. -1 if not found (0 is valid).
                        if (-1 != frow)
                        {
                            for (int fofcol = 1; fofcol != 1 + maxfriends; fofcol++) // fof column.
                            {
                                Int32 fofid = table[frow, fofcol];                   // fof user ID.
                                if (0 == fofid)
                                {
                                    break;
                                }
                                if (fofid != uidA && !htf.ContainsKey(fofid))
                                {
                                    // Fof only if not self and not one of my direct friends.
                                    htfof.Add(fofid, fid);
                                }
                            }
                        }
                    }
                    htf.DeepClean(); // !
                    // Count up friend dupes.
                    DHTSlot[] htslots = htfof.slots;
                    foreach (int i in htfof.dirtyslots)
                    {
                        int score = htslots[i].score; // 1 is base score.
                        if (score >= minscores)
                        {
                            if (null == ds.sw)
                            {
                                int ior = FindOutputRange(uidA);
                                if (-1 == ior)
                                {
                                    noranges++;
                                }
                                else
                                {
                                    lock (outranges[ior].sw)
                                    {
                                        int nfics = score;
                                        if (score > maxfics)
                                        {
                                            nfics = maxfics;
                                        }
                                        WriteOutputLine(outranges[ior].sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
                                    }
                                }
                            }
                            else
                            {
                                int nfics = score;
                                if (score > maxfics)
                                {
                                    nfics = maxfics;
                                }
                                WriteOutputLine(ds.sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
                            }
                        }
                        htfof.UnsetSlot(i); // Deep clean pt 1/2.
                    }
                    htfof.ZeroDirty();      // Deep clean pt 2/2.
                }

                if (null != ds.sw)
                {
                    ds.sw.Flush();
                }
            }
            catch (Exception e)
            {
                LogLine("\r\nProcessing error: tofofscorespartthreadproc catch-all exception: " + e.ToString() + "\r\n");
            }
        }
Esempio n. 7
0
        // fofmax is max number of friends-of-friends.
        // fofhashsize should be prime near (fofmax * 10)
        // fhashsize should be prime near (MaxFriends * 10)
        public void ToFofScores(int iTotalThreads, IList<int> iMyThreads, string base_output_filename, int fofmax, int fofhashsize, int fhashsize)
        {
            LogLine("---- JaggedFriendsLists.ToFofScores ----");

            List<DoScores> threads = new List<DoScores>(iMyThreads.Count);
            for (int i = 0; i != iMyThreads.Count; i++)
            {
                DoScores ds = new DoScores();
                ds.iTotalThreads = iTotalThreads;
                ds.iMyThread = iMyThreads[i];
                ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
                ds.fofmax = fofmax;
                ds.fofhashsize = fofhashsize;
                ds.fhashsize = fhashsize;
                System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
                thd.Name = "CFO_ToFofScores_" + i.ToString();
                ds.thd = thd;
                thd.Start(ds);
                threads.Add(ds);
            }
            for (int i = 0; i != iMyThreads.Count; i++)
            {
                threads[i].thd.Join();
                threads[i].sw.Close();
            }

            //LogLine("Total exceptions caught: " + exceptioncount.ToString());

            LogLine("---- JaggedFriendsLists processing done ----");
        }
        // Note: iTotalThreads is >= ThreadCount; it is considering multiple passes.
        public void ToFofScores(int iTotalThreads, IList <int> iMyThreads, string base_output_filename)
        {
            LogLine("---- RectangularFriendsListsHugeMemory.ToFofScores ----");

            if (null == base_output_filename)
            {
                if (null == outranges)
                {
                    throw new Exception("No output file specified");
                }
            }

            if (iMyThreads.Count != nthreads)
            {
                throw new Exception("ToFofScores: iMyThreads.Count is not the same as ThreadCount");
            }

            List <DoScores> threads = new List <DoScores>(nthreads);

            for (int i = 0; i != iMyThreads.Count; i++)
            {
                DoScores ds = new DoScores();
                ds.tv            = smtable.ThreadViews[i];
                ds.iTotalThreads = iTotalThreads;
                ds.iMyThread     = iMyThreads[i];
                if (null != base_output_filename)
                {
                    ds.sw = System.IO.File.CreateText(base_output_filename + ".thread" + ds.iMyThread.ToString() + ".txt");
                }
                System.Threading.Thread thd = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(tofofscorespartthreadproc));
                thd.Name = "CFO_ToFofScores_" + i.ToString();
                ds.thd   = thd;
                thd.Start(ds);
                threads.Add(ds);
            }
            for (int i = 0; i != iMyThreads.Count; i++)
            {
                threads[i].thd.Join();
                if (null != threads[i].sw)
                {
                    threads[i].sw.Close();
                }
            }
            if (null != outranges)
            {
                foreach (OutRange or in outranges)
                {
                    or.sw.Close();
                }
            }
            if (null != goterr)
            {
                Exception e = goterr;
                goterr = null;
                throw e;
            }

#if DEBUG
            if (smtable._batchspilled)
            {
                LogLine("RectangularFriendsListsHugeMemory.ToFofScores: SlaveMemory batching spilled to another slave");
                smtable._batchspilled = false;
            }
#endif

            //LogLine("Total exceptions caught: " + exceptioncount.ToString());

            LogLine("---- RectangularFriendsListsHugeMemory processing done ----");
        }
        void tofofscorespartthreadproc(object obj)
        {
            try
            {
                ThreadBatchData        fbatchdata   = new ThreadBatchData(this, nbatchedrows);
                ThreadBatchData        fofbatchdata = new ThreadBatchData(this, nbatchedrows);
                DoScores               ds           = (DoScores)obj;
                SlaveMemory.ThreadView tv           = ds.tv;

                Int32[] friendsbuf = new Int32[maxfriends];
                Int32[] afriends   = new Int32[maxfriends];

                DHT htfof = new DHT(maxfics); // Friends of my friends.
                DHT htf   = new DHT();        // My friends, excluding from fof.

                // Start at row 1 instead of 0...
                int rowA      = 1 + ds.iMyThread;
                int rowAstart = rowA;
                int ifbatch   = 0;
                for ( ; ; rowA += ds.iTotalThreads)
                {
                    if (ifbatch >= nbatchedrows ||
                        rowA >= realusercount)
                    {
                        fbatchdata.BatchGet(tv);
                        for (int br = 0; br < ifbatch; br++)
                        {
#if DEBUG
                            for (int i = 0; i < maxfriends; i++)
                            {
                                afriends[i] = -929292;
                            }
#endif
                            Int32 uidA = fbatchdata.GetRowData(br, afriends);
                            for (int ifriend = 0; ifriend < maxfriends; ifriend++)
                            {
                                Int32 fid = afriends[ifriend];
                                if (0 == fid)
                                {
                                    break;
                                }
#if DEBUG
                                if (-929292 == fid)
                                {
                                    throw new Exception("fid -929292");
                                }
#endif
                                htf.Add(fid);
                            }
                            {
                                int ibatch  = 0;
                                int ifriend = 0;
                                //int ifriendstart = 0;
                                for (; ; ifriend++)
                                {
                                    if (ibatch >= nbatchedrows ||
                                        ifriend >= maxfriends ||
                                        0 == afriends[ifriend])
                                    {
                                        fofbatchdata.BatchGet(tv);
                                        for (int bfriend = 0; bfriend < ibatch; bfriend++)
                                        {
#if DEBUG
                                            for (int i = 0; i < maxfriends; i++)
                                            {
                                                friendsbuf[i] = -929292;
                                            }
#endif
                                            int xfid = fofbatchdata.GetRowData(bfriend, friendsbuf);
                                            //int brealifriend = ifriendstart + bfriend;
                                            for (int ifof = 0; ifof < maxfriends; ifof++)
                                            {
                                                Int32 fofid = friendsbuf[ifof];
                                                if (0 == fofid)
                                                {
                                                    break;
                                                }
#if DEBUG
                                                if (-929292 == fofid)
                                                {
                                                    throw new Exception("fofid -929292");
                                                }
#endif
                                                if (fofid != uidA && !htf.ContainsKey(fofid))
                                                {
#if DEBUG
                                                    if (uidA == 17 && fofid == 10)
                                                    {
                                                        int i33 = 33 + 33;
                                                    }
#endif
                                                    // Fof only if not self and not one of my direct friends.
                                                    htfof.Add(fofid, xfid);
                                                }
                                            }
                                        }
                                        if (ifriend >= maxfriends ||
                                            0 == afriends[ifriend])
                                        {
                                            break;
                                        }
                                        ibatch = 0;
                                        //ifriendstart = ifriend + 1;
                                    }

                                    Int32 fid = afriends[ifriend];
                                    if (0 == fid)
                                    {
                                        break;
                                    }
#if DEBUG
                                    if (-929292 == fid)
                                    {
                                        throw new Exception("fid -929292");
                                    }
#endif
                                    //int frow = GetRowIDFromUserID(fid); // Row index of fid. -1 if not found (0 is valid).
                                    int frow = GetRowIDFromUserIDSameRange(fid, uidA); // Row index of fid in same id range as uidA. -1 if not found (0 is valid).
                                    if (-1 != frow)
                                    {
                                        fofbatchdata.BatchRow(ibatch++, frow);
                                    }
                                }
                            }

                            htf.DeepClean(); // !
                            // Count up friend dupes.
                            DHTSlot[] htslots = htfof.slots;
                            foreach (int i in htfof.dirtyslots)
                            {
                                int score = htslots[i].score; // 1 is base score.
                                if (score >= minscores)
                                {
                                    if (null == ds.sw)
                                    {
                                        int ior = FindOutputRange(uidA);
                                        if (-1 == ior)
                                        {
                                            noranges++;
                                        }
                                        else
                                        {
                                            lock (outranges[ior].sw)
                                            {
                                                int nfics = score;
                                                if (score > maxfics)
                                                {
                                                    nfics = maxfics;
                                                }
                                                WriteOutputLine(outranges[ior].sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
                                            }
                                        }
                                    }
                                    else
                                    {
                                        int nfics = score;
                                        if (score > maxfics)
                                        {
                                            nfics = maxfics;
                                        }
                                        WriteOutputLine(ds.sw, uidA, htslots[i].key, score, htslots[i].values, nfics);
                                    }
                                }
                                htfof.UnsetSlot(i); // Deep clean pt 1/2.
                            }
                            htfof.ZeroDirty();      // Deep clean pt 2/2.
                        }

                        if (rowA >= realusercount)
                        {
                            break;
                        }
                        ifbatch   = 0;
                        rowAstart = rowA + ds.iTotalThreads;
                    }

                    fbatchdata.BatchRow(ifbatch++, rowA);
                }

                if (null != ds.sw)
                {
                    ds.sw.Flush();
                }
            }
            catch (Exception e)
            {
                if (null == goterr)
                {
                    goterr = new Exception("Processing error: tofofscorespartthreadproc catch-all exception", e);
                }
                LogLine("\r\nProcessing error: tofofscorespartthreadproc catch-all exception: " + e.ToString() + "\r\n");
            }
        }