コード例 #1
0
 public Spider(Frontier frontier, Index index, Filtering.Filter filter, Action<Index> callback)
 {
     this.frontier = frontier;
     this.index = Index.CreateEmptyCopy(index);
     this.filter = filter;
     this.callback = callback;
 }
コード例 #2
0
        public void MergeIn(Index index)
        {
            List<Document> unique = new List<Document>(index.sites);

            for (int i = 0; i < unique.Count; i++)
                foreach (var s in sites)
                {
                    double simi = similarity.CalculateSimilarity(s, unique[i]);
                    if (simi >= 0.9)
                    {
                        unique.RemoveAt(i--);
                        break;
                    }
                }

            foreach (var doc in unique)
                sites.Add(doc);

            foreach (var term in index.stems.Keys)
            {
                if (!stems.ContainsKey(term))
                    stems.Add(term, index.stems[term]);
                else
                    stems[term].MergeInto(index.stems[term], (a, b) => a.Document.Id.CompareTo(b.Document.Id), d => !unique.Contains(d.Document));
            }
        }
コード例 #3
0
        public Ranker(Index index, TermStemmer stemmer)
        {
            int docCount = index.SiteCount;

            foreach (var t in index.GetStems())
                tf_idf.Add(t.Key, getTF_IDF(t.Value, docCount));

            Document[] keys = lengths.Keys.ToArray();
            foreach (var d in keys)
                lengths[d] = Math.Sqrt(lengths[d]);

            this.index = index;
            this.stemmer = stemmer;
            //this.TF_WT = getTF_WT();
            //this.IDF_WT = getIDF_WT();
            //this.TF_IDF_WT = getTF_IDF_WT();
            //this.NORM_WT = getNORM_WT();
        }
コード例 #4
0
        public static void StartAndWait(Frontier frontier, Index index, Filtering.Filter filter, int pagecount)
        {
            int count = (int)Math.Ceiling(pagecount / (double)SPIDER_PAGE_COUNT);

            Spider[] spiders = new Spider[count];
            Thread[] threads = new Thread[count];

            for (int i = 0; i < count; i++)
            {
                Spider sp = spiders[i] = new Spider(frontier, index, filter, ind =>
                {
                    Console.ForegroundColor = ConsoleColor.Cyan;
                    Console.WriteLine("Merging Index of {0}", ind.SiteCount);
                    Console.ForegroundColor = ConsoleColor.Gray;

                    lock (index) { index.MergeIn(ind); }
                });
                threads[i] = new Thread(() => sp.Run());
                threads[i].Start();
            }

            for (int i = 0; i < count; i++)
                threads[i].Join();
        }
コード例 #5
0
 public static Index CreateEmptyCopy(Index copyFrom)
 {
     return new Index(copyFrom.stemmer, copyFrom.similarity);
 }
コード例 #6
0
 public static Index CreateEmptyCopy(Index copyFrom)
 {
     return(new Index(copyFrom.stemmer, copyFrom.similarity));
 }