public static void TestTokenSearcher(Index indexer, string searchstring, int documents)
{
var ts = new TokenSearcher(indexer);
var tokens = Tokenizor.StringToToken(searchstring, indexer.Tokens).ToList();
var searchres = ts.OrderByScore(tokens, ts.And(tokens), documents).OrderByDescending(x => x.Item1);
Console.WriteLine("Found {0} results for \"{1}\":", searchres.Count(), searchstring);
foreach (var t in searchres.Take(50))
{
Console.WriteLine("{0,10:N7}: {1}", t.Item1, t.Item2.Url);
}
}
static void Main(string[] args)
{
var websiteUrls = new List <Uri>();
//websiteUrls.Add(@"https://en.wikipedia.org");
//websiteUrls.Add(@"https://www.satai.dk");
//websiteUrls.Add(@"https://www.google.dk");
//websiteUrls.Add(@"http://stackoverflow.com");
//websiteUrls.Add(@"https://news.ycombinator.com");
//websiteUrls.Add(@"https://twitter.com");
//websiteUrls.Add(@"http://www.mmo-champion.com");
//websiteUrls.Add(@"http://www.imdb.com");
//websiteUrls.Add(@"https://www.instagram.com");
//websiteUrls.Add(@"https://www.youtube.com");
var seedUrl = new List <Uri>
{
new Uri("http://dr.dk"),
new Uri("https://en.wikipedia.org"),
new Uri("https://news.ycombinator.com"),
new Uri("http://www.mmo-champion.com"),
new Uri("https://msdn.microsoft.com")
};
var crawler = SerializationHelper.RestoreCrawler(seedUrl, new List <Host>(), 1000);
if (crawler.Limit >= crawler.SitesVisited.Count)
{
Task.Run(() => crawler.Run()).GetAwaiter().GetResult();
SerializationHelper.SaveCrawler(crawler);
}
var indexer = new Index();
var sw = Stopwatch.StartNew();
foreach (var site in crawler.SitesVisited.Values)
{
Tokenizor.AddTokensToTokenList(site, indexer.Tokens);
}
Console.WriteLine("Indexing {0} documents took {1} ms ({2} pr. sec)", crawler.SitesVisited.Count, sw.ElapsedMilliseconds, (double)crawler.SitesVisited.Count / (sw.ElapsedMilliseconds / 1000));
//SerializationHelper.SaveIndex(indexer);
//TestTFIDF(indexer);
//TestBooleanSearcher(indexer);
//TestTokenSearcher(indexer);
string s;
do
{
Console.Write("Enter search Query: ");
s = Console.ReadLine();
if (s == null)
{
continue;
}
TestTokenSearcher(indexer, s, crawler.SitesVisited.Count);
} while (s != "");
//Console.Write("Enter search Query: ");
//while ((s = Console.ReadLine()) != "")
//{
// if(s == null)
// continue;
// TestTokenSearcher(indexer, s);
// Console.Write("Enter search Query: ");
//}
}
