public TfIdfCalculator(IDocumentFrequencyProvider dfProvider) { if (dfProvider == null) { throw new ArgumentNullException("dfProvider"); } _dfProvider = dfProvider; }
private static void Main(string[] args) { var htmlCleaner = new HtmlCleaner(new ISiteHtmlCleaner[] { new DailyHtmlCleaner(), new MirrorHtmlCleaner() }); var htmlLoader = new HtmlLoader(); var articleProvider = new ArticleProvider(htmlCleaner, htmlLoader); var cosineSimilarityCalculator = new CosineSimilarityCalculator(); IDocumentFrequencyProvider dfProvider = LoadFrequencies(); var tfIdfCalculator = new TfIdfCalculator(dfProvider); var tokenizer = new Tokenizer(); var articleProcessor = new ArticleProcessor(tfIdfCalculator, tokenizer); var articleComparer = new TextProcessing.ArticleComparer(articleProvider, cosineSimilarityCalculator, articleProcessor); Console.WriteLine("Similar articles:"); double similarity = articleComparer.Compare( @"http://www.dailymail.co.uk/news/article-2489957/Britains-spy-chiefs-grilled-MPs-television-time.html", @"http://www.mirror.co.uk/news/uk-news/mi6-mi5-gchq-bosses-questioned-2685310"); Console.WriteLine(similarity); similarity = articleComparer.Compare( @"http://www.dailymail.co.uk/news/article-2489640/80-parents-caught-children-copying-p**n-style-dances-offensive-lyrics.html", @"http://www.mirror.co.uk/news/uk-news/miley-cyrus-twerking-kids-copying-2685363"); Console.WriteLine(similarity); Console.WriteLine("Same article:"); similarity = articleComparer.Compare( @"http://www.dailymail.co.uk/news/article-2490296/You-STILL-likely-lose-job-recession-25s-shop-workers-risk.html", @"http://www.dailymail.co.uk/news/article-2490296/You-STILL-likely-lose-job-recession-25s-shop-workers-risk.html"); Console.WriteLine(similarity); Console.WriteLine("Different articles:"); similarity = articleComparer.Compare( @"http://www.dailymail.co.uk/femail/article-2489984/Needy-people-likely-cheat.html", @"http://www.dailymail.co.uk/news/article-2490531/Worlds-oldest-paperboy-deliver-round-71-years-route.html"); Console.WriteLine(similarity); similarity = articleComparer.Compare( @"http://www.dailymail.co.uk/news/article-2490412/Wikileaks-journalist-spent-4-months-Edward-Snowden-leaves-Russia.html", @"http://www.dailymail.co.uk/news/article-2489994/Twitter-share-prices-soar-firms-day-trading.html"); Console.WriteLine(similarity); Console.ReadKey(); }