/// <summary> /// Main method for web crawler. /// <para> /// Argument 1 is log level. /// Argument 2 is word limit. /// Argument 3 is list of words separated by commas to be excluded in the crawler. /// There cannot be spaces between the words unless the list is in " ". /// </para> /// </summary> /// /// <param name="args"> /// </param> static void Main(string[] args) { // get the command line args int wordLimit = 10; List <string> excludedWords = new List <string>(); string level = "Information"; if (args.Length > 0) { if (args.Length >= 1) { var arg = args[0].Trim(); level = arg; } if (args.Length >= 2) { var arg = args[1].Trim(); if (!Int32.TryParse(arg, out wordLimit)) { Log.Logger.Debug($"Could not set word limit to desired configuration {arg}."); } } if (args.Length >= 3) { var arg = args[2].Trim().Replace(" ", ","); excludedWords = arg.TokenizeToList(","); } } // configure the logger LogHelper.ConfigureLogger(level); Log.Logger.Information($"Starting Crawler with configurations: Log_Level={level}, Word_Limit={wordLimit}, Excluded_Words={excludedWords.ToPrettyString()}"); // start the crawler CrawlerResult result = new CrawlerResult(); using (var crawler = new Crawler(wordLimit, excludedWords)) { Task.Run(async() => { result = await crawler.CrawlAsync(); }).Wait(); } if (result.Error != null) { Log.Logger.Error($"Crawl failed with Error: {result.Error.Message}\n{result.Error.StackTrace}"); return; } Log.Logger.Information($"\n***MOST FREQUENT*** \n{result.Words.ToPrettyString()}"); }
private async void BtnStart_Click(object sender, RoutedEventArgs e) { LblMessage.Content = string.Empty; if (!ValidateInput()) { LblMessage.Content = "Source URL or Nesting level are invalid!"; return; } SwitchButtons(true); if (_cancellationSource != null) { _cancellationSource.Dispose(); } _cancellationSource = new CancellationTokenSource(); string sourceUrl = TxtUrl.Text; string searchPhrase = TxtPhrase.Text; int nestingLevel = Convert.ToInt32(TxtNestingLevel.Text); _crawler.NestingLevel = nestingLevel; _crawler.SearchPhrase = searchPhrase; try { bool configureAwait = true; await _crawler.CrawlAsync(sourceUrl, configureAwait, _cancellationSource.Token).ConfigureAwait(configureAwait); } catch (OperationCanceledException ex) { Debug.WriteLine(ex.Message); } SwitchButtons(false); }