/// <summary>
/// Main method for web crawler.
/// <para>
/// Argument 1 is log level.
/// Argument 2 is word limit.
/// Argument 3 is list of words separated by commas to be excluded in the crawler.
/// There cannot be spaces between the words unless the list is in " ".
/// </para>
/// </summary>
///
/// <param name="args">
/// </param>
static void Main(string[] args)
{
// get the command line args
int wordLimit = 10;
List <string> excludedWords = new List <string>();
string level = "Information";
if (args.Length > 0)
{
if (args.Length >= 1)
{
var arg = args[0].Trim();
level = arg;
}
if (args.Length >= 2)
{
var arg = args[1].Trim();
if (!Int32.TryParse(arg, out wordLimit))
{
Log.Logger.Debug($"Could not set word limit to desired configuration {arg}.");
}
}
if (args.Length >= 3)
{
var arg = args[2].Trim().Replace(" ", ",");
excludedWords = arg.TokenizeToList(",");
}
}
// configure the logger
LogHelper.ConfigureLogger(level);
Log.Logger.Information($"Starting Crawler with configurations: Log_Level={level}, Word_Limit={wordLimit}, Excluded_Words={excludedWords.ToPrettyString()}");
// start the crawler
CrawlerResult result = new CrawlerResult();
using (var crawler = new Crawler(wordLimit, excludedWords))
{
Task.Run(async() =>
{
result = await crawler.CrawlAsync();
}).Wait();
}
if (result.Error != null)
{
Log.Logger.Error($"Crawl failed with Error: {result.Error.Message}\n{result.Error.StackTrace}");
return;
}
Log.Logger.Information($"\n***MOST FREQUENT*** \n{result.Words.ToPrettyString()}");
}
private async void BtnStart_Click(object sender, RoutedEventArgs e)
{
LblMessage.Content = string.Empty;
if (!ValidateInput())
{
LblMessage.Content = "Source URL or Nesting level are invalid!";
return;
}
SwitchButtons(true);
if (_cancellationSource != null)
{
_cancellationSource.Dispose();
}
_cancellationSource = new CancellationTokenSource();
string sourceUrl = TxtUrl.Text;
string searchPhrase = TxtPhrase.Text;
int nestingLevel = Convert.ToInt32(TxtNestingLevel.Text);
_crawler.NestingLevel = nestingLevel;
_crawler.SearchPhrase = searchPhrase;
try
{
bool configureAwait = true;
await _crawler.CrawlAsync(sourceUrl, configureAwait, _cancellationSource.Token).ConfigureAwait(configureAwait);
}
catch (OperationCanceledException ex)
{
Debug.WriteLine(ex.Message);
}
SwitchButtons(false);
}