static void Main(string[] args)
{
var sw = new Stopwatch();
sw.Start();
try
{
var options = new Options();
if (CommandLine.Parser.Default.ParseArguments(args, options))
{
var processor = new Processor();
if (options.SiteAnalysisMode)
{
processor.AnalyzeWebSites(options).Wait();
}
else
{
processor.AnalyzeWebPages(options).Wait();
}
}
}
catch (AggregateException ex)
{
foreach (var exception in ex.InnerExceptions)
{
Console.Error.WriteLine(exception.Message);
}
}
sw.Stop();
Console.WriteLine(sw.ElapsedMilliseconds);
}
public async Task AnalyzeWebPages(Options options)
{
Cleanup(options);
var tasks = options.Urls
.Select(UrlHelper.CreateInvariantToScheme)
.Select(uri => Task.Run(async () =>
{
var analyzer = new PageAnalyzer(_cssStylesheets);
try
{
var htmlDocument = await GetHtml(uri);
var analysisResult = await analyzer.Analyze(uri, htmlDocument);
WriteResults(analysisResult, options);
}
catch (UnsupportedContentTypeException ex)
{
}
})).ToList();
await Task.WhenAll(tasks);
}
public async Task AnalyzeWebSites(Options options)
{
Cleanup(options);
var analyzedPages = new ConcurrentDictionary<string, Uri>();
var pageUrl = UrlHelper.CreateInvariantToScheme(options.Urls[0]);
analyzedPages.TryAdd(pageUrl.ToString(), pageUrl);
await AnalyzePage(pageUrl, analyzedPages, options);
}
private async Task AnalyzePage(Uri pageUrl, ConcurrentDictionary<string, Uri> analyzedPages, Options options)
{
PageAnalysisResult analysisResult;
try
{
analysisResult = await GetPageAnalysis(pageUrl);
}
catch (UnsupportedSelectorException ex)
{
return;
}
WriteResults(analysisResult, options);
foreach (var internalLink in analysisResult.InternalLinks)
{
if (analyzedPages.Count < options.MaximumPages || options.MaximumPages == 0)
{
if (options.MaximumDepth != 0)
{
if (internalLink.Segments.Length > options.MaximumDepth)
return;
}
if (analyzedPages.TryAdd(internalLink.ToString(), internalLink))
{
await AnalyzePage(internalLink, analyzedPages, options);
}
}
}
}
private void WriteResults(PageAnalysisResult pageAnalysisResult, Options options)
{
var formattedResults = FormatResults(pageAnalysisResult);
Console.Write(formattedResults);
if (!String.IsNullOrWhiteSpace(options.OutputFile))
File.AppendAllText(options.GetOutputFileName(), formattedResults);
}
private void Cleanup(Options options)
{
if (!String.IsNullOrWhiteSpace(options.OutputFile))
{
File.Create(options.GetOutputFileName());
}
}