public static async Task <Dictionary <string, string[]> > Scrape(string input)
        {
            var urlStandardiser = new UrlStandardiser(input);
            var linkExtractor   = new SameDomainLinkExtractor(urlStandardiser);

            var scrape = new Dictionary <string, string[]>();

            var url = urlStandardiser.Standardise(input);

            var html = await HtmlDownloader.GetHtml(url);

            var links = linkExtractor.Extract(html.Text);

            scrape[url] = links;

            foreach (var link in links)
            {
                if (scrape.ContainsKey(link))
                {
                    Console.WriteLine("URL already scraped: " + link);
                }
                else
                {
                    var html2 = await HtmlDownloader.GetHtml(link);

                    if (html2 == null)
                    {
                        Console.WriteLine("Not valid HTML: " + link);
                        scrape[link] = Array.Empty <string>();
                    }
                    else
                    {
                        var links2 = linkExtractor.Extract(html2.Text);
                        scrape[link] = links2;
                        Console.WriteLine("scrape[" + link + "]: " + string.Join(',', links2));
                    }
                }
            }

            return(scrape);
        }
 public SameDomainLinkExtractor(UrlStandardiser urlStandardiser)
 {
     UrlStandardiser = urlStandardiser;
 }