Esempio n. 1
0
        public void AHrefWithBase(string baseUrl, string href, string expected)
        {
            var html = string.Format("<html><head><base href='{0}'></head><body><div><a href='{1}'>a</a></div></body></html>", baseUrl, href);
            var links = new LinkExtractor().ExtractLinks(new Uri("http://y.com"), Read(html), MediaTypeNames.Text.Html);

            Assert.AreElementsEqualIgnoringOrder(new[] { expected }, links.Select(l => l.ToString()));
        }
Esempio n. 2
0
        public void SimpleAHref(string url, string href, string expected)
        {
            var html = string.Format("<html><body><div><a href='{0}'>a</a></div></body></html>", href);
            var links = new LinkExtractor().ExtractLinks(new Uri(url), Read(html), MediaTypeNames.Text.Html);

            Assert.AreElementsEqualIgnoringOrder(new[] { expected }, links.Select(l => l.ToString()));
        }
Esempio n. 3
0
        private static void Main(ApplicationArguments arguments)
        {
            EnsureConsoleWindowHandle();
            if (arguments.IgnoreCertificateValidation)
                ServicePointManager.ServerCertificateValidationCallback = (x1, x2, x3, x4) => true;

            Console.Title = "LightGet";
            Console.CursorVisible = false;

            var loggerForDownloader = new LoggerForDownloader { Prefix = ConsoleIndent };

            var mapper = new UrlToPathMapper(arguments.OutputPathFormat, arguments.Url);
            var directory = new DirectoryInfo(Directory.GetCurrentDirectory());

            var downloader = new Downloader(loggerForDownloader, (url, fileName) => new FileInfo(Path.Combine(directory.FullName, mapper.GetPath(url, fileName))));
            var extractor = new LinkExtractor();

            var credentials = arguments.User != null ? new NetworkCredential(arguments.User, arguments.Password) : null;
            var visited = new HashSet<Uri>();

            var queue = new Queue<Uri>();
            queue.Enqueue(arguments.Url);

            while (queue.Count > 0) {
                var url = queue.Dequeue();
                ConsoleUI.WriteLine(ConsoleColor.White, url);

                DownloaderResult result;
                try {
                    result = Download(downloader, url, credentials);
                }
                catch (Exception ex) {
                    ConsoleUI.WriteLine(ConsoleColor.Red, ex.Message);
                    continue;
                }
                finally {
                    Console.WriteLine();
                }

                visited.Add(url);
                if (arguments.FollowLinks == LinkFollowingRule.None)
                    break;

                using (var reader = result.File.OpenText()) {
                    var links = extractor.ExtractLinks(result.Url, reader, result.ContentType);
                    foreach (var link in links.Where(l => !visited.Contains(l))) {
                        if (!ShouldFollow(link, arguments))
                            continue;

                        queue.Enqueue(link);
                    }
                }
            }

            Console.CursorVisible = true;
        }