/// <summary>
        /// Returns Apple Touch Icons for a Uri.
        /// </summary>
        /// <param name="uri">Uri to explore.</param>
        /// <param name="doc">Instance of <see cref="HtmlDocument"/> containing parsed Html for the Uri.</param>
        /// <param name="settings">Extraction settings.</param>
        /// <returns>List of Apple Touch Icons as instances of <see cref="WebImage"/>.</returns>
        public static async Task <List <WebImage> > GetAppleTouchIcons(Uri uri, HtmlDocument doc, ExtractionSettings settings)
        {
            List <WebImage> arr = new List <WebImage>();

            // Check default apple touch icon location
            if (!settings.SvgOnly)
            {
                Uri atiUri = uri.AddHtmlLink(Constants.DefaultAppleTouchIconPath);
                if (await ImageDownloader.IsOkUri(atiUri))
                {
                    if (!Recurser.FoundUris.Contains(atiUri.ToString()))
                    {
                        Recurser.FoundUris.Add(atiUri.ToString());
                        arr.Add(new WebImage(atiUri.ToString(), false, true));
                    }
                }
            }

            // Parse HTML Tree
            if (doc == null)
            {
                return(arr);
            }

            // Get icons through link info
            IEnumerable <HtmlLinkTag> htmlLinkTags = HtmlExtractor.ExtractLinks(doc);

            if (htmlLinkTags != null)
            {
                foreach (HtmlLinkTag link in htmlLinkTags)
                {
                    if (link.IsAppleTouchIcon())
                    {
                        Uri absUri = uri.AddHtmlLink(link.Link);
                        if (settings.SvgOnly && !absUri.HasSvgExtension())
                        {
                            continue;
                        }

                        if (!settings.SvgOnly)
                        {
                            if (!absUri.HasImageExtension())
                            {
                                continue;
                            }
                        }

                        if (Recurser.FoundUris.Contains(absUri.ToString()))
                        {
                            continue;
                        }

                        if (await ImageDownloader.IsOkUri(absUri))
                        {
                            Recurser.FoundUris.Add(absUri.ToString());
                            arr.Add(new WebImage(absUri.ToString(), false, true));
                        }
                    }
                }
            }

            return(arr);
        }
        /// <summary>
        /// Returns images for a Uri (excluding Favicons and Apple Touch Icons).
        /// </summary>
        /// <param name="uri">Uri to explore.</param>
        /// <param name="doc">Instance of <see cref="HtmlDocument"/> containing parsed Html for the Uri.</param>
        /// <param name="settings">Extraction settings.</param>
        /// <returns>List of images as instances of <see cref="WebImage"/>.</returns>
        public static async Task <List <WebImage> > GetPageImages(Uri uri, HtmlDocument doc, ExtractionSettings settings)
        {
            List <WebImage> arr = new List <WebImage>();

            if (doc == null)
            {
                return(arr);
            }

            if (settings.GetLinkTagImages)
            {
                IEnumerable <HtmlLinkTag> htmlLinkTags = HtmlExtractor.ExtractLinks(doc);
                if (htmlLinkTags != null)
                {
                    foreach (HtmlLinkTag link in htmlLinkTags)
                    {
                        if (!link.IsFavicon() && !link.IsAppleTouchIcon())
                        {
                            Uri absUri = uri.AddHtmlLink(link.Link);
                            if (settings.SvgOnly && !absUri.HasSvgExtension())
                            {
                                continue;
                            }

                            if (!settings.SvgOnly)
                            {
                                if (!absUri.HasImageExtension())
                                {
                                    continue;
                                }
                            }

                            if (Recurser.FoundUris.Contains(absUri.ToString()))
                            {
                                continue;
                            }

                            if (await ImageDownloader.IsOkUri(absUri))
                            {
                                Recurser.FoundUris.Add(absUri.ToString());
                                arr.Add(new WebImage(absUri.ToString(), false, false));
                            }
                        }
                    }
                }
            }

            if (settings.GetMetaTagImages)
            {
                IEnumerable <HtmlMetaTag> htmlMetaTags = HtmlExtractor.ExtractMetadata(doc);
                if (htmlMetaTags != null)
                {
                    foreach (HtmlMetaTag meta in htmlMetaTags)
                    {
                        Uri absUri = uri.AddHtmlLink(meta.Content);
                        if (settings.SvgOnly && !absUri.HasSvgExtension())
                        {
                            continue;
                        }

                        if (!settings.SvgOnly)
                        {
                            if (!absUri.HasImageExtension())
                            {
                                continue;
                            }
                        }

                        if (Recurser.FoundUris.Contains(absUri.ToString()))
                        {
                            continue;
                        }

                        if (await ImageDownloader.IsOkUri(absUri))
                        {
                            Recurser.FoundUris.Add(absUri.ToString());
                            arr.Add(new WebImage(absUri.ToString(), false, false));
                        }
                    }
                }
            }

            if (settings.GetInlineBackgroundImages)
            {
                // Inline background images
                HtmlNode bodyNode = doc.DocumentNode.SelectSingleNode("//body");
                if (bodyNode != null)
                {
                    await TraverseNode(bodyNode, async (HtmlNode node) =>
                    {
                        string styleValue = node.GetAttributeValue("style", null);
                        if (!string.IsNullOrEmpty(styleValue))
                        {
                            MatchCollection matches = Regex.Matches(styleValue, @"(?<=url\()(.*)(?=\))");
                            foreach (Match match in matches)
                            {
                                string value = match.Value;
                                if (!string.IsNullOrEmpty(value))
                                {
                                    if ((value[0] == '\'' && value[value.Length - 1] == '\'') || (value[0] == '\"' && value[value.Length - 1] == '\"'))
                                    {
                                        value = value.Substring(1, value.Length - 2);
                                    }

                                    Uri link = uri.AddHtmlLink(value);
                                    if (settings.SvgOnly && !link.HasSvgExtension())
                                    {
                                        continue;
                                    }

                                    if (!settings.SvgOnly)
                                    {
                                        if (!link.HasImageExtension())
                                        {
                                            continue;
                                        }
                                    }

                                    if (Recurser.FoundUris.Contains(link.ToString()))
                                    {
                                        continue;
                                    }

                                    if (await ImageDownloader.IsOkUri(link))
                                    {
                                        Recurser.FoundUris.Add(link.ToString());
                                        arr.Add(new WebImage(link.ToString(), false, false, true));
                                    }
                                }
                            }
                        }
                    });
                }

                // TODO: Add Option to get background images from CSS files
            }

            IEnumerable <HtmlImgTag> imgTags = HtmlExtractor.ExtractImages(doc);

            if (imgTags != null)
            {
                foreach (HtmlImgTag img in imgTags)
                {
                    Uri absUri = uri.AddHtmlLink(img.Src);
                    if (settings.SvgOnly && !absUri.HasSvgExtension())
                    {
                        continue;
                    }

                    if (absUri.IsBadMagickType())
                    {
                        continue;
                    }

                    if (Recurser.FoundUris.Contains(absUri.ToString()))
                    {
                        continue;
                    }

                    if (await ImageDownloader.IsOkUri(absUri))
                    {
                        Recurser.FoundUris.Add(absUri.ToString());
                        arr.Add(new WebImage(absUri.ToString(), false, false));
                    }
                }
            }

            return(arr);
        }
Exemple #3
0
        /// <summary>
        /// Extracts images for a page and linked pages.
        /// </summary>
        /// <param name="method">Method which extracts images from speicific Uri.</param>
        /// <param name="uri">Uri to extract images and start hyperlink recursion from.</param>
        /// <param name="settings">Extraction Settings.</param>
        /// <param name="depth">Depth to recurse hyperlinks to.</param>
        /// <param name="cancellationToken">Cancellation Token.</param>
        /// <returns>Returns extracted images for given Uri and linked pages.</returns>
        public static async Task <List <WebImage> > HyperlinkRecurse(ExtractionMethod method, Uri uri, ExtractionSettings settings, int depth, CancellationToken cancellationToken)
        {
            HtmlDocument    doc    = null;
            bool            gotDoc = false;
            List <WebImage> images = new List <WebImage>();

            if (!exploredUris.Contains(uri.ToString()))
            {
                exploredUris.Add(uri.ToString());

                if (settings.OnStartNewPage != null)
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        return(null);
                    }

                    await settings.OnStartNewPage.Invoke(uri.ToString());
                }

                if (cancellationToken.IsCancellationRequested)
                {
                    return(null);
                }

                gotDoc = true;
                doc    = await GetDocumnent(uri, cancellationToken);

                images = await method.Invoke(uri, doc, settings);

                if (cancellationToken.IsCancellationRequested)
                {
                    return(null);
                }

                if (settings.OnEndNewPage != null)
                {
                    await settings.OnEndNewPage.Invoke(uri.ToString(), images);
                }

                if (settings.ShouldStopOnFoundImage != null)
                {
                    // Take all images up to the point where should stop
                    int index = images.TakeWhile(i => !settings.ShouldStopOnFoundImage.Invoke(i)).Count();
                    if (index != images.Count)
                    {
                        images.RemoveRange(index + 1, images.Count - index - 1);
                        stopAlg = true;
                    }
                }

                if (!settings.LazyDownload)
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        return(null);
                    }

                    await Task.WhenAll(images.Select(i => i.GetImageAsync(cancellationToken)).ToArray());
                }

                if (settings.OnFoundImage != null)
                {
                    images.ForEach(i => settings.OnFoundImage.Invoke(i));
                }
            }

            if (!stopAlg && settings.RecurseHyperlinks && depth < settings.HyperlinkRecursionDepth)
            {
                if (!gotDoc)
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        return(null);
                    }

                    doc = await GetDocumnent(uri, cancellationToken);
                }

                if (doc != null)
                {
                    IEnumerable <HtmlATag> aTags = HtmlExtractor.ExtractATags(doc);
                    foreach (HtmlATag aTag in aTags)
                    {
                        if (cancellationToken.IsCancellationRequested)
                        {
                            return(null);
                        }

                        Uri             newUri     = uri.AddHtmlLink(aTag.Href);
                        List <WebImage> moreImages = await HyperlinkRecurse(method, newUri, settings, depth + 1, cancellationToken);

                        images.AddRange(moreImages);
                    }
                }
            }

            return(images);
        }