예제 #1
0
        /// <summary>
        /// Retrieves a Sitemap from the specified location with the specified fetch options.
        /// </summary>
        /// <param name="sitemapLocation"></param>
        /// <param name="options"></param>
        /// <returns></returns>
        public SitemapFile RetrieveSitemap(Uri sitemapLocation, SitemapFetchOptions options)
        {
            var type = GetSitemapType(sitemapLocation);

            //Perform sitemap type-check
            if (type == SitemapType.Unknown)
            {
                if (options.ThrowExceptionOnUnknownType)
                {
                    throw new NotSupportedException("Specified sitemap is unsupported!");
                }
                else
                {
                    return(null);
                }
            }

            var rawSitemap    = requestService.RetrieveRawSitemap(sitemapLocation);
            var parsedSitemap = ParseSitemap(type, rawSitemap);

            if (parsedSitemap == null)
            {
                return(null);
            }

            //Set the location of the parsed sitemap
            parsedSitemap.Location = sitemapLocation;

            if (options.ApplyDomainRestrictions)
            {
                var validEntries = new List <SitemapEntry>();

                //For every entry, check the host matches the sitemap it is specified in
                foreach (var entry in parsedSitemap.Urls)
                {
                    if (entry.Location.Host == sitemapLocation.Host)
                    {
                        validEntries.Add(entry);
                    }
                }

                parsedSitemap.Urls = validEntries;
            }

            if (options.FetchInnerSitemaps)
            {
                var fetchedInnerSitemaps = new List <SitemapFile>();

                //For every sitemap index, fetch the sitemap
                foreach (var indexedSitemap in parsedSitemap.Sitemaps)
                {
                    var tmpInnerSitemap = RetrieveSitemap(indexedSitemap.Location, options);

                    //Copy over the last modified from the sitemap index
                    tmpInnerSitemap.LastModified = indexedSitemap.LastModified;

                    fetchedInnerSitemaps.Add(tmpInnerSitemap);
                }

                parsedSitemap.Sitemaps = fetchedInnerSitemaps;
            }

            return(parsedSitemap);
        }
예제 #2
0
        /// <summary>
        /// Finds the available sitemaps for the domain, retrieving each sitemap with the specified fetch options.
        /// </summary>
        /// <param name="domain"></param>
        /// <returns></returns>
        public IEnumerable <SitemapFile> RetrieveSitemapsForDomain(string domainName, SitemapFetchOptions options)
        {
            var sitemapLocations = requestService.GetAvailableSitemapsForDomain(domainName);
            var result           = new List <SitemapFile>();

            foreach (var location in sitemapLocations)
            {
                var tmpSitemap = RetrieveSitemap(location, options);
                if (tmpSitemap != null)
                {
                    result.Add(tmpSitemap);
                }
            }

            return(result);
        }