/// <summary>
/// Retrieves a Sitemap from the specified location with the specified fetch options.
/// </summary>
/// <param name="sitemapLocation"></param>
/// <param name="options"></param>
/// <returns></returns>
public SitemapFile RetrieveSitemap(Uri sitemapLocation, SitemapFetchOptions options)
{
var type = GetSitemapType(sitemapLocation);
//Perform sitemap type-check
if (type == SitemapType.Unknown)
{
if (options.ThrowExceptionOnUnknownType)
{
throw new NotSupportedException("Specified sitemap is unsupported!");
}
else
{
return(null);
}
}
var rawSitemap = requestService.RetrieveRawSitemap(sitemapLocation);
var parsedSitemap = ParseSitemap(type, rawSitemap);
if (parsedSitemap == null)
{
return(null);
}
//Set the location of the parsed sitemap
parsedSitemap.Location = sitemapLocation;
if (options.ApplyDomainRestrictions)
{
var validEntries = new List <SitemapEntry>();
//For every entry, check the host matches the sitemap it is specified in
foreach (var entry in parsedSitemap.Urls)
{
if (entry.Location.Host == sitemapLocation.Host)
{
validEntries.Add(entry);
}
}
parsedSitemap.Urls = validEntries;
}
if (options.FetchInnerSitemaps)
{
var fetchedInnerSitemaps = new List <SitemapFile>();
//For every sitemap index, fetch the sitemap
foreach (var indexedSitemap in parsedSitemap.Sitemaps)
{
var tmpInnerSitemap = RetrieveSitemap(indexedSitemap.Location, options);
//Copy over the last modified from the sitemap index
tmpInnerSitemap.LastModified = indexedSitemap.LastModified;
fetchedInnerSitemaps.Add(tmpInnerSitemap);
}
parsedSitemap.Sitemaps = fetchedInnerSitemaps;
}
return(parsedSitemap);
}
/// <summary>
/// Finds the available sitemaps for the domain, retrieving each sitemap with the specified fetch options.
/// </summary>
/// <param name="domain"></param>
/// <returns></returns>
public IEnumerable <SitemapFile> RetrieveSitemapsForDomain(string domainName, SitemapFetchOptions options)
{
var sitemapLocations = requestService.GetAvailableSitemapsForDomain(domainName);
var result = new List <SitemapFile>();
foreach (var location in sitemapLocations)
{
var tmpSitemap = RetrieveSitemap(location, options);
if (tmpSitemap != null)
{
result.Add(tmpSitemap);
}
}
return(result);
}
