protected virtual IEnumerable <XmlSitemapNode> GetProductNodes(string protocol)
{
var nodes = new List <XmlSitemapNode>();
var searchQuery = new CatalogSearchQuery()
.VisibleOnly()
.VisibleIndividuallyOnly(true)
.HasStoreId(_services.StoreContext.CurrentStoreIdIfMultiStoreMode);
var query = _catalogSearchService.PrepareQuery(searchQuery);
query = query.OrderByDescending(x => x.CreatedOnUtc);
for (var pageIndex = 0; pageIndex < 9999999; ++pageIndex)
{
var products = new PagedList <Product>(query, pageIndex, 1000);
nodes.AddRange(products.Select(x =>
{
var node = new XmlSitemapNode
{
Loc = _urlHelper.RouteUrl("Product", new { SeName = x.GetSeName() }, protocol),
LastMod = x.UpdatedOnUtc,
//ChangeFreq = ChangeFrequency.Weekly,
//Priority = 0.8f
};
// TODO: add hreflang links if LangCount is > 1 and PrependSeoCode is true
return(node);
}));
_services.DbContext.DetachAll();
if (!products.HasNextPage)
{
break;
}
}
return(nodes);
}
protected virtual IEnumerable <XmlSitemapNode> GetManufacturerNodes(string protocol)
{
var manufacturers = _manufacturerService.GetAllManufacturers(false);
_services.DbContext.DetachAll();
return(manufacturers.Select(x =>
{
var node = new XmlSitemapNode
{
Loc = _urlHelper.RouteUrl("Manufacturer", new { SeName = x.GetSeName() }, protocol),
LastMod = x.UpdatedOnUtc,
//ChangeFreq = ChangeFrequency.Weekly,
//Priority = 0.8f
};
// TODO: add hreflang links if LangCount is > 1 and PrependSeoCode is true
return node;
}));
}
protected virtual IEnumerable <XmlSitemapNode> GetTopicNodes(string protocol)
{
var topics = _topicService.GetAllTopics(_services.StoreContext.CurrentStore.Id).ToList().FindAll(t => t.IncludeInSitemap && !t.RenderAsWidget);
_services.DbContext.DetachAll();
return(topics.Select(x =>
{
var node = new XmlSitemapNode
{
Loc = _urlHelper.RouteUrl("Topic", new { SystemName = x.SystemName }, protocol),
LastMod = DateTime.UtcNow,
//ChangeFreq = ChangeFrequency.Weekly,
//Priority = 0.8f
};
// TODO: add hreflang links if LangCount is > 1 and PrependSeoCode is true
return node;
}));
}
protected virtual IEnumerable <XmlSitemapNode> GetCategoryNodes(int parentCategoryId, string protocol)
{
var categories = _categoryService.GetAllCategories(showHidden: false);
_services.DbContext.DetachAll();
return(categories.Select(x =>
{
var node = new XmlSitemapNode
{
Loc = _urlHelper.RouteUrl("Category", new { SeName = x.GetSeName() }, protocol),
LastMod = x.UpdatedOnUtc,
//ChangeFreq = ChangeFrequency.Weekly,
//Priority = 0.8f
};
// TODO: add hreflang links if LangCount is > 1 and PrependSeoCode is true
return node;
}));
}
public virtual async Task RebuildAsync(XmlSitemapBuildContext ctx)
{
Guard.NotNull(ctx, nameof(ctx));
var languageData = new Dictionary <int, LanguageData>();
foreach (var language in ctx.Languages)
{
var lockFilePath = GetLockFilePath(ctx.Store.Id, language.Id);
if (_lockFileManager.TryAcquireLock(lockFilePath, out var lockFile))
{
// Process only languages that are unlocked right now
// It is possible that an HTTP request triggered the generation
// of a language specific sitemap.
var sitemapDir = BuildSitemapDirPath(ctx.Store.Id, language.Id);
var data = new LanguageData
{
Store = ctx.Store,
Language = language,
LockFile = lockFile,
LockFilePath = lockFilePath,
TempDir = sitemapDir + "~",
FinalDir = sitemapDir,
BaseUrl = BuildBaseUrl(ctx.Store, language)
};
_tenantFolder.TryDeleteDirectory(data.TempDir);
_tenantFolder.CreateDirectory(data.TempDir);
languageData[language.Id] = data;
}
}
if (languageData.Count == 0)
{
Logger.Warn("XML sitemap rebuild already in process.");
return;
}
var languages = languageData.Values.Select(x => x.Language);
var languageIds = languages.Select(x => x.Id).Concat(new[] { 0 }).ToArray();
// All sitemaps grouped by language
var sitemaps = new Multimap <int, XmlSitemapNode>();
var compositeFileLock = new ActionDisposable(() =>
{
foreach (var data in languageData.Values)
{
data.LockFile.Release();
}
});
using (compositeFileLock)
{
// Impersonate
var prevCustomer = _services.WorkContext.CurrentCustomer;
// no need to vary xml sitemap by customer roles: it's relevant to crawlers only.
_services.WorkContext.CurrentCustomer = _customerService.GetCustomerBySystemName(SystemCustomerNames.SearchEngine);
try
{
var nodes = new List <XmlSitemapNode>();
var queries = CreateQueries(ctx);
var total = await queries.GetTotalRecordCountAsync();
var totalSegments = (int)Math.Ceiling(total / (double)MaximumSiteMapNodeCount);
var hasIndex = totalSegments > 1;
var indexNodes = new Multimap <int, XmlSitemapNode>();
var segment = 0;
var numProcessed = 0;
CheckSitemapCount(totalSegments);
using (new DbContextScope(autoDetectChanges: false, forceNoTracking: true, proxyCreation: false, lazyLoading: false))
{
var entities = await EnumerateEntitiesAsync(queries);
foreach (var batch in entities.Slice(MaximumSiteMapNodeCount))
{
if (ctx.CancellationToken.IsCancellationRequested)
{
break;
}
segment++;
numProcessed = segment * MaximumSiteMapNodeCount;
ctx.ProgressCallback?.Invoke(numProcessed, total, "{0} / {1}".FormatCurrent(numProcessed, total));
var firstEntityName = batch.First().EntityName;
var lastEntityName = batch.Last().EntityName;
var slugs = GetUrlRecordCollectionsForBatch(batch, languageIds);
foreach (var data in languageData.Values)
{
var language = data.Language;
var baseUrl = data.BaseUrl;
// Create all node entries for this segment
sitemaps[language.Id].AddRange(batch.Select(x => new XmlSitemapNode
{
LastMod = x.LastMod,
Loc = BuildNodeUrl(baseUrl, x, slugs[x.EntityName], language)
}));
// Create index node for this segment/language combination
if (hasIndex)
{
indexNodes[language.Id].Add(new XmlSitemapNode
{
LastMod = sitemaps[language.Id].Select(x => x.LastMod).Where(x => x.HasValue).DefaultIfEmpty().Max(),
Loc = GetSitemapIndexUrl(segment, baseUrl),
});
}
if (segment % 5 == 0 || segment == totalSegments)
{
// Commit every 5th segment (10.000 nodes) temporarily to disk to minimize RAM usage
var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[language.Id]);
await SaveTempAsync(documents, data, segment - documents.Count + (hasIndex ? 1 : 0));
documents.Clear();
sitemaps.RemoveAll(language.Id);
}
}
slugs.Clear();
//GC.Collect();
//GC.WaitForPendingFinalizers();
}
// Process custom nodes
if (!ctx.CancellationToken.IsCancellationRequested)
{
ctx.ProgressCallback?.Invoke(numProcessed, total, "Processing custom nodes".FormatCurrent(numProcessed, total));
ProcessCustomNodes(ctx, sitemaps);
foreach (var data in languageData.Values)
{
if (sitemaps.ContainsKey(data.Language.Id) && sitemaps[data.Language.Id].Count > 0)
{
var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[data.Language.Id]);
await SaveTempAsync(documents, data, (segment + 1) - documents.Count + (hasIndex ? 1 : 0));
}
else if (segment == 0)
{
// Ensure that at least one entry exists. Otherwise,
// the system will try to rebuild again.
var homeNode = new XmlSitemapNode {
LastMod = DateTime.UtcNow, Loc = data.BaseUrl
};
var documents = GetSiteMapDocuments(new List <XmlSitemapNode> {
homeNode
});
await SaveTempAsync(documents, data, 0);
}
}
}
}
ctx.CancellationToken.ThrowIfCancellationRequested();
ctx.ProgressCallback?.Invoke(totalSegments, totalSegments, "Finalizing...'");
foreach (var data in languageData.Values)
{
// Create index documents (if any)
if (hasIndex && indexNodes.Any())
{
var indexDocument = CreateSitemapIndexDocument(indexNodes[data.Language.Id]);
await SaveTempAsync(new List <string> {
indexDocument
}, data, 0);
}
// Save finally (actually renames temp folder)
SaveFinal(data);
}
}
finally
{
// Undo impersonation
_services.WorkContext.CurrentCustomer = prevCustomer;
sitemaps.Clear();
foreach (var data in languageData.Values)
{
if (_tenantFolder.DirectoryExists(data.TempDir))
{
_tenantFolder.TryDeleteDirectory(data.TempDir);
}
}
//GC.Collect();
//GC.WaitForPendingFinalizers();
}
}
}
