private XmlSitemapProvider[] CreateProviders(XmlSitemapBuildContext context)
{
return(_publishers
.Select(x => x.Value.PublishXmlSitemap(context))
.Where(x => x != null)
.OrderBy(x => x.Order)
.ToArray());
}
private QueryHolder CreateQueries(XmlSitemapBuildContext ctx)
{
var storeId = ctx.Store.Id;
if (_services.StoreService.IsSingleStoreMode())
{
storeId = 0;
}
// Always work with store-dependant setting
var seoSettings = _services.Settings.LoadSetting <SeoSettings>(storeId);
var holder = new QueryHolder();
if (seoSettings.XmlSitemapIncludesCategories)
{
holder.Categories = _categoryService.BuildCategoriesQuery(showHidden: false, storeId: storeId);
}
if (seoSettings.XmlSitemapIncludesManufacturers)
{
holder.Manufacturers = _manufacturerService.GetManufacturers(false).OrderBy(x => x.DisplayOrder).ThenBy(x => x.Name);
}
if (seoSettings.XmlSitemapIncludesTopics)
{
holder.Topics = _topicService.GetAllTopics(storeId).AlterQuery(q =>
{
return(q.Where(t => t.IncludeInSitemap && !t.RenderAsWidget));
}).SourceQuery;
}
if (seoSettings.XmlSitemapIncludesProducts)
{
var searchQuery = new CatalogSearchQuery()
.VisibleOnly()
.VisibleIndividuallyOnly(true)
.HasStoreId(storeId);
holder.Products = _catalogSearchService.PrepareQuery(searchQuery);
}
return(holder);
}
public override async Task ExecuteAsync(TaskExecutionContext ctx)
{
var stores = _storeService.GetAllStores();
foreach (var store in stores)
{
var languages = _languageService.GetAllLanguages(false, store.Id);
var buildContext = new XmlSitemapBuildContext(store, languages.ToArray())
{
CancellationToken = ctx.CancellationToken,
ProgressCallback = OnProgress
};
await _generator.RebuildAsync(buildContext);
}
void OnProgress(int value, int max, string msg)
{
ctx.SetProgress(value, max, msg, true);
}
}
protected void ProcessCustomNodes(XmlSitemapBuildContext ctx, Multimap <int, XmlSitemapNode> sitemaps)
{
}
public virtual async Task RebuildAsync(XmlSitemapBuildContext ctx)
{
Guard.NotNull(ctx, nameof(ctx));
var languageData = new Dictionary <int, LanguageData>();
foreach (var language in ctx.Languages)
{
var lockFilePath = GetLockFilePath(ctx.Store.Id, language.Id);
if (_lockFileManager.TryAcquireLock(lockFilePath, out var lockFile))
{
// Process only languages that are unlocked right now
// It is possible that an HTTP request triggered the generation
// of a language specific sitemap.
var sitemapDir = BuildSitemapDirPath(ctx.Store.Id, language.Id);
var data = new LanguageData
{
Store = ctx.Store,
Language = language,
LockFile = lockFile,
LockFilePath = lockFilePath,
TempDir = sitemapDir + "~",
FinalDir = sitemapDir,
BaseUrl = BuildBaseUrl(ctx.Store, language)
};
_tenantFolder.TryDeleteDirectory(data.TempDir);
_tenantFolder.CreateDirectory(data.TempDir);
languageData[language.Id] = data;
}
}
if (languageData.Count == 0)
{
Logger.Warn("XML sitemap rebuild already in process.");
return;
}
var languages = languageData.Values.Select(x => x.Language);
var languageIds = languages.Select(x => x.Id).Concat(new[] { 0 }).ToArray();
// All sitemaps grouped by language
var sitemaps = new Multimap <int, XmlSitemapNode>();
var compositeFileLock = new ActionDisposable(() =>
{
foreach (var data in languageData.Values)
{
data.LockFile.Release();
}
});
using (compositeFileLock)
{
// Impersonate
var prevCustomer = _services.WorkContext.CurrentCustomer;
// no need to vary xml sitemap by customer roles: it's relevant to crawlers only.
_services.WorkContext.CurrentCustomer = _customerService.GetCustomerBySystemName(SystemCustomerNames.SearchEngine);
try
{
var nodes = new List <XmlSitemapNode>();
var queries = CreateQueries(ctx);
var total = await queries.GetTotalRecordCountAsync();
var totalSegments = (int)Math.Ceiling(total / (double)MaximumSiteMapNodeCount);
var hasIndex = totalSegments > 1;
var indexNodes = new Multimap <int, XmlSitemapNode>();
var segment = 0;
var numProcessed = 0;
CheckSitemapCount(totalSegments);
using (new DbContextScope(autoDetectChanges: false, forceNoTracking: true, proxyCreation: false, lazyLoading: false))
{
var entities = await EnumerateEntitiesAsync(queries);
foreach (var batch in entities.Slice(MaximumSiteMapNodeCount))
{
if (ctx.CancellationToken.IsCancellationRequested)
{
break;
}
segment++;
numProcessed = segment * MaximumSiteMapNodeCount;
ctx.ProgressCallback?.Invoke(numProcessed, total, "{0} / {1}".FormatCurrent(numProcessed, total));
var firstEntityName = batch.First().EntityName;
var lastEntityName = batch.Last().EntityName;
var slugs = GetUrlRecordCollectionsForBatch(batch, languageIds);
foreach (var data in languageData.Values)
{
var language = data.Language;
var baseUrl = data.BaseUrl;
// Create all node entries for this segment
sitemaps[language.Id].AddRange(batch.Select(x => new XmlSitemapNode
{
LastMod = x.LastMod,
Loc = BuildNodeUrl(baseUrl, x, slugs[x.EntityName], language)
}));
// Create index node for this segment/language combination
if (hasIndex)
{
indexNodes[language.Id].Add(new XmlSitemapNode
{
LastMod = sitemaps[language.Id].Select(x => x.LastMod).Where(x => x.HasValue).DefaultIfEmpty().Max(),
Loc = GetSitemapIndexUrl(segment, baseUrl),
});
}
if (segment % 5 == 0 || segment == totalSegments)
{
// Commit every 5th segment (10.000 nodes) temporarily to disk to minimize RAM usage
var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[language.Id]);
await SaveTempAsync(documents, data, segment - documents.Count + (hasIndex ? 1 : 0));
documents.Clear();
sitemaps.RemoveAll(language.Id);
}
}
slugs.Clear();
//GC.Collect();
//GC.WaitForPendingFinalizers();
}
// Process custom nodes
if (!ctx.CancellationToken.IsCancellationRequested)
{
ctx.ProgressCallback?.Invoke(numProcessed, total, "Processing custom nodes".FormatCurrent(numProcessed, total));
ProcessCustomNodes(ctx, sitemaps);
foreach (var data in languageData.Values)
{
if (sitemaps.ContainsKey(data.Language.Id) && sitemaps[data.Language.Id].Count > 0)
{
var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[data.Language.Id]);
await SaveTempAsync(documents, data, (segment + 1) - documents.Count + (hasIndex ? 1 : 0));
}
else if (segment == 0)
{
// Ensure that at least one entry exists. Otherwise,
// the system will try to rebuild again.
var homeNode = new XmlSitemapNode {
LastMod = DateTime.UtcNow, Loc = data.BaseUrl
};
var documents = GetSiteMapDocuments(new List <XmlSitemapNode> {
homeNode
});
await SaveTempAsync(documents, data, 0);
}
}
}
}
ctx.CancellationToken.ThrowIfCancellationRequested();
ctx.ProgressCallback?.Invoke(totalSegments, totalSegments, "Finalizing...'");
foreach (var data in languageData.Values)
{
// Create index documents (if any)
if (hasIndex && indexNodes.Any())
{
var indexDocument = CreateSitemapIndexDocument(indexNodes[data.Language.Id]);
await SaveTempAsync(new List <string> {
indexDocument
}, data, 0);
}
// Save finally (actually renames temp folder)
SaveFinal(data);
}
}
finally
{
// Undo impersonation
_services.WorkContext.CurrentCustomer = prevCustomer;
sitemaps.Clear();
foreach (var data in languageData.Values)
{
if (_tenantFolder.DirectoryExists(data.TempDir))
{
_tenantFolder.TryDeleteDirectory(data.TempDir);
}
}
//GC.Collect();
//GC.WaitForPendingFinalizers();
}
}
}
private async Task <XmlSitemapPartition> GetSitemapPartAsync(int index, bool isRetry)
{
Guard.NotNegative(index, nameof(index));
var store = _services.StoreContext.CurrentStore;
var language = _services.WorkContext.WorkingLanguage;
var exists = SitemapFileExists(store.Id, language.Id, index, out var path, out var name);
if (exists)
{
return(new XmlSitemapPartition
{
Index = index,
Name = name,
LanguageId = language.Id,
StoreId = store.Id,
ModifiedOnUtc = _tenantFolder.GetFileLastWriteTimeUtc(path),
Stream = _tenantFolder.OpenFile(path)
});
}
if (isRetry)
{
var msg = "Could not generate XML sitemap. Index: {0}, Date: {1}".FormatInvariant(index, DateTime.UtcNow);
Logger.Error(msg);
throw new SmartException(msg);
}
if (index > 0)
{
// File with index greater 0 has been requested, but it does not exist.
// Now we have to determine whether just the passed index is out of range
// or the files has never been created before.
// If the main file (index 0) exists, the action should return NotFoundResult,
// otherwise the rebuild process should be started or waited for.
if (SitemapFileExists(store.Id, language.Id, 0, out path, out name))
{
throw new IndexOutOfRangeException("The sitemap file '{0}' does not exist.".FormatInvariant(name));
}
}
// The main sitemap document with index 0 does not exist, meaning: the whole sitemap
// needs to be created and cached by partitions.
var wasRebuilding = false;
var lockFilePath = GetLockFilePath(store.Id, language.Id);
while (IsRebuilding(lockFilePath))
{
// The rebuild process is already running, either started
// by the task scheduler or another HTTP request.
// We should wait for completion.
wasRebuilding = true;
Thread.Sleep(1000);
}
if (!wasRebuilding)
{
// No lock. Rebuild now.
var buildContext = new XmlSitemapBuildContext(store, new[] { language })
{
CancellationToken = CancellationToken.None
};
await RebuildAsync(buildContext);
}
// DRY: call self to get sitemap partition object
return(await GetSitemapPartAsync(index, true));
}