protected virtual IEnumerable <XmlSitemapNode> GetProductNodes(string protocol) { var nodes = new List <XmlSitemapNode>(); var searchQuery = new CatalogSearchQuery() .VisibleOnly() .VisibleIndividuallyOnly(true) .HasStoreId(_services.StoreContext.CurrentStoreIdIfMultiStoreMode); var query = _catalogSearchService.PrepareQuery(searchQuery); query = query.OrderByDescending(x => x.CreatedOnUtc); for (var pageIndex = 0; pageIndex < 9999999; ++pageIndex) { var products = new PagedList <Product>(query, pageIndex, 1000); nodes.AddRange(products.Select(x => { var node = new XmlSitemapNode { Loc = _urlHelper.RouteUrl("Product", new { SeName = x.GetSeName() }, protocol), LastMod = x.UpdatedOnUtc, //ChangeFreq = ChangeFrequency.Weekly, //Priority = 0.8f }; // TODO: add hreflang links if LangCount is > 1 and PrependSeoCode is true return(node); })); _services.DbContext.DetachAll(); if (!products.HasNextPage) { break; } } return(nodes); }
protected virtual IEnumerable <XmlSitemapNode> GetManufacturerNodes(string protocol) { var manufacturers = _manufacturerService.GetAllManufacturers(false); _services.DbContext.DetachAll(); return(manufacturers.Select(x => { var node = new XmlSitemapNode { Loc = _urlHelper.RouteUrl("Manufacturer", new { SeName = x.GetSeName() }, protocol), LastMod = x.UpdatedOnUtc, //ChangeFreq = ChangeFrequency.Weekly, //Priority = 0.8f }; // TODO: add hreflang links if LangCount is > 1 and PrependSeoCode is true return node; })); }
protected virtual IEnumerable <XmlSitemapNode> GetTopicNodes(string protocol) { var topics = _topicService.GetAllTopics(_services.StoreContext.CurrentStore.Id).ToList().FindAll(t => t.IncludeInSitemap && !t.RenderAsWidget); _services.DbContext.DetachAll(); return(topics.Select(x => { var node = new XmlSitemapNode { Loc = _urlHelper.RouteUrl("Topic", new { SystemName = x.SystemName }, protocol), LastMod = DateTime.UtcNow, //ChangeFreq = ChangeFrequency.Weekly, //Priority = 0.8f }; // TODO: add hreflang links if LangCount is > 1 and PrependSeoCode is true return node; })); }
protected virtual IEnumerable <XmlSitemapNode> GetCategoryNodes(int parentCategoryId, string protocol) { var categories = _categoryService.GetAllCategories(showHidden: false); _services.DbContext.DetachAll(); return(categories.Select(x => { var node = new XmlSitemapNode { Loc = _urlHelper.RouteUrl("Category", new { SeName = x.GetSeName() }, protocol), LastMod = x.UpdatedOnUtc, //ChangeFreq = ChangeFrequency.Weekly, //Priority = 0.8f }; // TODO: add hreflang links if LangCount is > 1 and PrependSeoCode is true return node; })); }
public virtual async Task RebuildAsync(XmlSitemapBuildContext ctx) { Guard.NotNull(ctx, nameof(ctx)); var languageData = new Dictionary <int, LanguageData>(); foreach (var language in ctx.Languages) { var lockFilePath = GetLockFilePath(ctx.Store.Id, language.Id); if (_lockFileManager.TryAcquireLock(lockFilePath, out var lockFile)) { // Process only languages that are unlocked right now // It is possible that an HTTP request triggered the generation // of a language specific sitemap. var sitemapDir = BuildSitemapDirPath(ctx.Store.Id, language.Id); var data = new LanguageData { Store = ctx.Store, Language = language, LockFile = lockFile, LockFilePath = lockFilePath, TempDir = sitemapDir + "~", FinalDir = sitemapDir, BaseUrl = BuildBaseUrl(ctx.Store, language) }; _tenantFolder.TryDeleteDirectory(data.TempDir); _tenantFolder.CreateDirectory(data.TempDir); languageData[language.Id] = data; } } if (languageData.Count == 0) { Logger.Warn("XML sitemap rebuild already in process."); return; } var languages = languageData.Values.Select(x => x.Language); var languageIds = languages.Select(x => x.Id).Concat(new[] { 0 }).ToArray(); // All sitemaps grouped by language var sitemaps = new Multimap <int, XmlSitemapNode>(); var compositeFileLock = new ActionDisposable(() => { foreach (var data in languageData.Values) { data.LockFile.Release(); } }); using (compositeFileLock) { // Impersonate var prevCustomer = _services.WorkContext.CurrentCustomer; // no need to vary xml sitemap by customer roles: it's relevant to crawlers only. _services.WorkContext.CurrentCustomer = _customerService.GetCustomerBySystemName(SystemCustomerNames.SearchEngine); try { var nodes = new List <XmlSitemapNode>(); var queries = CreateQueries(ctx); var total = await queries.GetTotalRecordCountAsync(); var totalSegments = (int)Math.Ceiling(total / (double)MaximumSiteMapNodeCount); var hasIndex = totalSegments > 1; var indexNodes = new Multimap <int, XmlSitemapNode>(); var segment = 0; var numProcessed = 0; CheckSitemapCount(totalSegments); using (new DbContextScope(autoDetectChanges: false, forceNoTracking: true, proxyCreation: false, lazyLoading: false)) { var entities = await EnumerateEntitiesAsync(queries); foreach (var batch in entities.Slice(MaximumSiteMapNodeCount)) { if (ctx.CancellationToken.IsCancellationRequested) { break; } segment++; numProcessed = segment * MaximumSiteMapNodeCount; ctx.ProgressCallback?.Invoke(numProcessed, total, "{0} / {1}".FormatCurrent(numProcessed, total)); var firstEntityName = batch.First().EntityName; var lastEntityName = batch.Last().EntityName; var slugs = GetUrlRecordCollectionsForBatch(batch, languageIds); foreach (var data in languageData.Values) { var language = data.Language; var baseUrl = data.BaseUrl; // Create all node entries for this segment sitemaps[language.Id].AddRange(batch.Select(x => new XmlSitemapNode { LastMod = x.LastMod, Loc = BuildNodeUrl(baseUrl, x, slugs[x.EntityName], language) })); // Create index node for this segment/language combination if (hasIndex) { indexNodes[language.Id].Add(new XmlSitemapNode { LastMod = sitemaps[language.Id].Select(x => x.LastMod).Where(x => x.HasValue).DefaultIfEmpty().Max(), Loc = GetSitemapIndexUrl(segment, baseUrl), }); } if (segment % 5 == 0 || segment == totalSegments) { // Commit every 5th segment (10.000 nodes) temporarily to disk to minimize RAM usage var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[language.Id]); await SaveTempAsync(documents, data, segment - documents.Count + (hasIndex ? 1 : 0)); documents.Clear(); sitemaps.RemoveAll(language.Id); } } slugs.Clear(); //GC.Collect(); //GC.WaitForPendingFinalizers(); } // Process custom nodes if (!ctx.CancellationToken.IsCancellationRequested) { ctx.ProgressCallback?.Invoke(numProcessed, total, "Processing custom nodes".FormatCurrent(numProcessed, total)); ProcessCustomNodes(ctx, sitemaps); foreach (var data in languageData.Values) { if (sitemaps.ContainsKey(data.Language.Id) && sitemaps[data.Language.Id].Count > 0) { var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[data.Language.Id]); await SaveTempAsync(documents, data, (segment + 1) - documents.Count + (hasIndex ? 1 : 0)); } else if (segment == 0) { // Ensure that at least one entry exists. Otherwise, // the system will try to rebuild again. var homeNode = new XmlSitemapNode { LastMod = DateTime.UtcNow, Loc = data.BaseUrl }; var documents = GetSiteMapDocuments(new List <XmlSitemapNode> { homeNode }); await SaveTempAsync(documents, data, 0); } } } } ctx.CancellationToken.ThrowIfCancellationRequested(); ctx.ProgressCallback?.Invoke(totalSegments, totalSegments, "Finalizing...'"); foreach (var data in languageData.Values) { // Create index documents (if any) if (hasIndex && indexNodes.Any()) { var indexDocument = CreateSitemapIndexDocument(indexNodes[data.Language.Id]); await SaveTempAsync(new List <string> { indexDocument }, data, 0); } // Save finally (actually renames temp folder) SaveFinal(data); } } finally { // Undo impersonation _services.WorkContext.CurrentCustomer = prevCustomer; sitemaps.Clear(); foreach (var data in languageData.Values) { if (_tenantFolder.DirectoryExists(data.TempDir)) { _tenantFolder.TryDeleteDirectory(data.TempDir); } } //GC.Collect(); //GC.WaitForPendingFinalizers(); } } }