private XmlSitemapProvider[] CreateProviders(XmlSitemapBuildContext context)
 {
     return(_publishers
            .Select(x => x.Value.PublishXmlSitemap(context))
            .Where(x => x != null)
            .OrderBy(x => x.Order)
            .ToArray());
 }
        public async Task Run(TaskExecutionContext ctx, CancellationToken cancelToken = default)
        {
            var stores = _storeContext.GetAllStores();

            foreach (var store in stores)
            {
                var languages    = _languageService.GetAllLanguages(false, store.Id);
                var buildContext = new XmlSitemapBuildContext(store, languages.ToArray(), _settingFactory, _storeContext.IsSingleStoreMode())
                {
                    CancellationToken = cancelToken,
                    ProgressCallback  = OnProgress
                };

                await _generator.RebuildAsync(buildContext);
            }

            Task OnProgress(int value, int max, string msg)
            {
                return(ctx.SetProgressAsync(value, max, msg, true));
            }
        }
 protected virtual Task ProcessCustomNodesAsync(XmlSitemapBuildContext ctx, Multimap <int, XmlSitemapNode> sitemaps)
 {
     // For inheritors
     return(Task.CompletedTask);
 }
        public virtual async Task RebuildAsync(XmlSitemapBuildContext ctx)
        {
            Guard.NotNull(ctx, nameof(ctx));

            // TODO: (core) Check later if this is still necessary
            //var dataTokens = _httpContextAccessor.HttpContext?.GetRouteData()?.DataTokens;
            //if (dataTokens != null)
            //{
            //    // Double seo code otherwise
            //    dataTokens["CultureCodeReplacement"] = string.Empty;
            //}

            var languageData = new Dictionary <int, LanguageData>();

            foreach (var language in ctx.Languages)
            {
                var lockFilePath = GetLockFilePath(ctx.Store.Id, language.Id);

                if (_lockFileManager.TryAcquireLock(lockFilePath, out var lockFile))
                {
                    // Process only languages that are unlocked right now
                    // It is possible that an HTTP request triggered the generation
                    // of a language specific sitemap.

                    try
                    {
                        var sitemapDir = BuildSitemapDirPath(ctx.Store.Id, language.Id);
                        var data       = new LanguageData
                        {
                            Store        = ctx.Store,
                            Language     = language,
                            LockFile     = lockFile,
                            LockFilePath = lockFilePath,
                            TempDir      = sitemapDir + "~",
                            FinalDir     = sitemapDir,
                            BaseUrl      = await BuildBaseUrlAsync(ctx.Store, language)
                        };

                        _tenantRoot.TryDeleteDirectory(data.TempDir);
                        _tenantRoot.TryCreateDirectory(data.TempDir);

                        languageData[language.Id] = data;
                    }
                    catch
                    {
                        await lockFile.ReleaseAsync();

                        throw;
                    }
                }
            }

            if (languageData.Count == 0)
            {
                Logger.Warn("XML sitemap rebuild already in process.");
                return;
            }

            var languages   = languageData.Values.Select(x => x.Language);
            var languageIds = languages.Select(x => x.Id).Concat(new[] { 0 }).ToArray();

            // All sitemaps grouped by language
            var sitemaps = new Multimap <int, XmlSitemapNode>();

            var compositeFileLock = new AsyncActionDisposable(async() =>
            {
                foreach (var data in languageData.Values)
                {
                    await data.LockFile.ReleaseAsync();
                }
            });

            await using (compositeFileLock)
            {
                // Impersonate
                var prevCustomer = _services.WorkContext.CurrentCustomer;
                // no need to vary xml sitemap by customer roles: it's relevant to crawlers only.
                // TODO: (core) Do not attempt to update CurrentCustomer entity if it is untracked (determine where applicable)
                _services.WorkContext.CurrentCustomer = (await _customerService.GetCustomerBySystemNameAsync(SystemCustomerNames.SearchEngine, false)) ?? prevCustomer;

                try
                {
                    var nodes = new List <XmlSitemapNode>();

                    var providers = CreateProviders(ctx);
                    var total     = await providers.SelectAsync(x => x.GetTotalCountAsync()).SumAsync(ctx.CancellationToken);

                    var totalSegments = (int)Math.Ceiling(total / (double)MaximumSiteMapNodeCount);
                    var hasIndex      = totalSegments > 1;
                    var indexNodes    = new Multimap <int, XmlSitemapNode>();
                    var segment       = 0;
                    var numProcessed  = 0;

                    CheckSitemapCount(totalSegments);

                    using (new DbContextScope(_db, autoDetectChanges: false, forceNoTracking: true, lazyLoading: false))
                    {
                        var entities = EnlistEntitiesAsync(providers);

                        await foreach (var batch in entities.SliceAsync(ctx.CancellationToken, MaximumSiteMapNodeCount))
                        {
                            if (ctx.CancellationToken.IsCancellationRequested)
                            {
                                break;
                            }

                            segment++;
                            numProcessed = segment * MaximumSiteMapNodeCount;
                            ctx.ProgressCallback?.Invoke(numProcessed, total, "{0} / {1}".FormatCurrent(numProcessed, total));

                            var slugs = await GetUrlRecordCollectionsForBatchAsync(batch.Select(x => x.Entry).ToList(), languageIds);

                            foreach (var data in languageData.Values)
                            {
                                var language = data.Language;
                                var baseUrl  = data.BaseUrl;

                                // Create all node entries for this segment
                                var entries = batch
                                              .Where(x => x.Entry.LanguageId.GetValueOrDefault() == 0 || x.Entry.LanguageId.Value == language.Id)
                                              .Select(x => x.Provider.CreateNode(_linkGenerator, baseUrl, x.Entry, slugs[x.Entry.EntityName], language));
                                sitemaps[language.Id].AddRange(entries.Where(x => x != null));

                                // Create index node for this segment/language combination
                                if (hasIndex)
                                {
                                    indexNodes[language.Id].Add(new XmlSitemapNode
                                    {
                                        LastMod = sitemaps[language.Id].Select(x => x.LastMod).Where(x => x.HasValue).DefaultIfEmpty().Max(),
                                        Loc     = GetSitemapIndexUrl(segment, baseUrl),
                                    });
                                }

                                if (segment % 5 == 0 || segment == totalSegments)
                                {
                                    // Commit every 5th segment (10.000 nodes) temporarily to disk to minimize RAM usage
                                    var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[language.Id]);
                                    await SaveTempAsync(documents, data, segment - documents.Count + (hasIndex ? 1 : 0));

                                    documents.Clear();
                                    sitemaps.RemoveAll(language.Id);
                                }
                            }

                            slugs.Clear();
                        }

                        // Process custom nodes
                        if (!ctx.CancellationToken.IsCancellationRequested)
                        {
                            ctx.ProgressCallback?.Invoke(numProcessed, total, "Processing custom nodes".FormatCurrent(numProcessed, total));
                            await ProcessCustomNodesAsync(ctx, sitemaps);

                            foreach (var data in languageData.Values)
                            {
                                if (sitemaps.ContainsKey(data.Language.Id) && sitemaps[data.Language.Id].Count > 0)
                                {
                                    var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[data.Language.Id]);
                                    await SaveTempAsync(documents, data, (segment + 1) - documents.Count + (hasIndex ? 1 : 0));
                                }
                                else if (segment == 0)
                                {
                                    // Ensure that at least one entry exists. Otherwise,
                                    // the system will try to rebuild again.
                                    var homeNode = new XmlSitemapNode {
                                        LastMod = DateTime.UtcNow, Loc = data.BaseUrl
                                    };
                                    var documents = GetSiteMapDocuments(new List <XmlSitemapNode> {
                                        homeNode
                                    });
                                    await SaveTempAsync(documents, data, 0);
                                }
                            }
                        }
                    }

                    ctx.CancellationToken.ThrowIfCancellationRequested();

                    ctx.ProgressCallback?.Invoke(totalSegments, totalSegments, "Finalizing...'");

                    foreach (var data in languageData.Values)
                    {
                        // Create index documents (if any)
                        if (hasIndex && indexNodes.Any())
                        {
                            var indexDocument = CreateSitemapIndexDocument(indexNodes[data.Language.Id]);
                            await SaveTempAsync(new List <string> {
                                indexDocument
                            }, data, 0);
                        }

                        // Save finally (actually renames temp folder)
                        await SaveFinalAsync(data);
                    }
                }
                finally
                {
                    // Undo impersonation
                    _services.WorkContext.CurrentCustomer = prevCustomer;
                    sitemaps.Clear();

                    foreach (var data in languageData.Values)
                    {
                        if (_tenantRoot.DirectoryExists(data.TempDir))
                        {
                            _tenantRoot.TryDeleteDirectory(data.TempDir);
                        }
                    }
                }
            }
        }
        private async Task <XmlSitemapPartition> GetSitemapPartAsync(int index, bool isRetry)
        {
            Guard.NotNegative(index, nameof(index));

            var store    = _services.StoreContext.CurrentStore;
            var language = _services.WorkContext.WorkingLanguage;

            var exists = TryGetSitemapFile(store.Id, language.Id, index, out var file);

            if (exists)
            {
                return(new XmlSitemapPartition
                {
                    Index = index,
                    Name = file.Name,
                    LanguageId = language.Id,
                    StoreId = store.Id,
                    ModifiedOnUtc = file.LastModified.UtcDateTime,
                    Stream = file.OpenRead()
                });
            }

            if (isRetry)
            {
                var msg = "Could not generate XML sitemap. Index: {0}, Date: {1}".FormatInvariant(index, DateTime.UtcNow);
                Logger.Error(msg);
                throw new SmartException(msg);
            }

            if (index > 0)
            {
                // File with index greater 0 has been requested, but it does not exist.
                // Now we have to determine whether just the passed index is out of range
                // or the files have never been created before.
                // If the main file (index 0) exists, the action should return NotFoundResult,
                // otherwise the rebuild process should be started or waited for.

                if (TryGetSitemapFile(store.Id, language.Id, 0, out file))
                {
                    throw new IndexOutOfRangeException("The sitemap file '{0}' does not exist.".FormatInvariant(file.Name));
                }
            }

            // The main sitemap document with index 0 does not exist, meaning: the whole sitemap
            // needs to be created and cached by partitions.

            var wasRebuilding = false;
            var lockFilePath  = GetLockFilePath(store.Id, language.Id);

            while (await IsRebuildingAsync(lockFilePath))
            {
                // The rebuild process is already running, either started
                // by the task scheduler or another HTTP request.
                // We should wait for completion.

                wasRebuilding = true;
                await Task.Delay(1000);
            }

            if (!wasRebuilding)
            {
                // No lock. Rebuild now.
                var buildContext = new XmlSitemapBuildContext(store, new[] { language }, _services.SettingFactory, _services.StoreContext.IsSingleStoreMode())
                {
                    CancellationToken = _asyncRunner.AppShutdownCancellationToken
                };

                await RebuildAsync(buildContext);
            }

            // DRY: call self to get sitemap partition object
            return(await GetSitemapPartAsync(index, true));
        }