示例#1
0
 private XmlSitemapProvider[] CreateProviders(XmlSitemapBuildContext context)
 {
     return(_publishers
            .Select(x => x.Value.PublishXmlSitemap(context))
            .Where(x => x != null)
            .OrderBy(x => x.Order)
            .ToArray());
 }
示例#2
0
        private QueryHolder CreateQueries(XmlSitemapBuildContext ctx)
        {
            var storeId = ctx.Store.Id;

            if (_services.StoreService.IsSingleStoreMode())
            {
                storeId = 0;
            }

            // Always work with store-dependant setting
            var seoSettings = _services.Settings.LoadSetting <SeoSettings>(storeId);

            var holder = new QueryHolder();

            if (seoSettings.XmlSitemapIncludesCategories)
            {
                holder.Categories = _categoryService.BuildCategoriesQuery(showHidden: false, storeId: storeId);
            }

            if (seoSettings.XmlSitemapIncludesManufacturers)
            {
                holder.Manufacturers = _manufacturerService.GetManufacturers(false).OrderBy(x => x.DisplayOrder).ThenBy(x => x.Name);
            }

            if (seoSettings.XmlSitemapIncludesTopics)
            {
                holder.Topics = _topicService.GetAllTopics(storeId).AlterQuery(q =>
                {
                    return(q.Where(t => t.IncludeInSitemap && !t.RenderAsWidget));
                }).SourceQuery;
            }

            if (seoSettings.XmlSitemapIncludesProducts)
            {
                var searchQuery = new CatalogSearchQuery()
                                  .VisibleOnly()
                                  .VisibleIndividuallyOnly(true)
                                  .HasStoreId(storeId);

                holder.Products = _catalogSearchService.PrepareQuery(searchQuery);
            }

            return(holder);
        }
示例#3
0
        public override async Task ExecuteAsync(TaskExecutionContext ctx)
        {
            var stores = _storeService.GetAllStores();

            foreach (var store in stores)
            {
                var languages    = _languageService.GetAllLanguages(false, store.Id);
                var buildContext = new XmlSitemapBuildContext(store, languages.ToArray())
                {
                    CancellationToken = ctx.CancellationToken,
                    ProgressCallback  = OnProgress
                };

                await _generator.RebuildAsync(buildContext);
            }

            void OnProgress(int value, int max, string msg)
            {
                ctx.SetProgress(value, max, msg, true);
            }
        }
示例#4
0
 protected void ProcessCustomNodes(XmlSitemapBuildContext ctx, Multimap <int, XmlSitemapNode> sitemaps)
 {
 }
示例#5
0
        public virtual async Task RebuildAsync(XmlSitemapBuildContext ctx)
        {
            Guard.NotNull(ctx, nameof(ctx));

            var languageData = new Dictionary <int, LanguageData>();

            foreach (var language in ctx.Languages)
            {
                var lockFilePath = GetLockFilePath(ctx.Store.Id, language.Id);

                if (_lockFileManager.TryAcquireLock(lockFilePath, out var lockFile))
                {
                    // Process only languages that are unlocked right now
                    // It is possible that an HTTP request triggered the generation
                    // of a language specific sitemap.

                    var sitemapDir = BuildSitemapDirPath(ctx.Store.Id, language.Id);
                    var data       = new LanguageData
                    {
                        Store        = ctx.Store,
                        Language     = language,
                        LockFile     = lockFile,
                        LockFilePath = lockFilePath,
                        TempDir      = sitemapDir + "~",
                        FinalDir     = sitemapDir,
                        BaseUrl      = BuildBaseUrl(ctx.Store, language)
                    };

                    _tenantFolder.TryDeleteDirectory(data.TempDir);
                    _tenantFolder.CreateDirectory(data.TempDir);

                    languageData[language.Id] = data;
                }
            }

            if (languageData.Count == 0)
            {
                Logger.Warn("XML sitemap rebuild already in process.");
                return;
            }

            var languages   = languageData.Values.Select(x => x.Language);
            var languageIds = languages.Select(x => x.Id).Concat(new[] { 0 }).ToArray();

            // All sitemaps grouped by language
            var sitemaps = new Multimap <int, XmlSitemapNode>();

            var compositeFileLock = new ActionDisposable(() =>
            {
                foreach (var data in languageData.Values)
                {
                    data.LockFile.Release();
                }
            });

            using (compositeFileLock)
            {
                // Impersonate
                var prevCustomer = _services.WorkContext.CurrentCustomer;
                // no need to vary xml sitemap by customer roles: it's relevant to crawlers only.
                _services.WorkContext.CurrentCustomer = _customerService.GetCustomerBySystemName(SystemCustomerNames.SearchEngine);

                try
                {
                    var nodes = new List <XmlSitemapNode>();

                    var queries = CreateQueries(ctx);
                    var total   = await queries.GetTotalRecordCountAsync();

                    var totalSegments = (int)Math.Ceiling(total / (double)MaximumSiteMapNodeCount);
                    var hasIndex      = totalSegments > 1;
                    var indexNodes    = new Multimap <int, XmlSitemapNode>();
                    var segment       = 0;
                    var numProcessed  = 0;

                    CheckSitemapCount(totalSegments);

                    using (new DbContextScope(autoDetectChanges: false, forceNoTracking: true, proxyCreation: false, lazyLoading: false))
                    {
                        var entities = await EnumerateEntitiesAsync(queries);

                        foreach (var batch in entities.Slice(MaximumSiteMapNodeCount))
                        {
                            if (ctx.CancellationToken.IsCancellationRequested)
                            {
                                break;
                            }

                            segment++;
                            numProcessed = segment * MaximumSiteMapNodeCount;
                            ctx.ProgressCallback?.Invoke(numProcessed, total, "{0} / {1}".FormatCurrent(numProcessed, total));

                            var firstEntityName = batch.First().EntityName;
                            var lastEntityName  = batch.Last().EntityName;

                            var slugs = GetUrlRecordCollectionsForBatch(batch, languageIds);

                            foreach (var data in languageData.Values)
                            {
                                var language = data.Language;
                                var baseUrl  = data.BaseUrl;

                                // Create all node entries for this segment
                                sitemaps[language.Id].AddRange(batch.Select(x => new XmlSitemapNode
                                {
                                    LastMod = x.LastMod,
                                    Loc     = BuildNodeUrl(baseUrl, x, slugs[x.EntityName], language)
                                }));

                                // Create index node for this segment/language combination
                                if (hasIndex)
                                {
                                    indexNodes[language.Id].Add(new XmlSitemapNode
                                    {
                                        LastMod = sitemaps[language.Id].Select(x => x.LastMod).Where(x => x.HasValue).DefaultIfEmpty().Max(),
                                        Loc     = GetSitemapIndexUrl(segment, baseUrl),
                                    });
                                }

                                if (segment % 5 == 0 || segment == totalSegments)
                                {
                                    // Commit every 5th segment (10.000 nodes) temporarily to disk to minimize RAM usage
                                    var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[language.Id]);
                                    await SaveTempAsync(documents, data, segment - documents.Count + (hasIndex ? 1 : 0));

                                    documents.Clear();
                                    sitemaps.RemoveAll(language.Id);
                                }
                            }

                            slugs.Clear();

                            //GC.Collect();
                            //GC.WaitForPendingFinalizers();
                        }

                        // Process custom nodes
                        if (!ctx.CancellationToken.IsCancellationRequested)
                        {
                            ctx.ProgressCallback?.Invoke(numProcessed, total, "Processing custom nodes".FormatCurrent(numProcessed, total));
                            ProcessCustomNodes(ctx, sitemaps);

                            foreach (var data in languageData.Values)
                            {
                                if (sitemaps.ContainsKey(data.Language.Id) && sitemaps[data.Language.Id].Count > 0)
                                {
                                    var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[data.Language.Id]);
                                    await SaveTempAsync(documents, data, (segment + 1) - documents.Count + (hasIndex ? 1 : 0));
                                }
                                else if (segment == 0)
                                {
                                    // Ensure that at least one entry exists. Otherwise,
                                    // the system will try to rebuild again.
                                    var homeNode = new XmlSitemapNode {
                                        LastMod = DateTime.UtcNow, Loc = data.BaseUrl
                                    };
                                    var documents = GetSiteMapDocuments(new List <XmlSitemapNode> {
                                        homeNode
                                    });
                                    await SaveTempAsync(documents, data, 0);
                                }
                            }
                        }
                    }

                    ctx.CancellationToken.ThrowIfCancellationRequested();

                    ctx.ProgressCallback?.Invoke(totalSegments, totalSegments, "Finalizing...'");

                    foreach (var data in languageData.Values)
                    {
                        // Create index documents (if any)
                        if (hasIndex && indexNodes.Any())
                        {
                            var indexDocument = CreateSitemapIndexDocument(indexNodes[data.Language.Id]);
                            await SaveTempAsync(new List <string> {
                                indexDocument
                            }, data, 0);
                        }

                        // Save finally (actually renames temp folder)
                        SaveFinal(data);
                    }
                }
                finally
                {
                    // Undo impersonation
                    _services.WorkContext.CurrentCustomer = prevCustomer;
                    sitemaps.Clear();

                    foreach (var data in languageData.Values)
                    {
                        if (_tenantFolder.DirectoryExists(data.TempDir))
                        {
                            _tenantFolder.TryDeleteDirectory(data.TempDir);
                        }
                    }

                    //GC.Collect();
                    //GC.WaitForPendingFinalizers();
                }
            }
        }
示例#6
0
        private async Task <XmlSitemapPartition> GetSitemapPartAsync(int index, bool isRetry)
        {
            Guard.NotNegative(index, nameof(index));

            var store    = _services.StoreContext.CurrentStore;
            var language = _services.WorkContext.WorkingLanguage;

            var exists = SitemapFileExists(store.Id, language.Id, index, out var path, out var name);

            if (exists)
            {
                return(new XmlSitemapPartition
                {
                    Index = index,
                    Name = name,
                    LanguageId = language.Id,
                    StoreId = store.Id,
                    ModifiedOnUtc = _tenantFolder.GetFileLastWriteTimeUtc(path),
                    Stream = _tenantFolder.OpenFile(path)
                });
            }

            if (isRetry)
            {
                var msg = "Could not generate XML sitemap. Index: {0}, Date: {1}".FormatInvariant(index, DateTime.UtcNow);
                Logger.Error(msg);
                throw new SmartException(msg);
            }

            if (index > 0)
            {
                // File with index greater 0 has been requested, but it does not exist.
                // Now we have to determine whether just the passed index is out of range
                // or the files has never been created before.
                // If the main file (index 0) exists, the action should return NotFoundResult,
                // otherwise the rebuild process should be started or waited for.

                if (SitemapFileExists(store.Id, language.Id, 0, out path, out name))
                {
                    throw new IndexOutOfRangeException("The sitemap file '{0}' does not exist.".FormatInvariant(name));
                }
            }

            // The main sitemap document with index 0 does not exist, meaning: the whole sitemap
            // needs to be created and cached by partitions.

            var wasRebuilding = false;
            var lockFilePath  = GetLockFilePath(store.Id, language.Id);

            while (IsRebuilding(lockFilePath))
            {
                // The rebuild process is already running, either started
                // by the task scheduler or another HTTP request.
                // We should wait for completion.

                wasRebuilding = true;
                Thread.Sleep(1000);
            }

            if (!wasRebuilding)
            {
                // No lock. Rebuild now.
                var buildContext = new XmlSitemapBuildContext(store, new[] { language })
                {
                    CancellationToken = CancellationToken.None
                };

                await RebuildAsync(buildContext);
            }

            // DRY: call self to get sitemap partition object
            return(await GetSitemapPartAsync(index, true));
        }