private bool DeleteProducts( IReadOnlyCollection <ProductCatalog> productCatalogs, ICatalogPublisher catalogPublisher) { Stopwatch watchTotalDelete = Stopwatch.StartNew(); bool changesDetected = false; //// 1: delete listings for the catalogs which are no longer exist in AX (were retracted for instance). //// This is the one of 2 fastest steps (because we don't need to query CRT for each product to figure out whether it is still there or not) DataAccessor dataAccessor = new DataAccessor(this.onlineChannel.RecordId, this.runtime.Configuration.ConnectionString, this.publishingConfig.CRTListingPageSize); Stopwatch watch = Stopwatch.StartNew(); Dictionary <string, List <long> > catalogsToBeDeleted = dataAccessor.GetNotExistingCatalogs(productCatalogs); watch.Stop(); this.LogTimingMessage(Resources.Duration_GetNotExistingCatalogs, watch.Elapsed, catalogsToBeDeleted.Count, catalogsToBeDeleted.Values.Count); if (catalogsToBeDeleted.Count > 0) { watch.Restart(); catalogPublisher.OnDeleteProductsByCatalogIdRequested(catalogsToBeDeleted); watch.Stop(); this.LogTimingMessage(Resources.Duration_Processor_DeleteListingsByCatalogs, watch.Elapsed); } watch.Restart(); dataAccessor.DeleteListingsByCatalogs(catalogsToBeDeleted.SelectMany(c => c.Value)); watch.Stop(); this.LogTimingMessage(Resources.Duration_Manager_DeleteListingsByCatalogs, watch.Elapsed); // 2. delete listings for languages which are no longer exist on channel, this is another fast operation (in terms of querying CRT). watch.Restart(); Dictionary <string, List <string> > languagesToBeDeleted = dataAccessor.GetNotExistingLanguages(this.ChannelLanguages); watch.Stop(); this.LogTimingMessage(Resources.Duration_GetNotExistingLanguages, watch.Elapsed, languagesToBeDeleted.Keys.Count, languagesToBeDeleted.Values.Count); if (languagesToBeDeleted.Count > 0) { watch.Restart(); catalogPublisher.OnDeleteProductsByLanguageIdRequested(languagesToBeDeleted); watch.Stop(); this.LogTimingMessage(Resources.Duration_Processor_DeleteListingsByLanguage, watch.Elapsed); } if (languagesToBeDeleted.Count > 0) { watch.Restart(); dataAccessor.DeleteListingsByLanguages(languagesToBeDeleted.SelectMany(c => c.Value)); watch.Stop(); this.LogTimingMessage(Resources.Duration_Processor_DeleteListingsByLanguages, watch.Elapsed); } changesDetected |= (catalogsToBeDeleted.Count > 0) || (languagesToBeDeleted.Count > 0); if (this.publishingConfig.CheckEveryListingForRemoval) { // 3: Finally read all listings left from published listings table and ask CRT whehter the product still available there or not watch.Restart(); Dictionary <long, List <ListingIdentity> > catalogs = dataAccessor.LoadAllListingsMap(); watch.Stop(); int listingsCount = 0; foreach (List <ListingIdentity> list in catalogs.Values) { listingsCount += list.Count; } this.LogTimingMessage(Resources.Duration_LoadListingsMap, watch.Elapsed, listingsCount, catalogs.Keys.Count); // Loop over published listings which are grouped by a catalog foreach (KeyValuePair <long, List <ListingIdentity> > catalog in catalogs) { int bottomIndex = 0; List <ListingIdentity> publishedIds = catalog.Value; // Calling CRT, in a separate pages, to find out whether the products are still available or not. while (bottomIndex < publishedIds.Count) { int topIndex = bottomIndex + this.publishingConfig.CRTListingPageSize - 1; if (topIndex + 1 >= publishedIds.Count) { topIndex = publishedIds.Count - 1; } List <ListingIdentity> currentPagePublishedIds = publishedIds.GetRange(bottomIndex, topIndex + 1 - bottomIndex); int previousBottomIndex = bottomIndex; bottomIndex = topIndex + 1; watch.Restart(); System.Collections.ObjectModel.ReadOnlyCollection <ProductExistenceId> crtIds = this.VerifyProductExistence(catalog.Key, ConvertToProductExistenceId(currentPagePublishedIds)); watch.Stop(); this.LogTimingMessage(Resources.Duration_VerifyProductExistence, watch.Elapsed, currentPagePublishedIds.Count, previousBottomIndex, publishedIds.Count, crtIds.Count); IList <ListingIdentity> idsToBeRemoved = GetIdsToBeRemoved(crtIds, currentPagePublishedIds); changesDetected |= idsToBeRemoved.Any(); if (idsToBeRemoved.Count > 0) { watch.Restart(); catalogPublisher.OnDeleteIndividualProductsRequested(idsToBeRemoved); watch.Stop(); this.LogTimingMessage(Resources.Duration_Processor_DeleteListingsByCompositeIds, watch.Elapsed, idsToBeRemoved.Count); watch.Restart(); dataAccessor.DeleteListingsByCompositeIds(catalog.Key, idsToBeRemoved); watch.Stop(); this.LogTimingMessage(Resources.Duration_Manager_DeleteListingsByCompositeIds, watch.Elapsed, idsToBeRemoved.Count); } List <ListingPublishStatus> statuses = new List <ListingPublishStatus>(); foreach (ListingIdentity id in idsToBeRemoved) { statuses.Add(Listing.CreateStatusSuccessfullyDeleted(this.onlineChannel.RecordId, id.CatalogId, id.ProductId, id.LanguageId)); } if (statuses.Count > 0) { this.UpdateListingPublishingStatus(statuses); } } } watchTotalDelete.Stop(); this.LogTimingMessage(Resources.Duration_DeleteProducts, watchTotalDelete.Elapsed, changesDetected); } return(changesDetected); }
/// <summary> /// Initiates a catalog publishing. /// </summary> /// <param name="catalogPublisher">Instance of the object which implements ICatalogPublisher.</param> /// <returns>True if changed products were found in CRT, False otherwise.</returns> /// <remarks>Retrieves the channel's catalogs from CRT and then checks whether CRT contains changed products for each of the catalogs. If changed products are found then /// ICatalogPublisher's callbacks are executed to let the caller's code process changed products.</remarks> public bool PublishCatalog(ICatalogPublisher catalogPublisher) { if (catalogPublisher == null) { throw new ArgumentNullException(nameof(catalogPublisher)); } List <long> productCatalogIds = new List <long>(1); // If catalogs were published to this channel, a given product will be published into SP for each catalog // in which it appears, so catalogless publishing would not yield different results for those products. // If, however, a product was published directly from the assortment, that product will only be detected // and published to SP if the ForceCataloglessPublishing flag is set to 'true' (1) in the job configuration file. // The semantics of forcing catalogless publishing as strict, in that catalog-less products will be published // if and only if the flag is set. That means, for instance, that if the flag is not set and there are no // catalogs published to this channel, the SP job will not detect/publish any products to SP. if (this.publishingConfig.ForceNoCatalogPublishing) { NetTracer.Information(Resources.ProductCatalogToPublish, 0, "unspecified", "(not a proper catalog)"); productCatalogIds.Add(0); } IReadOnlyCollection <ProductCatalog> productCatalogs = this.GetCatalogs(); bool deletesFound = this.DeleteProducts(productCatalogs, catalogPublisher); foreach (ProductCatalog productCatalog in productCatalogs) { productCatalogIds.Add(productCatalog.RecordId); } ChangedProductsSearchCriteria searchCriteria = new ChangedProductsSearchCriteria { DataLevel = CommerceEntityDataLevel.Complete }; searchCriteria.Context.ChannelId = this.onlineChannel.RecordId; bool isInitialSync; QueryResultSettings productsQuerySettings = this.CreateGetListingsCriteria( this.onlineChannel.ChannelProperties, searchCriteria, out isInitialSync); bool changesFound = false; try { Stopwatch readChangedProductsWatch = Stopwatch.StartNew(); searchCriteria.Session = this.productManager.BeginReadChangedProducts(searchCriteria); readChangedProductsWatch.Stop(); this.LogTimingMessage(Resources.Duration_ReadChangedProducts, readChangedProductsWatch.Elapsed, searchCriteria.Session.TotalNumberOfProducts); if (searchCriteria.Session.TotalNumberOfProducts > 0) { changesFound = true; int totalProductsCount = 0; Stopwatch timerCummulativeListingRetrieval = new Stopwatch(); // loop through the product catalogs, retrieving products. foreach (long productCatalogId in productCatalogIds) { NetTracer.Information(Resources.StartReadProductsFromCatalog, productCatalogId); // set the catalog id on the search criteria searchCriteria.Context.CatalogId = productCatalogId; searchCriteria.Session.ResetNumberOfProductsRead(); int pageNumberForCatalog = 0; int catalogProductsCount = 0; // inner loop: load changes, page by page, up to catalog max size do { timerCummulativeListingRetrieval.Start(); ChangedProductsSearchResult getProductsResults = this.LoadChangedProducts(searchCriteria, productsQuerySettings); timerCummulativeListingRetrieval.Stop(); int numberOfReadProducts = getProductsResults.Results.Count; totalProductsCount += numberOfReadProducts; catalogProductsCount += numberOfReadProducts; this.LogTimingMessage(Resources.NumberOfReadProductsInPageSummary, productCatalogId, catalogProductsCount, totalProductsCount, timerCummulativeListingRetrieval.Elapsed); catalogPublisher.OnChangedProductsFound(getProductsResults, pageNumberForCatalog, productCatalogId); pageNumberForCatalog++; }while (searchCriteria.Session.NumberOfProductsRead < searchCriteria.Session.TotalNumberOfProducts); this.LogTimingMessage(Resources.CatalogReadCompleted, productCatalogId, catalogProductsCount, totalProductsCount, timerCummulativeListingRetrieval.Elapsed); catalogPublisher.OnCatalogReadCompleted(productCatalogId, this); } // for each product catalog this.LogTimingMessage(Resources.AllCatalogsReadCompleted, totalProductsCount, timerCummulativeListingRetrieval.Elapsed); } // if changed products were found } finally { this.productManager.EndReadChangedProducts(searchCriteria.Session); } ChannelProperty channelProperty = new ChannelProperty { Name = KeySyncAnchor, Value = new string(searchCriteria.Session.NextSynchronizationToken) }; this.channelManager.UpdateChannelProperties(new ChannelProperty[] { channelProperty }); return(changesFound || deletesFound); }