private bool DeleteProducts(
IReadOnlyCollection <ProductCatalog> productCatalogs,
ICatalogPublisher catalogPublisher)
{
Stopwatch watchTotalDelete = Stopwatch.StartNew();
bool changesDetected = false;
//// 1: delete listings for the catalogs which are no longer exist in AX (were retracted for instance).
//// This is the one of 2 fastest steps (because we don't need to query CRT for each product to figure out whether it is still there or not)
DataAccessor dataAccessor = new DataAccessor(this.onlineChannel.RecordId, this.runtime.Configuration.ConnectionString, this.publishingConfig.CRTListingPageSize);
Stopwatch watch = Stopwatch.StartNew();
Dictionary <string, List <long> > catalogsToBeDeleted = dataAccessor.GetNotExistingCatalogs(productCatalogs);
watch.Stop();
this.LogTimingMessage(Resources.Duration_GetNotExistingCatalogs, watch.Elapsed, catalogsToBeDeleted.Count, catalogsToBeDeleted.Values.Count);
if (catalogsToBeDeleted.Count > 0)
{
watch.Restart();
catalogPublisher.OnDeleteProductsByCatalogIdRequested(catalogsToBeDeleted);
watch.Stop();
this.LogTimingMessage(Resources.Duration_Processor_DeleteListingsByCatalogs, watch.Elapsed);
}
watch.Restart();
dataAccessor.DeleteListingsByCatalogs(catalogsToBeDeleted.SelectMany(c => c.Value));
watch.Stop();
this.LogTimingMessage(Resources.Duration_Manager_DeleteListingsByCatalogs, watch.Elapsed);
// 2. delete listings for languages which are no longer exist on channel, this is another fast operation (in terms of querying CRT).
watch.Restart();
Dictionary <string, List <string> > languagesToBeDeleted = dataAccessor.GetNotExistingLanguages(this.ChannelLanguages);
watch.Stop();
this.LogTimingMessage(Resources.Duration_GetNotExistingLanguages, watch.Elapsed, languagesToBeDeleted.Keys.Count, languagesToBeDeleted.Values.Count);
if (languagesToBeDeleted.Count > 0)
{
watch.Restart();
catalogPublisher.OnDeleteProductsByLanguageIdRequested(languagesToBeDeleted);
watch.Stop();
this.LogTimingMessage(Resources.Duration_Processor_DeleteListingsByLanguage, watch.Elapsed);
}
if (languagesToBeDeleted.Count > 0)
{
watch.Restart();
dataAccessor.DeleteListingsByLanguages(languagesToBeDeleted.SelectMany(c => c.Value));
watch.Stop();
this.LogTimingMessage(Resources.Duration_Processor_DeleteListingsByLanguages, watch.Elapsed);
}
changesDetected |= (catalogsToBeDeleted.Count > 0) || (languagesToBeDeleted.Count > 0);
if (this.publishingConfig.CheckEveryListingForRemoval)
{
// 3: Finally read all listings left from published listings table and ask CRT whehter the product still available there or not
watch.Restart();
Dictionary <long, List <ListingIdentity> > catalogs = dataAccessor.LoadAllListingsMap();
watch.Stop();
int listingsCount = 0;
foreach (List <ListingIdentity> list in catalogs.Values)
{
listingsCount += list.Count;
}
this.LogTimingMessage(Resources.Duration_LoadListingsMap, watch.Elapsed, listingsCount, catalogs.Keys.Count);
// Loop over published listings which are grouped by a catalog
foreach (KeyValuePair <long, List <ListingIdentity> > catalog in catalogs)
{
int bottomIndex = 0;
List <ListingIdentity> publishedIds = catalog.Value;
// Calling CRT, in a separate pages, to find out whether the products are still available or not.
while (bottomIndex < publishedIds.Count)
{
int topIndex = bottomIndex + this.publishingConfig.CRTListingPageSize - 1;
if (topIndex + 1 >= publishedIds.Count)
{
topIndex = publishedIds.Count - 1;
}
List <ListingIdentity> currentPagePublishedIds = publishedIds.GetRange(bottomIndex, topIndex + 1 - bottomIndex);
int previousBottomIndex = bottomIndex;
bottomIndex = topIndex + 1;
watch.Restart();
System.Collections.ObjectModel.ReadOnlyCollection <ProductExistenceId> crtIds = this.VerifyProductExistence(catalog.Key, ConvertToProductExistenceId(currentPagePublishedIds));
watch.Stop();
this.LogTimingMessage(Resources.Duration_VerifyProductExistence, watch.Elapsed, currentPagePublishedIds.Count, previousBottomIndex, publishedIds.Count, crtIds.Count);
IList <ListingIdentity> idsToBeRemoved = GetIdsToBeRemoved(crtIds, currentPagePublishedIds);
changesDetected |= idsToBeRemoved.Any();
if (idsToBeRemoved.Count > 0)
{
watch.Restart();
catalogPublisher.OnDeleteIndividualProductsRequested(idsToBeRemoved);
watch.Stop();
this.LogTimingMessage(Resources.Duration_Processor_DeleteListingsByCompositeIds, watch.Elapsed, idsToBeRemoved.Count);
watch.Restart();
dataAccessor.DeleteListingsByCompositeIds(catalog.Key, idsToBeRemoved);
watch.Stop();
this.LogTimingMessage(Resources.Duration_Manager_DeleteListingsByCompositeIds, watch.Elapsed, idsToBeRemoved.Count);
}
List <ListingPublishStatus> statuses = new List <ListingPublishStatus>();
foreach (ListingIdentity id in idsToBeRemoved)
{
statuses.Add(Listing.CreateStatusSuccessfullyDeleted(this.onlineChannel.RecordId, id.CatalogId, id.ProductId, id.LanguageId));
}
if (statuses.Count > 0)
{
this.UpdateListingPublishingStatus(statuses);
}
}
}
watchTotalDelete.Stop();
this.LogTimingMessage(Resources.Duration_DeleteProducts, watchTotalDelete.Elapsed, changesDetected);
}
return(changesDetected);
}
/// <summary>
/// Initiates a catalog publishing.
/// </summary>
/// <param name="catalogPublisher">Instance of the object which implements ICatalogPublisher.</param>
/// <returns>True if changed products were found in CRT, False otherwise.</returns>
/// <remarks>Retrieves the channel's catalogs from CRT and then checks whether CRT contains changed products for each of the catalogs. If changed products are found then
/// ICatalogPublisher's callbacks are executed to let the caller's code process changed products.</remarks>
public bool PublishCatalog(ICatalogPublisher catalogPublisher)
{
if (catalogPublisher == null)
{
throw new ArgumentNullException(nameof(catalogPublisher));
}
List <long> productCatalogIds = new List <long>(1);
// If catalogs were published to this channel, a given product will be published into SP for each catalog
// in which it appears, so catalogless publishing would not yield different results for those products.
// If, however, a product was published directly from the assortment, that product will only be detected
// and published to SP if the ForceCataloglessPublishing flag is set to 'true' (1) in the job configuration file.
// The semantics of forcing catalogless publishing as strict, in that catalog-less products will be published
// if and only if the flag is set. That means, for instance, that if the flag is not set and there are no
// catalogs published to this channel, the SP job will not detect/publish any products to SP.
if (this.publishingConfig.ForceNoCatalogPublishing)
{
NetTracer.Information(Resources.ProductCatalogToPublish, 0, "unspecified", "(not a proper catalog)");
productCatalogIds.Add(0);
}
IReadOnlyCollection <ProductCatalog> productCatalogs = this.GetCatalogs();
bool deletesFound = this.DeleteProducts(productCatalogs, catalogPublisher);
foreach (ProductCatalog productCatalog in productCatalogs)
{
productCatalogIds.Add(productCatalog.RecordId);
}
ChangedProductsSearchCriteria searchCriteria = new ChangedProductsSearchCriteria
{
DataLevel = CommerceEntityDataLevel.Complete
};
searchCriteria.Context.ChannelId = this.onlineChannel.RecordId;
bool isInitialSync;
QueryResultSettings productsQuerySettings = this.CreateGetListingsCriteria(
this.onlineChannel.ChannelProperties,
searchCriteria,
out isInitialSync);
bool changesFound = false;
try
{
Stopwatch readChangedProductsWatch = Stopwatch.StartNew();
searchCriteria.Session = this.productManager.BeginReadChangedProducts(searchCriteria);
readChangedProductsWatch.Stop();
this.LogTimingMessage(Resources.Duration_ReadChangedProducts, readChangedProductsWatch.Elapsed, searchCriteria.Session.TotalNumberOfProducts);
if (searchCriteria.Session.TotalNumberOfProducts > 0)
{
changesFound = true;
int totalProductsCount = 0;
Stopwatch timerCummulativeListingRetrieval = new Stopwatch();
// loop through the product catalogs, retrieving products.
foreach (long productCatalogId in productCatalogIds)
{
NetTracer.Information(Resources.StartReadProductsFromCatalog, productCatalogId);
// set the catalog id on the search criteria
searchCriteria.Context.CatalogId = productCatalogId;
searchCriteria.Session.ResetNumberOfProductsRead();
int pageNumberForCatalog = 0;
int catalogProductsCount = 0;
// inner loop: load changes, page by page, up to catalog max size
do
{
timerCummulativeListingRetrieval.Start();
ChangedProductsSearchResult getProductsResults = this.LoadChangedProducts(searchCriteria, productsQuerySettings);
timerCummulativeListingRetrieval.Stop();
int numberOfReadProducts = getProductsResults.Results.Count;
totalProductsCount += numberOfReadProducts;
catalogProductsCount += numberOfReadProducts;
this.LogTimingMessage(Resources.NumberOfReadProductsInPageSummary, productCatalogId, catalogProductsCount, totalProductsCount, timerCummulativeListingRetrieval.Elapsed);
catalogPublisher.OnChangedProductsFound(getProductsResults, pageNumberForCatalog, productCatalogId);
pageNumberForCatalog++;
}while (searchCriteria.Session.NumberOfProductsRead < searchCriteria.Session.TotalNumberOfProducts);
this.LogTimingMessage(Resources.CatalogReadCompleted, productCatalogId, catalogProductsCount, totalProductsCount, timerCummulativeListingRetrieval.Elapsed);
catalogPublisher.OnCatalogReadCompleted(productCatalogId, this);
} // for each product catalog
this.LogTimingMessage(Resources.AllCatalogsReadCompleted, totalProductsCount, timerCummulativeListingRetrieval.Elapsed);
} // if changed products were found
}
finally
{
this.productManager.EndReadChangedProducts(searchCriteria.Session);
}
ChannelProperty channelProperty = new ChannelProperty
{
Name = KeySyncAnchor,
Value = new string(searchCriteria.Session.NextSynchronizationToken)
};
this.channelManager.UpdateChannelProperties(new ChannelProperty[] { channelProperty });
return(changesFound || deletesFound);
}
