private void ProcessItems(IUnitOfWork db, IMarketApi api, ITime time, ListingLineProcessing listingProcessor, IList <ItemDTO> reportListings, bool itemsWasModified) { Log.Debug("ProcessItems begin"); var syncInfo = Context.SyncInformer; //Exclude standalone listings Log.Debug("Before exclude standalone listings: " + reportListings.Count()); var stanaloneSKUs = reportListings.Where(l => !String.IsNullOrEmpty(l.ASIN) && StringHelper.ContainsNoCase(l.SKU, "-" + l.ASIN)) .Select(l => l.SKU) .ToList(); Log.Debug("Standalone SKUs: " + stanaloneSKUs.Count()); Log.Debug("SKUs: " + String.Join(", ", stanaloneSKUs)); reportListings = reportListings.Where(l => !stanaloneSKUs.Contains(l.SKU)).ToList(); var publishingInProgressListings = (from i in db.Items.GetAll() join l in db.Listings.GetAll() on i.Id equals l.ItemId where l.Market == (int)api.Market && (l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId)) && (i.ItemPublishedStatus == (int)PublishedStatuses.New || i.ItemPublishedStatus == (int)PublishedStatuses.PublishingErrors || i.ItemPublishedStatus == (int)PublishedStatuses.PublishedInProgress || i.ItemPublishedStatus == (int)PublishedStatuses.PublishingErrors) && !l.IsRemoved select new { SKU = l.SKU, ListingId = l.ListingId }).ToList(); var publishingInProgressListingIds = publishingInProgressListings.Select(l => l.ListingId).ToList(); var publishingInProgressSKUs = publishingInProgressListings.Select(l => l.SKU).ToList(); #region Step 1. Mark all except parsing as removed if (!itemsWasModified) { //TODO: Temp removed new listings //var excludeFromRemoveListingIds = reportListings.Select(r => r.ListingId).ToList(); //excludeFromRemoveListingIds.AddRange(publishingInProgressListingIds); //var removedList = db.Listings.MarkNotExistingAsRemoved(excludeFromRemoveListingIds, api.Market, api.MarketplaceId); //foreach (var removedItem in removedList) // Log.Debug("Mark as removed, listingId=" + removedItem); } #endregion #region Process new report listings //Get all not exist in DB //var newReportListings = db.Listings.CheckForExistence(reportListings, api.Market, api.MarketplaceId); //var existListingId = db.Listings.GetFiltered(l => !l.IsRemoved // && l.Market == (int)api.Market // && l.MarketplaceId == api.MarketplaceId) // .Select(l => l.ListingId) // .ToList(); var existListingSKUs = db.Listings.GetFiltered(l => !l.IsRemoved && l.Market == (int)api.Market && l.MarketplaceId == api.MarketplaceId) .Select(l => l.SKU) .ToList(); var newReportListings = reportListings.Where(r => !existListingSKUs.Contains(r.SKU)).ToList(); newReportListings = newReportListings.Where(l => !publishingInProgressSKUs.Contains(l.SKU)).ToList(); foreach (var newListingItem in newReportListings) { Log.Debug("New listing, listingId=" + newListingItem.ListingId); } var listingsWithNewParents = new List <ItemDTO>(); if (newReportListings.Any()) { var newListingsWithError = new List <ItemDTO>(); //0. Get ParentASIN for listings (and other infos) try { api.FillWithAdditionalInfo(Log, time, newReportListings, IdType.SKU, ItemFillMode.NoAdv, //NOTE: TODO: Request barcodes by separate service out newListingsWithError); } catch (Exception ex) //Can continue if only part of records was filled { syncInfo.AddError("", "Can't fill new listing items with additional info", ex); Log.Error("Can't fill new listing items with additional info", ex); } Log.Debug("Error when GetItems for new listings, asins: " + String.Join(", ", newListingsWithError.Select(i => i.ASIN).ToList())); //1. Process Items listingProcessor.ProcessNewListingsWithItems(db, api, time, newReportListings); //2. Process ParentItems, and creating styles based on there names var listingsWithParentASIN = newReportListings.Where(l => !String.IsNullOrEmpty(l.ParentASIN)).ToList(); listingsWithNewParents = db.ParentItems.CheckForExistence(listingsWithParentASIN, api.Market, api.MarketplaceId); if (listingsWithNewParents.Any()) { Log.Debug("Process new parents"); listingProcessor.ProcessNewParents(db, api, listingsWithNewParents, newReportListings); } } #endregion #region Step 3. Process existing listings var existingListings = reportListings.Where(r => !newReportListings.Select(i => i.SKU).Contains(r.SKU)).ToList(); existingListings = existingListings.Where(l => !publishingInProgressSKUs.Contains(l.SKU)).ToList(); if (existingListings.Any()) { var existingListingsWithError = new List <ItemDTO>(); //0. Get ParentASIN for listings (and other infos) //NOTE: Get base info. Should go after "get barcodes" try { api.FillWithAdditionalInfo(Log, time, existingListings, IdType.SKU, ItemFillMode.NoAdv, out existingListingsWithError); } catch (Exception ex) { syncInfo.AddError("", "Can't fill exist items with additional info", ex); Log.Error("Can't fill exist items with additional info", ex); } Log.Debug("Error when GetItems for existing listings, asins: " + String.Join(", ", existingListingsWithError.Select(i => i.ASIN).ToList())); //1. Process Items (with remap ParentASIN) listingProcessor.ProcessExistingItems(db, api, existingListings); //2. Process ParentItems, and creating styles based on there names //Need in some rare cases when items have empty Parent Item, ex.: parent item asin was changed but not for all product items var parentsASINs = existingListings.Where(l => !String.IsNullOrEmpty(l.ParentASIN)).ToList(); var newParents = db.ParentItems.CheckForExistence(parentsASINs, api.Market, api.MarketplaceId); if (newParents.Any()) { listingsWithNewParents.AddRange(newParents); listingProcessor.ProcessNewParents(db, api, newParents, existingListings); } } var processedParentASINs = listingsWithNewParents.Select(l => l.ParentASIN).ToList(); var existParents = reportListings.Where(r => !processedParentASINs.Contains(r.ParentASIN) && !String.IsNullOrEmpty(r.ParentASIN)).ToList(); if (existParents.Any()) { listingProcessor.ProcessExistingParents(db, api, syncInfo, existParents.Select(p => p.ParentASIN).Distinct().ToList(), reportListings); } #endregion Log.Debug("ProcessItems end"); }
private void ProcessItems(IUnitOfWork db, IMarketApi api, ITime time, IList <ItemDTO> listings) { var syncInfo = Context.SyncInformer; Log.Debug("ProcessItems begin"); var allDbListings = db.Listings.GetAll().Where(l => l.Market == (int)api.Market && (l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId))) .ToList(); var allDbItems = db.Items.GetAll().Where(l => l.Market == (int)api.Market && (l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId))) .ToList(); var updateItemIds = new List <long>(); var existsListingSKUs = new List <string>(); //STEP 1. Update listings ASINs Log.Debug("Update ASINs begin"); foreach (var listing in listings) { var dbListing = allDbListings.FirstOrDefault(l => l.SKU == listing.SKU); if (dbListing != null) { existsListingSKUs.Add(listing.SKU); if (dbListing.ListingId != listing.ListingId) { dbListing.ListingId = listing.ListingId; } if (dbListing.ASIN != listing.ASIN) { dbListing.ASIN = listing.ASIN; } if (dbListing.AmazonRealQuantity != listing.RealQuantity) { Log.Debug("Price changed: " + dbListing.SKU + ": " + dbListing.AmazonRealQuantity + "=>" + listing.RealQuantity); dbListing.AmazonRealQuantity = listing.RealQuantity; dbListing.AmazonRealQuantityUpdateDate = time.GetAppNowTime(); } if (dbListing.AmazonCurrentPrice != listing.CurrentPrice) { Log.Debug("Price changed: " + dbListing.SKU + ": " + dbListing.AmazonCurrentPrice + "=>" + listing.CurrentPrice); dbListing.AmazonCurrentPrice = listing.CurrentPrice; dbListing.AmazonCurrentPriceUpdateDate = time.GetAppNowTime(); } var dbItem = allDbItems.FirstOrDefault(i => i.Id == dbListing.ItemId); if (dbItem != null) { if (dbItem.ASIN != listing.ASIN) { Log.Debug("Item ASIN changed: " + dbItem.Id + ": " + dbItem.ASIN + " => " + listing.ASIN); dbItem.ASIN = listing.ASIN; } if (dbItem.SourceMarketId != listing.ASIN) { dbItem.SourceMarketId = listing.ASIN; } if (dbItem.ItemPublishedStatusFromMarket != listing.PublishedStatus) { dbItem.ItemPublishedStatusFromMarket = listing.PublishedStatus; dbItem.ItemPublishedStatusFromMarketDate = time.GetAppNowTime(); } if (listing.PublishedStatus == (int)PublishedStatuses.Published) { dbItem.ItemPublishedStatusBeforeRepublishing = dbItem.ItemPublishedStatus; dbItem.ItemPublishedStatus = listing.PublishedStatus; dbItem.ItemPublishedStatusReason = "Listings report"; dbItem.ItemPublishedStatusDate = time.GetAppNowTime(); } updateItemIds.Add(dbItem.Id); } } } db.Commit(); Log.Debug("Update ASINs end"); //STEP 1.2. Market all not exists as unpublished Log.Debug("Update items Unpublished begin"); var notUpdatedItems = allDbItems.Where(i => !updateItemIds.Contains(i.Id)).ToList(); Log.Debug("Not updated items: " + notUpdatedItems.Count()); foreach (var dbItem in notUpdatedItems) { if (dbItem.ItemPublishedStatus == (int)PublishedStatuses.Published) { Log.Debug("Status changes for: " + dbItem.ASIN + " - " + dbItem.ItemPublishedStatus + "=>" + PublishedStatuses.New); dbItem.ItemPublishedStatusBeforeRepublishing = dbItem.ItemPublishedStatus; dbItem.ItemPublishedStatusReason = "System Warning: the listing has the Published status, but it does not appear in the listing report."; dbItem.ItemPublishedStatusDate = time.GetAppNowTime(); dbItem.ItemPublishedStatus = (int)PublishedStatuses.New; dbItem.IsAmazonParentASIN = false; dbItem.IsExistOnAmazon = false; } } db.Commit(); Log.Debug("Update items Unpublished end"); //STEP 2. Update ParentASINs Log.Debug("Update ParentASINs begin"); var newListingsWithError = new List <ItemDTO>(); try { api.FillWithAdditionalInfo(Log, time, listings, IdType.SKU, ItemFillMode.Defualt, out newListingsWithError); } catch (Exception ex) //Can continue if only part of records was filled { syncInfo.AddError("", "Can't fill new listing items with additional info", ex); Log.Error("Can't fill new listing items with additional info", ex); } var allDbParents = db.ParentItems.GetAll().Where(pi => pi.Market == (int)api.Market && (pi.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId))).ToList(); var updatedParentIds = new List <long>(); foreach (var listing in listings) { var dbListing = allDbListings.FirstOrDefault(l => l.SKU == listing.SKU); var dbItem = dbListing == null ? null : allDbItems.FirstOrDefault(i => i.Id == dbListing.ItemId); var dbParentItem = dbItem == null ? null : allDbParents.FirstOrDefault(pi => pi.ASIN == dbItem.ParentASIN); if (dbItem != null && dbItem.IsExistOnAmazon != listing.IsExistOnAmazon) { Log.Debug("IsExistOnAmazon: " + listing.ASIN + ": " + dbItem.IsExistOnAmazon + "=>" + listing.IsExistOnAmazon); dbItem.IsExistOnAmazon = listing.IsExistOnAmazon; } var parentASIN = listing.ParentASIN; if (String.IsNullOrEmpty(parentASIN)) { parentASIN = listing.ASIN; } if (dbItem != null && dbItem.IsExistOnAmazon == true) { dbItem.IsAmazonParentASIN = !String.IsNullOrEmpty(listing.ParentASIN); dbItem.LastUpdateFromAmazon = time.GetUtcTime(); } if (dbParentItem != null && listing.IsExistOnAmazon == true) { var allItemsForParentItem = allDbItems.Where(i => i.ParentASIN == dbParentItem.ASIN).ToList(); if (dbParentItem.ASIN != parentASIN) { Log.Debug("ParentItem ASIN: " + dbParentItem.ASIN + "=>" + parentASIN); dbParentItem.ASIN = parentASIN; allItemsForParentItem.ForEach(i => i.ParentASIN = parentASIN); //UPDATE for all, in case it may have different Parents on Amazon (us last one for all) } if (listing.ParentASIN != dbParentItem.ASIN) { dbParentItem.IsAmazonUpdated = false; } else { dbParentItem.IsAmazonUpdated = listing.IsExistOnAmazon; } } } db.Commit(); Log.Debug("Update ParentASINs end"); //2.1 Update not exist parent asins var notUpdatedParents = allDbParents.Where(pi => !updatedParentIds.Contains(pi.Id)).ToList(); Log.Debug("Not exist parent items: " + notUpdatedParents.Count()); foreach (var dbParentItem in notUpdatedParents) { Log.Debug("Mark as not processed: " + dbParentItem.ASIN); dbParentItem.IsAmazonUpdated = false; } db.Commit(); //3.0 Set to inactive not exist listings var notExistListings = listings.Where(l => !existsListingSKUs.Contains(l.SKU)).ToList(); foreach (var notExistListing in notExistListings) { Log.Debug("Request qty=0 for SKU=" + notExistListing.SKU); if (Context.ActionService != null) { Context.ActionService.AddAction(db, SystemActionType.UpdateOnMarketProductQuantity, notExistListing.SKU, new UpdateQtyInput() { Market = api.Market, MarketplaceId = api.MarketplaceId, ListingId = null, SKU = notExistListing.SKU, SourceMarketId = notExistListing.SourceMarketId, NewQty = 0, }, null, null); } } Log.Debug("ProcessItems end"); }
private void ProcessItems(IMarketApi api, ITime time, ListingLineProcessing listingProcessor, IList <ItemDTO> reportListings, bool itemsWasModified) { Log.Debug("ProcessItems begin"); var syncInfo = Context.SyncInformer; var dbFactory = Context.DbFactory; //Exclude standalone listings Log.Debug("Before exclude standalone listings: " + reportListings.Count()); var stanaloneSKUs = reportListings.Where(l => !String.IsNullOrEmpty(l.ASIN) && StringHelper.ContainsNoCase(l.SKU, "-" + l.ASIN)) .Select(l => l.SKU) .ToList(); Log.Debug("Standalone SKUs: " + stanaloneSKUs.Count()); Log.Debug("SKUs: " + String.Join(", ", stanaloneSKUs)); reportListings = reportListings.Where(l => !stanaloneSKUs.Contains(l.SKU)).ToList(); IList <string> publishingInProgressListingIds = new List <string>(); using (var db = dbFactory.GetRWDb()) { var publishingPeriod = _removePeriodForPublishingInProgress; var publishingInProgressListings = (from i in db.Items.GetAll() join l in db.Listings.GetAll() on i.Id equals l.ItemId where l.Market == (int)api.Market && !l.IsRemoved && (l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId)) && (i.ItemPublishedStatus == (int)PublishedStatuses.New || i.ItemPublishedStatus == (int)PublishedStatuses.PublishingErrors || i.ItemPublishedStatus == (int)PublishedStatuses.HasChanges || i.ItemPublishedStatus == (int)PublishedStatuses.PublishedInProgress || (i.ItemPublishedStatus == (int)PublishedStatuses.Published && !i.IsExistOnAmazon.HasValue)) select new { SKU = l.SKU, ListingId = l.ListingId, CreateDate = l.CreateDate }); if (publishingPeriod.HasValue) { publishingInProgressListings = publishingInProgressListings.Where(l => l.CreateDate > publishingPeriod); } publishingInProgressListingIds = publishingInProgressListings.Select(l => l.ListingId).ToList(); } #region Step 1. Mark all except parsing as removed if (!itemsWasModified) { var excludeMarketListingIds = reportListings.Select(r => r.ListingId).ToList(); excludeMarketListingIds.AddRange(publishingInProgressListingIds); using (var db = dbFactory.GetRWDb()) { var toUnbulishListingIds = (from l in db.Listings.GetAll() join i in db.Items.GetAll() on l.ItemId equals i.Id where !String.IsNullOrEmpty(l.ListingId) && l.Market == (int)api.Market && (l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId)) && !l.IsRemoved && (i.ItemPublishedStatus == (int)PublishedStatuses.HasUnpublishRequest || i.ItemPublishedStatus == (int)PublishedStatuses.Unpublished) select l.Id) .ToList(); var allMarketListings = (from l in db.Listings.GetAll() join i in db.Items.GetAll() on l.ItemId equals i.Id where !String.IsNullOrEmpty(l.ListingId) && l.Market == (int)api.Market && (l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId)) && !l.IsRemoved //&& i.ItemPublishedStatus != (int)PublishedStatuses.HasChanges //NOTE: already going to republish select new { l.Id, l.ListingId, ItemId = i.Id, l.CreateDate }) .ToList(); var missingOnMarketListingInfoList = allMarketListings .Where(l => !excludeMarketListingIds.Contains(l.ListingId)) .Distinct() .ToList(); foreach (var listingInfo in missingOnMarketListingInfoList) { if (toUnbulishListingIds.Contains(listingInfo.Id) || (_removePeriodForPublishingInProgress.HasValue && listingInfo.CreateDate < _removePeriodForPublishingInProgress)) { //db.Items.SetItemPublishingStatus(itemId, (int)PublishedStatuses.Unpublished, "Requested by user via UnpublishRequest", time.GetAmazonNowTime()); var dbListings = db.Listings.GetAll().Where(l => l.Id == listingInfo.Id).ToList(); foreach (var dbListing in dbListings) { dbListing.IsRemoved = true; Log.Info("Listing, SKU: " + dbListing.SKU + " set as removed"); } } else { try { db.Items.SetItemPublishingStatus(listingInfo.ItemId, (int)PublishedStatuses.HasChanges, "System Warning: the listing has the Published status, but it does not appear in the listing report.", time.GetAppNowTime()); } catch (Exception ex) //NOTE: usually update twice one Item { Log.Error("SetItemPublishingStatus error, itemId=" + listingInfo.ItemId, ex); } Log.Info("Item, id: " + listingInfo.ItemId + " set to republish"); } } db.Commit(); //db.Listings.MarkAsRemoved(toRemoveDbListingIds); //TODO: change status to HasChanges to restore items Log.Debug("Missing on marketplace items, count=" + missingOnMarketListingInfoList.Count()); Log.Debug("Mark to republish, ItemIds=" + String.Join(",", missingOnMarketListingInfoList.Select(i => i.ItemId))); } } #endregion #region Process new report listings IList <ItemDTO> newReportListings = new List <ItemDTO>(); using (var db = dbFactory.GetRWDb()) { var existListingSKUs = db.Listings.GetFiltered(l => !l.IsRemoved && l.Market == (int)api.Market && l.MarketplaceId == api.MarketplaceId) .Select(l => l.SKU) .ToList(); newReportListings = reportListings.Where(r => !existListingSKUs.Contains(r.SKU)).ToList(); foreach (var newListingItem in newReportListings) { Log.Debug("New listing, listingId=" + newListingItem.ListingId); } var listingsWithNewParents = new List <ItemDTO>(); if (newReportListings.Any()) { var newListingsWithError = new List <ItemDTO>(); //0. Get ParentASIN for listings (and other infos) try { api.FillWithAdditionalInfo(Log, time, newReportListings, IdType.SKU, ItemFillMode.NoAdv, //NOTE: TODO: Request barcodes by separate service out newListingsWithError); if (_enableFakeParentASIN) { foreach (var l in newReportListings) { if (l.IsExistOnAmazon == true && String.IsNullOrEmpty(l.ParentASIN)) { l.ParentASIN = l.ASIN; } } } } catch (Exception ex) //Can continue if only part of records was filled { syncInfo.AddError("", "Can't fill new listing items with additional info", ex); Log.Error("Can't fill new listing items with additional info", ex); } Log.Debug("Listings with errors when GetItems for new listings, asins: " + String.Join(", ", newListingsWithError.Select(i => i.ASIN).ToList())); //1. Process Items listingProcessor.ProcessNewListingsWithItems(db, api, time, newReportListings); //2. Process ParentItems, and creating styles based on there names var listingsWithParentASIN = newReportListings.Where(l => !String.IsNullOrEmpty(l.ParentASIN)).ToList(); listingsWithNewParents = db.ParentItems.CheckForExistence(listingsWithParentASIN, api.Market, api.MarketplaceId); if (listingsWithNewParents.Any()) { Log.Debug("Process new parents"); listingProcessor.ProcessNewParents(db, api, listingsWithNewParents, newReportListings); } } } #endregion #region Step 3. Process existing listings var existingListings = reportListings.Where(r => !newReportListings.Select(i => i.SKU).Contains(r.SKU)).ToList(); if (existingListings.Any()) { var existingListingsWithError = new List <ItemDTO>(); //0. Get ParentASIN for listings (and other infos) //NOTE: Get base info. Should go after "get barcodes" try { api.FillWithAdditionalInfo(Log, time, existingListings, IdType.SKU, ItemFillMode.NoAdv, out existingListingsWithError); if (_enableFakeParentASIN) { foreach (var l in existingListings) { if (l.IsExistOnAmazon == true && String.IsNullOrEmpty(l.ParentASIN)) { l.ParentASIN = l.ASIN; } } } } catch (Exception ex) { syncInfo.AddError("", "Can't fill exist items with additional info", ex); Log.Error("Can't fill exist items with additional info", ex); } Log.Debug("Error when GetItems for existing listings, asins: " + String.Join(", ", existingListingsWithError.Select(i => i.ASIN).ToList())); //1. Process Items (keep Parent ASIN to keep source relationships) using (var db = dbFactory.GetRWDb()) { listingProcessor.ProcessExistingItems(db, api, existingListings); } //DISABLED: we don't creating new parents for existing listings, always keep ParentASINs (on CCEN we have required relationships, real ParentASIN may be others) //2. Process ParentItems, and creating styles based on there names //Need in some rare cases when items have empty Parent Item, ex.: parent item asin was changed but not for all product items var parentsASINs = existingListings.Where(l => !String.IsNullOrEmpty(l.ParentASIN)).ToList(); using (var db = dbFactory.GetRWDb()) { var newParents = db.ParentItems.CheckForExistence(parentsASINs, api.Market, api.MarketplaceId); if (newParents.Any()) { listingProcessor.ProcessNewParents(db, api, newParents, existingListings); } } } using (var db = dbFactory.GetRWDb()) { listingProcessor.UpdateParentsForExistantItems(db, api, syncInfo); } #endregion Log.Debug("ProcessItems end"); }