private void ProcessItems(IUnitOfWork db,
IMarketApi api,
ITime time,
ListingLineProcessing listingProcessor,
IList <ItemDTO> reportListings,
bool itemsWasModified)
{
Log.Debug("ProcessItems begin");
var syncInfo = Context.SyncInformer;
//Exclude standalone listings
Log.Debug("Before exclude standalone listings: " + reportListings.Count());
var stanaloneSKUs = reportListings.Where(l => !String.IsNullOrEmpty(l.ASIN) &&
StringHelper.ContainsNoCase(l.SKU, "-" + l.ASIN))
.Select(l => l.SKU)
.ToList();
Log.Debug("Standalone SKUs: " + stanaloneSKUs.Count());
Log.Debug("SKUs: " + String.Join(", ", stanaloneSKUs));
reportListings = reportListings.Where(l => !stanaloneSKUs.Contains(l.SKU)).ToList();
var publishingInProgressListings = (from i in db.Items.GetAll()
join l in db.Listings.GetAll() on i.Id equals l.ItemId
where l.Market == (int)api.Market &&
(l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId)) &&
(i.ItemPublishedStatus == (int)PublishedStatuses.New ||
i.ItemPublishedStatus == (int)PublishedStatuses.PublishingErrors ||
i.ItemPublishedStatus == (int)PublishedStatuses.PublishedInProgress ||
i.ItemPublishedStatus == (int)PublishedStatuses.PublishingErrors) &&
!l.IsRemoved
select new
{
SKU = l.SKU,
ListingId = l.ListingId
}).ToList();
var publishingInProgressListingIds = publishingInProgressListings.Select(l => l.ListingId).ToList();
var publishingInProgressSKUs = publishingInProgressListings.Select(l => l.SKU).ToList();
#region Step 1. Mark all except parsing as removed
if (!itemsWasModified)
{
//TODO: Temp removed new listings
//var excludeFromRemoveListingIds = reportListings.Select(r => r.ListingId).ToList();
//excludeFromRemoveListingIds.AddRange(publishingInProgressListingIds);
//var removedList = db.Listings.MarkNotExistingAsRemoved(excludeFromRemoveListingIds, api.Market, api.MarketplaceId);
//foreach (var removedItem in removedList)
// Log.Debug("Mark as removed, listingId=" + removedItem);
}
#endregion
#region Process new report listings
//Get all not exist in DB
//var newReportListings = db.Listings.CheckForExistence(reportListings, api.Market, api.MarketplaceId);
//var existListingId = db.Listings.GetFiltered(l => !l.IsRemoved
// && l.Market == (int)api.Market
// && l.MarketplaceId == api.MarketplaceId)
// .Select(l => l.ListingId)
// .ToList();
var existListingSKUs = db.Listings.GetFiltered(l => !l.IsRemoved &&
l.Market == (int)api.Market &&
l.MarketplaceId == api.MarketplaceId)
.Select(l => l.SKU)
.ToList();
var newReportListings = reportListings.Where(r => !existListingSKUs.Contains(r.SKU)).ToList();
newReportListings = newReportListings.Where(l => !publishingInProgressSKUs.Contains(l.SKU)).ToList();
foreach (var newListingItem in newReportListings)
{
Log.Debug("New listing, listingId=" + newListingItem.ListingId);
}
var listingsWithNewParents = new List <ItemDTO>();
if (newReportListings.Any())
{
var newListingsWithError = new List <ItemDTO>();
//0. Get ParentASIN for listings (and other infos)
try
{
api.FillWithAdditionalInfo(Log,
time,
newReportListings,
IdType.SKU,
ItemFillMode.NoAdv, //NOTE: TODO: Request barcodes by separate service
out newListingsWithError);
}
catch (Exception ex) //Can continue if only part of records was filled
{
syncInfo.AddError("", "Can't fill new listing items with additional info", ex);
Log.Error("Can't fill new listing items with additional info", ex);
}
Log.Debug("Error when GetItems for new listings, asins: " + String.Join(", ", newListingsWithError.Select(i => i.ASIN).ToList()));
//1. Process Items
listingProcessor.ProcessNewListingsWithItems(db, api, time, newReportListings);
//2. Process ParentItems, and creating styles based on there names
var listingsWithParentASIN = newReportListings.Where(l => !String.IsNullOrEmpty(l.ParentASIN)).ToList();
listingsWithNewParents = db.ParentItems.CheckForExistence(listingsWithParentASIN, api.Market, api.MarketplaceId);
if (listingsWithNewParents.Any())
{
Log.Debug("Process new parents");
listingProcessor.ProcessNewParents(db, api, listingsWithNewParents, newReportListings);
}
}
#endregion
#region Step 3. Process existing listings
var existingListings = reportListings.Where(r => !newReportListings.Select(i => i.SKU).Contains(r.SKU)).ToList();
existingListings = existingListings.Where(l => !publishingInProgressSKUs.Contains(l.SKU)).ToList();
if (existingListings.Any())
{
var existingListingsWithError = new List <ItemDTO>();
//0. Get ParentASIN for listings (and other infos)
//NOTE: Get base info. Should go after "get barcodes"
try
{
api.FillWithAdditionalInfo(Log,
time,
existingListings,
IdType.SKU,
ItemFillMode.NoAdv,
out existingListingsWithError);
}
catch (Exception ex)
{
syncInfo.AddError("", "Can't fill exist items with additional info", ex);
Log.Error("Can't fill exist items with additional info", ex);
}
Log.Debug("Error when GetItems for existing listings, asins: " + String.Join(", ", existingListingsWithError.Select(i => i.ASIN).ToList()));
//1. Process Items (with remap ParentASIN)
listingProcessor.ProcessExistingItems(db, api, existingListings);
//2. Process ParentItems, and creating styles based on there names
//Need in some rare cases when items have empty Parent Item, ex.: parent item asin was changed but not for all product items
var parentsASINs = existingListings.Where(l => !String.IsNullOrEmpty(l.ParentASIN)).ToList();
var newParents = db.ParentItems.CheckForExistence(parentsASINs, api.Market, api.MarketplaceId);
if (newParents.Any())
{
listingsWithNewParents.AddRange(newParents);
listingProcessor.ProcessNewParents(db, api, newParents, existingListings);
}
}
var processedParentASINs = listingsWithNewParents.Select(l => l.ParentASIN).ToList();
var existParents = reportListings.Where(r => !processedParentASINs.Contains(r.ParentASIN) &&
!String.IsNullOrEmpty(r.ParentASIN)).ToList();
if (existParents.Any())
{
listingProcessor.ProcessExistingParents(db,
api,
syncInfo,
existParents.Select(p => p.ParentASIN).Distinct().ToList(),
reportListings);
}
#endregion
Log.Debug("ProcessItems end");
}
private void ProcessItems(IUnitOfWork db,
IMarketApi api,
ITime time,
IList <ItemDTO> listings)
{
var syncInfo = Context.SyncInformer;
Log.Debug("ProcessItems begin");
var allDbListings = db.Listings.GetAll().Where(l => l.Market == (int)api.Market &&
(l.MarketplaceId == api.MarketplaceId ||
String.IsNullOrEmpty(api.MarketplaceId)))
.ToList();
var allDbItems = db.Items.GetAll().Where(l => l.Market == (int)api.Market &&
(l.MarketplaceId == api.MarketplaceId ||
String.IsNullOrEmpty(api.MarketplaceId)))
.ToList();
var updateItemIds = new List <long>();
var existsListingSKUs = new List <string>();
//STEP 1. Update listings ASINs
Log.Debug("Update ASINs begin");
foreach (var listing in listings)
{
var dbListing = allDbListings.FirstOrDefault(l => l.SKU == listing.SKU);
if (dbListing != null)
{
existsListingSKUs.Add(listing.SKU);
if (dbListing.ListingId != listing.ListingId)
{
dbListing.ListingId = listing.ListingId;
}
if (dbListing.ASIN != listing.ASIN)
{
dbListing.ASIN = listing.ASIN;
}
if (dbListing.AmazonRealQuantity != listing.RealQuantity)
{
Log.Debug("Price changed: " + dbListing.SKU + ": " + dbListing.AmazonRealQuantity + "=>" + listing.RealQuantity);
dbListing.AmazonRealQuantity = listing.RealQuantity;
dbListing.AmazonRealQuantityUpdateDate = time.GetAppNowTime();
}
if (dbListing.AmazonCurrentPrice != listing.CurrentPrice)
{
Log.Debug("Price changed: " + dbListing.SKU + ": " + dbListing.AmazonCurrentPrice + "=>" + listing.CurrentPrice);
dbListing.AmazonCurrentPrice = listing.CurrentPrice;
dbListing.AmazonCurrentPriceUpdateDate = time.GetAppNowTime();
}
var dbItem = allDbItems.FirstOrDefault(i => i.Id == dbListing.ItemId);
if (dbItem != null)
{
if (dbItem.ASIN != listing.ASIN)
{
Log.Debug("Item ASIN changed: " + dbItem.Id + ": " + dbItem.ASIN + " => " + listing.ASIN);
dbItem.ASIN = listing.ASIN;
}
if (dbItem.SourceMarketId != listing.ASIN)
{
dbItem.SourceMarketId = listing.ASIN;
}
if (dbItem.ItemPublishedStatusFromMarket != listing.PublishedStatus)
{
dbItem.ItemPublishedStatusFromMarket = listing.PublishedStatus;
dbItem.ItemPublishedStatusFromMarketDate = time.GetAppNowTime();
}
if (listing.PublishedStatus == (int)PublishedStatuses.Published)
{
dbItem.ItemPublishedStatusBeforeRepublishing = dbItem.ItemPublishedStatus;
dbItem.ItemPublishedStatus = listing.PublishedStatus;
dbItem.ItemPublishedStatusReason = "Listings report";
dbItem.ItemPublishedStatusDate = time.GetAppNowTime();
}
updateItemIds.Add(dbItem.Id);
}
}
}
db.Commit();
Log.Debug("Update ASINs end");
//STEP 1.2. Market all not exists as unpublished
Log.Debug("Update items Unpublished begin");
var notUpdatedItems = allDbItems.Where(i => !updateItemIds.Contains(i.Id)).ToList();
Log.Debug("Not updated items: " + notUpdatedItems.Count());
foreach (var dbItem in notUpdatedItems)
{
if (dbItem.ItemPublishedStatus == (int)PublishedStatuses.Published)
{
Log.Debug("Status changes for: " + dbItem.ASIN + " - " + dbItem.ItemPublishedStatus + "=>" + PublishedStatuses.New);
dbItem.ItemPublishedStatusBeforeRepublishing = dbItem.ItemPublishedStatus;
dbItem.ItemPublishedStatusReason = "System Warning: the listing has the Published status, but it does not appear in the listing report.";
dbItem.ItemPublishedStatusDate = time.GetAppNowTime();
dbItem.ItemPublishedStatus = (int)PublishedStatuses.New;
dbItem.IsAmazonParentASIN = false;
dbItem.IsExistOnAmazon = false;
}
}
db.Commit();
Log.Debug("Update items Unpublished end");
//STEP 2. Update ParentASINs
Log.Debug("Update ParentASINs begin");
var newListingsWithError = new List <ItemDTO>();
try
{
api.FillWithAdditionalInfo(Log,
time,
listings,
IdType.SKU,
ItemFillMode.Defualt,
out newListingsWithError);
}
catch (Exception ex) //Can continue if only part of records was filled
{
syncInfo.AddError("", "Can't fill new listing items with additional info", ex);
Log.Error("Can't fill new listing items with additional info", ex);
}
var allDbParents = db.ParentItems.GetAll().Where(pi => pi.Market == (int)api.Market &&
(pi.MarketplaceId == api.MarketplaceId ||
String.IsNullOrEmpty(api.MarketplaceId))).ToList();
var updatedParentIds = new List <long>();
foreach (var listing in listings)
{
var dbListing = allDbListings.FirstOrDefault(l => l.SKU == listing.SKU);
var dbItem = dbListing == null ? null : allDbItems.FirstOrDefault(i => i.Id == dbListing.ItemId);
var dbParentItem = dbItem == null ? null : allDbParents.FirstOrDefault(pi => pi.ASIN == dbItem.ParentASIN);
if (dbItem != null &&
dbItem.IsExistOnAmazon != listing.IsExistOnAmazon)
{
Log.Debug("IsExistOnAmazon: " + listing.ASIN + ": " + dbItem.IsExistOnAmazon + "=>" + listing.IsExistOnAmazon);
dbItem.IsExistOnAmazon = listing.IsExistOnAmazon;
}
var parentASIN = listing.ParentASIN;
if (String.IsNullOrEmpty(parentASIN))
{
parentASIN = listing.ASIN;
}
if (dbItem != null &&
dbItem.IsExistOnAmazon == true)
{
dbItem.IsAmazonParentASIN = !String.IsNullOrEmpty(listing.ParentASIN);
dbItem.LastUpdateFromAmazon = time.GetUtcTime();
}
if (dbParentItem != null &&
listing.IsExistOnAmazon == true)
{
var allItemsForParentItem = allDbItems.Where(i => i.ParentASIN == dbParentItem.ASIN).ToList();
if (dbParentItem.ASIN != parentASIN)
{
Log.Debug("ParentItem ASIN: " + dbParentItem.ASIN + "=>" + parentASIN);
dbParentItem.ASIN = parentASIN;
allItemsForParentItem.ForEach(i => i.ParentASIN = parentASIN); //UPDATE for all, in case it may have different Parents on Amazon (us last one for all)
}
if (listing.ParentASIN != dbParentItem.ASIN)
{
dbParentItem.IsAmazonUpdated = false;
}
else
{
dbParentItem.IsAmazonUpdated = listing.IsExistOnAmazon;
}
}
}
db.Commit();
Log.Debug("Update ParentASINs end");
//2.1 Update not exist parent asins
var notUpdatedParents = allDbParents.Where(pi => !updatedParentIds.Contains(pi.Id)).ToList();
Log.Debug("Not exist parent items: " + notUpdatedParents.Count());
foreach (var dbParentItem in notUpdatedParents)
{
Log.Debug("Mark as not processed: " + dbParentItem.ASIN);
dbParentItem.IsAmazonUpdated = false;
}
db.Commit();
//3.0 Set to inactive not exist listings
var notExistListings = listings.Where(l => !existsListingSKUs.Contains(l.SKU)).ToList();
foreach (var notExistListing in notExistListings)
{
Log.Debug("Request qty=0 for SKU=" + notExistListing.SKU);
if (Context.ActionService != null)
{
Context.ActionService.AddAction(db,
SystemActionType.UpdateOnMarketProductQuantity,
notExistListing.SKU,
new UpdateQtyInput()
{
Market = api.Market,
MarketplaceId = api.MarketplaceId,
ListingId = null,
SKU = notExistListing.SKU,
SourceMarketId = notExistListing.SourceMarketId,
NewQty = 0,
},
null,
null);
}
}
Log.Debug("ProcessItems end");
}
private void ProcessItems(IMarketApi api,
ITime time,
ListingLineProcessing listingProcessor,
IList <ItemDTO> reportListings,
bool itemsWasModified)
{
Log.Debug("ProcessItems begin");
var syncInfo = Context.SyncInformer;
var dbFactory = Context.DbFactory;
//Exclude standalone listings
Log.Debug("Before exclude standalone listings: " + reportListings.Count());
var stanaloneSKUs = reportListings.Where(l => !String.IsNullOrEmpty(l.ASIN) &&
StringHelper.ContainsNoCase(l.SKU, "-" + l.ASIN))
.Select(l => l.SKU)
.ToList();
Log.Debug("Standalone SKUs: " + stanaloneSKUs.Count());
Log.Debug("SKUs: " + String.Join(", ", stanaloneSKUs));
reportListings = reportListings.Where(l => !stanaloneSKUs.Contains(l.SKU)).ToList();
IList <string> publishingInProgressListingIds = new List <string>();
using (var db = dbFactory.GetRWDb())
{
var publishingPeriod = _removePeriodForPublishingInProgress;
var publishingInProgressListings = (from i in db.Items.GetAll()
join l in db.Listings.GetAll() on i.Id equals l.ItemId
where l.Market == (int)api.Market &&
!l.IsRemoved &&
(l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId)) &&
(i.ItemPublishedStatus == (int)PublishedStatuses.New ||
i.ItemPublishedStatus == (int)PublishedStatuses.PublishingErrors ||
i.ItemPublishedStatus == (int)PublishedStatuses.HasChanges ||
i.ItemPublishedStatus == (int)PublishedStatuses.PublishedInProgress ||
(i.ItemPublishedStatus == (int)PublishedStatuses.Published && !i.IsExistOnAmazon.HasValue))
select new
{
SKU = l.SKU,
ListingId = l.ListingId,
CreateDate = l.CreateDate
});
if (publishingPeriod.HasValue)
{
publishingInProgressListings = publishingInProgressListings.Where(l => l.CreateDate > publishingPeriod);
}
publishingInProgressListingIds = publishingInProgressListings.Select(l => l.ListingId).ToList();
}
#region Step 1. Mark all except parsing as removed
if (!itemsWasModified)
{
var excludeMarketListingIds = reportListings.Select(r => r.ListingId).ToList();
excludeMarketListingIds.AddRange(publishingInProgressListingIds);
using (var db = dbFactory.GetRWDb())
{
var toUnbulishListingIds = (from l in db.Listings.GetAll()
join i in db.Items.GetAll() on l.ItemId equals i.Id
where !String.IsNullOrEmpty(l.ListingId) &&
l.Market == (int)api.Market &&
(l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId)) &&
!l.IsRemoved &&
(i.ItemPublishedStatus == (int)PublishedStatuses.HasUnpublishRequest ||
i.ItemPublishedStatus == (int)PublishedStatuses.Unpublished)
select l.Id)
.ToList();
var allMarketListings = (from l in db.Listings.GetAll()
join i in db.Items.GetAll() on l.ItemId equals i.Id
where !String.IsNullOrEmpty(l.ListingId) &&
l.Market == (int)api.Market &&
(l.MarketplaceId == api.MarketplaceId || String.IsNullOrEmpty(api.MarketplaceId)) &&
!l.IsRemoved
//&& i.ItemPublishedStatus != (int)PublishedStatuses.HasChanges //NOTE: already going to republish
select new
{
l.Id,
l.ListingId,
ItemId = i.Id,
l.CreateDate
})
.ToList();
var missingOnMarketListingInfoList = allMarketListings
.Where(l => !excludeMarketListingIds.Contains(l.ListingId))
.Distinct()
.ToList();
foreach (var listingInfo in missingOnMarketListingInfoList)
{
if (toUnbulishListingIds.Contains(listingInfo.Id) ||
(_removePeriodForPublishingInProgress.HasValue && listingInfo.CreateDate < _removePeriodForPublishingInProgress))
{
//db.Items.SetItemPublishingStatus(itemId, (int)PublishedStatuses.Unpublished, "Requested by user via UnpublishRequest", time.GetAmazonNowTime());
var dbListings = db.Listings.GetAll().Where(l => l.Id == listingInfo.Id).ToList();
foreach (var dbListing in dbListings)
{
dbListing.IsRemoved = true;
Log.Info("Listing, SKU: " + dbListing.SKU + " set as removed");
}
}
else
{
try
{
db.Items.SetItemPublishingStatus(listingInfo.ItemId, (int)PublishedStatuses.HasChanges, "System Warning: the listing has the Published status, but it does not appear in the listing report.", time.GetAppNowTime());
}
catch (Exception ex) //NOTE: usually update twice one Item
{
Log.Error("SetItemPublishingStatus error, itemId=" + listingInfo.ItemId, ex);
}
Log.Info("Item, id: " + listingInfo.ItemId + " set to republish");
}
}
db.Commit();
//db.Listings.MarkAsRemoved(toRemoveDbListingIds);
//TODO: change status to HasChanges to restore items
Log.Debug("Missing on marketplace items, count=" + missingOnMarketListingInfoList.Count());
Log.Debug("Mark to republish, ItemIds=" + String.Join(",", missingOnMarketListingInfoList.Select(i => i.ItemId)));
}
}
#endregion
#region Process new report listings
IList <ItemDTO> newReportListings = new List <ItemDTO>();
using (var db = dbFactory.GetRWDb())
{
var existListingSKUs = db.Listings.GetFiltered(l => !l.IsRemoved &&
l.Market == (int)api.Market &&
l.MarketplaceId == api.MarketplaceId)
.Select(l => l.SKU)
.ToList();
newReportListings = reportListings.Where(r => !existListingSKUs.Contains(r.SKU)).ToList();
foreach (var newListingItem in newReportListings)
{
Log.Debug("New listing, listingId=" + newListingItem.ListingId);
}
var listingsWithNewParents = new List <ItemDTO>();
if (newReportListings.Any())
{
var newListingsWithError = new List <ItemDTO>();
//0. Get ParentASIN for listings (and other infos)
try
{
api.FillWithAdditionalInfo(Log,
time,
newReportListings,
IdType.SKU,
ItemFillMode.NoAdv, //NOTE: TODO: Request barcodes by separate service
out newListingsWithError);
if (_enableFakeParentASIN)
{
foreach (var l in newReportListings)
{
if (l.IsExistOnAmazon == true &&
String.IsNullOrEmpty(l.ParentASIN))
{
l.ParentASIN = l.ASIN;
}
}
}
}
catch (Exception ex) //Can continue if only part of records was filled
{
syncInfo.AddError("", "Can't fill new listing items with additional info", ex);
Log.Error("Can't fill new listing items with additional info", ex);
}
Log.Debug("Listings with errors when GetItems for new listings, asins: " + String.Join(", ", newListingsWithError.Select(i => i.ASIN).ToList()));
//1. Process Items
listingProcessor.ProcessNewListingsWithItems(db, api, time, newReportListings);
//2. Process ParentItems, and creating styles based on there names
var listingsWithParentASIN = newReportListings.Where(l => !String.IsNullOrEmpty(l.ParentASIN)).ToList();
listingsWithNewParents = db.ParentItems.CheckForExistence(listingsWithParentASIN, api.Market, api.MarketplaceId);
if (listingsWithNewParents.Any())
{
Log.Debug("Process new parents");
listingProcessor.ProcessNewParents(db, api, listingsWithNewParents, newReportListings);
}
}
}
#endregion
#region Step 3. Process existing listings
var existingListings = reportListings.Where(r => !newReportListings.Select(i => i.SKU).Contains(r.SKU)).ToList();
if (existingListings.Any())
{
var existingListingsWithError = new List <ItemDTO>();
//0. Get ParentASIN for listings (and other infos)
//NOTE: Get base info. Should go after "get barcodes"
try
{
api.FillWithAdditionalInfo(Log,
time,
existingListings,
IdType.SKU,
ItemFillMode.NoAdv,
out existingListingsWithError);
if (_enableFakeParentASIN)
{
foreach (var l in existingListings)
{
if (l.IsExistOnAmazon == true &&
String.IsNullOrEmpty(l.ParentASIN))
{
l.ParentASIN = l.ASIN;
}
}
}
}
catch (Exception ex)
{
syncInfo.AddError("", "Can't fill exist items with additional info", ex);
Log.Error("Can't fill exist items with additional info", ex);
}
Log.Debug("Error when GetItems for existing listings, asins: " + String.Join(", ", existingListingsWithError.Select(i => i.ASIN).ToList()));
//1. Process Items (keep Parent ASIN to keep source relationships)
using (var db = dbFactory.GetRWDb())
{
listingProcessor.ProcessExistingItems(db, api, existingListings);
}
//DISABLED: we don't creating new parents for existing listings, always keep ParentASINs (on CCEN we have required relationships, real ParentASIN may be others)
//2. Process ParentItems, and creating styles based on there names
//Need in some rare cases when items have empty Parent Item, ex.: parent item asin was changed but not for all product items
var parentsASINs = existingListings.Where(l => !String.IsNullOrEmpty(l.ParentASIN)).ToList();
using (var db = dbFactory.GetRWDb())
{
var newParents = db.ParentItems.CheckForExistence(parentsASINs, api.Market, api.MarketplaceId);
if (newParents.Any())
{
listingProcessor.ProcessNewParents(db, api, newParents, existingListings);
}
}
}
using (var db = dbFactory.GetRWDb())
{
listingProcessor.UpdateParentsForExistantItems(db,
api,
syncInfo);
}
#endregion
Log.Debug("ProcessItems end");
}