public List <CseProductInfo> GetProductsFromPcm(CsePcmInfo pcmInfo) { var products = GetProductsFromPcmAction(pcmInfo, _context.Op); if (products.Count == 0) { products = GetProductsFromCseAction(1); var pcmProvider = new PcmProvider(_context); pcmProvider.UpsertProductToPcm(products); return(products); } var filterProvider = new FilterProvider(_context); products = filterProvider.FilterSameStore(products); products = filterProvider.FilterPriceRange(products); products = filterProvider.FilterDuplicateStores(products); products = filterProvider.FilterDuplicateProductId(products); return(products); }
private List <CseProductInfo> GetProductsFromCseAction(int minNumberOfOffers) { var timer = new TimerProvider(_context, "get-campaign-apis-sql"); var apis = GetCseApisAndCurrencyForCurrentCampaignId(); timer.StopTimer(); if (apis.Count == 0) { return(new List <CseProductInfo>()); } var cseDataProvider = new CseToMongoEtlProvider(); var productsList = new List <CseProductInfo>(); var rawProductsList = new List <CseProductInfo>(); var merchantsList = new List <CseMerchantInfo>(); int totalNumberOfOffers = 0; while (true) { for (int i = 0; i < apis.Count; i++) { MatchingData matchingData = null; try { matchingData = GetProtuctsFromCseByKeyword(apis[i]); if (matchingData == null) { matchingData = new MatchingData(); } } catch (Exception ex) { _logger.ErrorFormat("failed get data from cse [{0}], ex: [{1}]", apis[i], ex.Message); _context.AddCseExecutionFailure(apis[i].ToString(), "Failed"); } _context.AddCseNumberOfRawResults(apis[i].ToString(), matchingData.Matches.Count.ToString()); if (matchingData.Matches.Count == 0) { continue; } _context.AddCseExecutionFailure(apis[i].ToString(), "OK"); List <CseProductInfo> products = null; List <CseMerchantInfo> merchants = null; cseDataProvider.ParseProductAndMerchants(apis[i], "widget", matchingData.ds, out products, out merchants); if (products.Count == 0) { continue; } merchantsList.AddRange(merchants.Select(x => (CseMerchantInfo)x.Clone())); rawProductsList.AddRange(products.Select(x => (CseProductInfo)x.Clone())); var filterProvider = new FilterProvider(_context); products = filterProvider.FilterSameStore(products); products = filterProvider.FilterPriceRange(products); if (products.Count == 0) { continue; } productsList.AddRange(products); productsList = filterProvider.FilterDuplicateStores(productsList); productsList = filterProvider.FilterDuplicateProductId(productsList); if (productsList.Count == 0) { continue; } totalNumberOfOffers = CountProdcutOffers(productsList); if (totalNumberOfOffers >= minNumberOfOffers) { break; } } totalNumberOfOffers = CountProdcutOffers(productsList); if (totalNumberOfOffers >= minNumberOfOffers) { break; } if (!_context.IsDefaultPrice) { var filterProvider = new FilterProvider(_context); rawProductsList = filterProvider.FilterDuplicateStores(rawProductsList); rawProductsList = filterProvider.FilterDuplicateProductId(rawProductsList); totalNumberOfOffers = CountProdcutOffers(rawProductsList); if (totalNumberOfOffers >= minNumberOfOffers) { productsList = rawProductsList; break; } } if (!ShrinkKeywordStart()) { break; } } ThreadPool.QueueUserWorkItem(state => { dynamic param = state as dynamic; cseDataProvider.AddCseResultToMongoDb("widget", _context.Op, param.products, param.merchants, false); }, new { products = new List <CseProductInfo>(rawProductsList.Select(x => (CseProductInfo)x.Clone())), merchants = new List <CseMerchantInfo>(merchantsList.Select(x => (CseMerchantInfo)x.Clone())) }); return(productsList); }