public override string WriteTable(out int itemCount) { //set column indices for each catalog field var iPId = _rules.Fields.GetHeaderIndex(FieldName.ProductId); var iPrice = _rules.Fields.GetHeaderIndex(FieldName.Price); var iSale = _rules.Fields.GetHeaderIndex(FieldName.SalePrice); var iList = _rules.Fields.GetHeaderIndex(FieldName.ListPrice); var iCost = _rules.Fields.GetHeaderIndex(FieldName.Cost); var iInv = _rules.Fields.GetHeaderIndex(FieldName.Inventory); var iRate = _rules.Fields.GetHeaderIndex(FieldName.Rating); var iName = _rules.Fields.GetHeaderIndex(FieldName.Name); var iAtt1 = _rules.UseDepartmentsAsCategories ? _rules.Fields.GetHeaderIndex(FieldName.Department) : _rules.Fields.GetHeaderIndex(FieldName.Att1Id); var iAtt2 = _rules.Fields.GetHeaderIndex(FieldName.Att2Id); var iFilt = _rules.Fields.GetHeaderIndex(FieldName.Filter); var iLink = _rules.Fields.GetHeaderIndex(FieldName.Link); var iImag = _rules.Fields.GetHeaderIndex(FieldName.ImageLink); var iCode = _rules.Fields.GetHeaderIndex(FieldName.StandardCode); var iVis = _rules.Fields.GetHeaderIndex(FieldName.Visible); var iDep = _rules.Fields.GetHeaderIndex(FieldName.Department); //set the image format string imageLinkFormat = null; if (!string.IsNullOrEmpty(_rules.ImageLinkFormat)) imageLinkFormat = _rules.ImageLinkFormat; else if (!string.IsNullOrEmpty(_rules.ImageLinkBaseUrl)) //convert base url to a format { imageLinkFormat = _rules.ImageLinkBaseUrl + "/{0}"; //add the relative image address to the end } //parse the json data into product records _progress.StartTask("Parsing data", "items", null, Rows.Count); var errors = 0; var products = new List<ProductRecord>(); for (var i = 0; i < Rows.Count; i++) { var cols = Rows[i]; try { #if DEBUG var breakHere = false; if (TableAccess.Instance.DebugIds.Contains(cols[iPId])) breakHere = true; #endif var catIds = Input.GetColVal(cols, iAtt1); if (!string.IsNullOrEmpty(_rules.CategorySeparator)) catIds = catIds.Replace(_rules.CategorySeparator, ","); //assign values from the indexes created above var p = new ProductRecord { ProductId = Input.GetColVal(cols, iPId), Name = _cart.CleanUpTitle(Input.GetColVal(cols, iName)), Att1Id = _rules.Fields.Att1Enabled ? catIds : "", Att2Id = _rules.Fields.Att2Enabled ? Input.GetColVal(cols, iAtt2) : "", Filter = Input.GetColVal(cols, iFilt), Price = Input.GetColVal(cols, iPrice), SalePrice = Input.GetColVal(cols, iSale), ListPrice = Input.GetColVal(cols, iList), Cost = Input.GetColVal(cols, iCost), Link = Input.GetColVal(cols, iLink), ImageLink = Input.GetColVal(cols, iImag), Rating = Input.GetColVal(cols, iRate), StandardCode = Input.GetColVal(cols, iCode), Visible = Input.GetColVal(cols, iVis), Inventory = Input.GetColVal(cols, iInv) }; if (_rules.MapCategoriesToFilters) p.Filter = catIds; if (p.Link.Length > 0) p.Link = p.Link.Replace("\\/", "/"); //unescape slashes if (p.ImageLink.Length > 0) { p.ImageLink = p.ImageLink.Replace("\\/", "/"); //unescape slashes var start = p.ImageLink.IndexOf("//"); if (start > 0) p.ImageLink = p.ImageLink.Substring(start); //remove protocol if (imageLinkFormat != null) p.ImageLink = string.Format(imageLinkFormat, p.ImageLink); } //create an XElement for ApplyAltPrices and ApplyRules //var productX = new XElement("product"); //for (var j = 0; j < cols.Count; j++) //{ // try // { // var value = cols[j]; // //convert department ids to names // if (_rules.ExportDepartmentNames && j == iDep && _cart.Departments.Any()) // { // var idList = value.Split(new[] { ',' }); // string name; // var nameList = idList.Select(x => _cart.Departments.TryGetValue(x, out name) ? name : x); // value = nameList.Aggregate((w, z) => string.Format("{0},{1}", w, z)); // } // //productX will contain all fields retrieved, even if not part of standard list // //this is important to enable _rules to use other fields // productX.Add(new XElement(Header[j], value)); // } // catch // { // errors++; // } //} //get parent inventory, price ranges and additional alternate prices ApplyAltPricesAndLinks(ref p, ref cols); if (_rules.MapStockToVisibility && !string.IsNullOrEmpty(p.Inventory) && p.Inventory.Equals("0")) p.Visible = "0"; ApplyRules(ref p, cols); products.Add(p); } catch { errors++; } _progress.UpdateTask(products.Count, -1, null, string.Format("{0} errors", errors)); } if (products.Count < 1) throw new Exception("Catalog feed has no data"); itemCount = products.Count; _progress.EndTask(itemCount); if (_rules.ApiMinimumCatalogSize > itemCount) throw new Exception("Catalog size is less than the minimum required in the site _rules"); if (_migrationSlave) return ""; //don't need to save catalog for migration slaves (MigrationMap created in Apply_rules) _progress.UpdateTable(itemCount, -1, "Writing table"); return TableAccess.Instance.WriteTable(_rules.Alias, CartExtractor.CatalogFilename, products); }
/// <summary> /// Check all site rules that pertain to catalog items (exclusions, filters ,etc). /// Note that an XElement is always used even if the input format is not xml. /// This allows all fields to be analyzed, even if they are not part of the standard list. /// Also note that for efficiency, exclusion tests are stopped as soon as one is found to be true. /// This means that each item will only show one exclusion cause even if it matches more than one rule. /// That could be changed in the future if it is deemed worth the performance trade-off. /// </summary> /// <param name="p"></param> /// <param name="product"></param> public void ApplyRules(ref ProductRecord p, List<string> data) { try { if (_rules.ReverseVisibleFlag) { if (p.Visible.Equals("0") || p.Visible.Equals("false", StringComparison.OrdinalIgnoreCase) || p.Visible.Equals("no", StringComparison.OrdinalIgnoreCase)) p.Visible = "1"; else p.Visible = "0"; } #if DEBUG var breakHere = false; if (TableAccess.Instance.DebugIds.Contains(p.ProductId)) breakHere = true; #endif //first check for excluded categories var excluded = false; if (_rules.CategoryRules.AnyExcluded(p.Att1Id)) { _cart.Exclusions.Add(new ExclusionRecord(p.ProductId)); _cart.AddExclusionCause(p.ProductId, CartExtractor.ExcludedCategoryCause); excluded = true; } //then check all other exclusion rules (note: item replacements are handled in GetReplacements() else if (_rules.ExclusionRules != null) { foreach (var c in _rules.ExclusionRules.Where(c => c.Compare(Input.GetValue(Header, data, c.ResultField)))) { _cart.Exclusions.Add(new ExclusionRecord(p.ProductId)); _cart.AddExclusionCause(p.ProductId, c.Name); excluded = true; break; } } if (!excluded && _rules.ExclusionSet != null) { List<string> matchingNames; excluded = _rules.ExclusionSet.Evaluate(Header, data, out matchingNames); if (excluded) foreach (var name in matchingNames) _cart.AddExclusionCause(p.ProductId, name); } //exclude items that do not have images if AllowMissingPhotos is false (which is the default) //this is checked last so that hidden and out-of-stock items don't count toward missing image count if (!excluded && String.IsNullOrEmpty(p.ImageLink) && !_rules.AllowMissingPhotos) { //provides a means for clients to exclude items if image link empty _cart.Exclusions.Add(new ExclusionRecord(p.ProductId)); _cart.AddExclusionCause(p.ProductId, CartExtractor.MissingImageCause); excluded = true; } //remove ignored categories p.Att1Id = _rules.CategoryRules.RemoveIgnored(p.Att1Id); //apply filters if (_rules.FiltersOn) { //check category rules var filters = _rules.CategoryRules.AnyFiltered(p.Att1Id); if (_rules.CategoryRules.AnyUniversal(p.Att1Id)) filters.Add(_rules.UniversalFilterName); //check filter rules if (_rules.FilterRules != null && _rules.FilterRules.Any()) { var matches = _rules.FilterRules.Where(c => c.Compare(Input.GetValue(Header, data, c.ResultField))) .Select(c => c.Name); if (matches.Any()) filters.AddRange(matches); } //check filter parsing rules if (_rules.FilterParsingRules != null) { foreach (var f in _rules.FilterParsingRules) { List<string> results; if (f.ApplyRules(Header, data, out results)) filters.AddRange(results); } } //combine any filters found if (filters.Count > 0) { if (p.Filter.Length > 0) filters.AddRange(p.Filter.Split(new[] { ',' })); //combine first so we can remove duplicates p.Filter = filters.Distinct().Aggregate((w, j) => String.Format("{0},{1}", w, j)); } if (p.Filter.Length < 1) //if no matches then assume universal p.Filter = _rules.UniversalFilterName; } //check for full catalog replacement ApplyReplacementRules(data); //check for Featured recs if (_rules.FeaturedCrossSellOn) _cart.FeaturedCrossSells.AddRecords(p.ProductId, Header, data, _rules.FeaturedCrossSellRules); if (_rules.FeaturedUpSellOn) _cart.FeaturedUpSells.AddRecords(p.ProductId, Header, data, _rules.FeaturedUpSellRules); //check for migration mapping if (_rules.MigrationRules != null) _rules.MigrationRules.MapItem(p.ProductId, Header, data); } catch (Exception ex) { if (BoostLog.Instance != null) BoostLog.Instance.WriteEntry(EventLogEntryType.Information, "Error applying rules", ex, _cart.Alias); } }
protected override string GetCatalog(out int itemCount) { Progress.UpdateTable(-1, -1, "Extracting"); itemCount = 0; //get product count var pXml = XDocument.Load(Rules.CatalogFeedUrl); //var pXml = XDocument.Load("C:\\\\ProgramData\\4-Tell2.0\\CookDrct\\upload\\cookstest_optimize.xml"); if (pXml.Root == null) throw new Exception("Error retrieving the catalog feed."); var catalogXml = pXml.Root.Descendants("Offer"); if (catalogXml == null) throw new Exception("The catalog feed does not conatin any products."); var pRows = catalogXml.Count(); Progress.UpdateTable(pRows, 0, "Parsing data"); Products = new List<ProductRecord>(); try { //now get each page of products var errors = 0; foreach (var product in catalogXml) { try { var p = new ProductRecord { ProductId = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.ProductId]), Name = CleanUpTitle(Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Name])), Att1Id = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Att1Id]), Att2Id = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Att2Id]), Price = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Price]), SalePrice = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.SalePrice]), ListPrice = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.ListPrice]), Cost = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Cost]), Filter = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Filter]), Rating = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Rating]), StandardCode = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.StandardCode]), Link = StripUriPrefix(Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Link])), ImageLink = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.ImageLink]), Visible = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Visible]), Inventory = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Inventory]) .Equals("In Stock", StringComparison.OrdinalIgnoreCase) ? "1" : "0" }; if (string.IsNullOrEmpty(p.Visible)) p.Visible = "1"; if (p.Inventory.Equals("0")) p.Visible = "0"; //make sure out-of-stock items are marked not visible //check category conditions, exclusions, and filters ApplyRules(ref p, product); Products.Add(p); } catch (Exception ex) { Debug.WriteLine(ex.Message); errors++; } Progress.UpdateTask(Products.Count, -1, null, string.Format("{0} errors", errors)); } } catch (Exception ex) { Debug.WriteLine(ex.Message); } if (Products.Count < 1) throw new Exception("Unable to extract catalog"); itemCount = Products.Count; Progress.EndTask(itemCount); if (_migrationSlave) return ""; //don't need to save catalog for migration slaves (MigrationMap created in AllpyRules) Progress.UpdateTable(itemCount, -1, "Writing table"); return TableAccess.Instance.WriteTable(Alias, CatalogFilename, Products); }
public void ApplyAltPricesAndLinks(ref ProductRecord p, ref List<string> data) { if (p == null) return; //setup list to record alternate prices var altPriceList = new List<string>(); //map child price ranges gathered above to parents var id = p.ProductId; #if DEBUG var breakHere = false; if (TableAccess.Instance.DebugIds.Contains(id)) breakHere = true; #endif ParentItem parent; if (_parentProducts.TryGetValue(id, out parent)) { //if parent has no inventory, check accumulated inventory of children if (Input.SafeIntConvert(p.Inventory) < 1) { p.Inventory = parent.Inventory.ToString("N"); var fieldName = _rules.Fields.GetName(FieldName.Inventory); if (!string.IsNullOrEmpty(fieldName)) { var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase)); if (index > -1 && index < data.Count) data[index] = p.Inventory; //adjust for other later rules } //product.SetElementValue(_rules.Fields.GetName(FieldName.Inventory), p.Inventory); } //if parent has no rating, apply child ratings (leave blank for no ratings) if (string.IsNullOrEmpty(p.Rating)) { if (_rules.UseAverageChildRating) { if (parent.RatingSum > 0 && parent.RatingCount > 0) { var average = parent.RatingSum / (float)(parent.RatingCount); p.Rating = average.ToString("F2"); } } else if (parent.TopRating > 0) p.Rating = parent.TopRating.ToString("F2"); } //standard prices contain low end of ranges from child prices bool showSalePrice = false; if (parent.Price > 0) { p.Price = parent.Price.ToString("F2"); var fieldName = _rules.Fields.GetName(FieldName.Price); if (!string.IsNullOrEmpty(fieldName)) { var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase)); if (index > -1 && index < data.Count) data[index] = p.Price; //adjust for other later rules } //product.SetElementValue(_rules.Fields.GetName(FieldName.Price), p.Price); //always use parent sale price when using parent price if (parent.SalePrice > 0) { if (parent.SalePrice < parent.Price) { p.SalePrice = parent.SalePrice.ToString("F2"); showSalePrice = true; } else p.SalePrice = _rules.HiddenSalePriceText; } else p.SalePrice = ""; fieldName = _rules.Fields.GetName(FieldName.SalePrice); if (!string.IsNullOrEmpty(fieldName)) { var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase)); if (index > -1 && index < data.Count) data[index] = p.SalePrice; //adjust for other later rules } //product.SetElementValue(_rules.Fields.GetName(FieldName.SalePrice), p.SalePrice); } if (parent.ListPrice > 0) { p.ListPrice = parent.ListPrice.ToString("F2"); var fieldName = _rules.Fields.GetName(FieldName.ListPrice); if (!string.IsNullOrEmpty(fieldName)) { var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase)); if (index > -1 && index < data.Count) data[index] = p.ListPrice; //adjust for other later rules } //product.SetElementValue(_rules.Fields.GetName(FieldName.ListPrice), p.ListPrice); } if (parent.Cost > 0) { p.Cost = parent.Cost.ToString("F2"); var fieldName = _rules.Fields.GetName(FieldName.Cost); if (!string.IsNullOrEmpty(fieldName)) { var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase)); if (index > -1 && index < data.Count) data[index] = p.Cost; //adjust for other later rules } //product.SetElementValue(_rules.Fields.GetName(FieldName.Cost), p.Cost); } //alternate prices contain top end of ranges //NOTE: Altprices are not added to the product XElement so they cannot be used in rules altPriceList.Add(parent.TopPrice > parent.Price ? parent.TopPrice.ToString("F2") : ""); altPriceList.Add(showSalePrice && parent.TopSalePrice > parent.SalePrice ? parent.TopSalePrice.ToString("F2") : ""); altPriceList.Add(parent.TopListPrice > parent.ListPrice ? parent.TopListPrice.ToString("F2") : ""); altPriceList.Add(parent.TopCost > parent.Cost ? parent.TopCost.ToString("F2") : ""); } //process any AltPrice fields //var productX = product; //must make copy to pass in lambda var d = data; //must make copy to pass in lambda var fields = _rules.Fields.GetAltFields(AltFieldGroup.AltPriceFields); if (fields != null && fields.Any()) { while (altPriceList.Count < 4) altPriceList.Add(""); altPriceList.AddRange(fields.Select(t => CartExtractor.FormatPrice(Input.GetValue(Header, d, t)))); } if (altPriceList.Any(x => !string.IsNullOrEmpty(x))) _cart.AltPrices.Add(p.ProductId, altPriceList); //process any AltPage fields fields = _rules.Fields.GetAltFields(AltFieldGroup.AltPageFields); if (fields != null && fields.Any()) { var altPageList = new List<string>(); altPageList.AddRange(fields.Select(t => Input.GetValue(Header, d, t))); if (altPageList.Any(x => !string.IsNullOrEmpty(x))) _cart.AltPageLinks.Add(p.ProductId, altPageList); } //process any altImage fields fields = _rules.Fields.GetAltFields(AltFieldGroup.AltImageFields); if (fields != null && fields.Any()) { var altImageList = new List<string>(); altImageList.AddRange(fields.Select(t => Input.GetValue(Header, d, t))); if (altImageList.Any(x => !string.IsNullOrEmpty(x))) _cart.AltImageLinks.Add(p.ProductId, altImageList); } //process any altTitle fields fields = _rules.Fields.GetAltFields(AltFieldGroup.AltTitleFields); if (fields != null && fields.Any()) { var altTitleList = new List<string>(); altTitleList.AddRange(fields.Select(t => Input.GetValue(Header, d, t))); if (altTitleList.Any(x => !string.IsNullOrEmpty(x))) _cart.AltTitles.Add(p.ProductId, altTitleList); } }
protected override string GetCatalog(out int itemCount) { //extra fields are any fields defined in ClientSettings fieldNames or rules that are not in the standard feed // check rules and combine with active fields var extraList = Rules.GetRuleFields().Union(Rules.Fields.GetActiveFields(DataGroup.Catalog)) .Except(Rules.Fields.GetStandardFields(DataGroup.Catalog), StringComparer.OrdinalIgnoreCase).ToList(); //then convert into a comma-separated query string var extraFields = string.Empty; if (extraList.Count > 0) extraFields = extraList.Aggregate((w, j) => string.Format("{0},{1}", w, j)); Progress.StartTask("Retrieving data", "items"); itemCount = 0; XDocument xdoc; GetFeedData(out xdoc, DataGroup.Catalog, DateTime.Now, 1, 0, extraFields); if (xdoc == null) throw new Exception("no data"); var tag = Rules.Fields.GetName(FieldName.ProductGroupId); if (string.IsNullOrEmpty(tag)) throw new Exception("no product group xml tag defined"); XName name = tag; var data = xdoc.Descendants(name); var rowCount = data.Count(); if (rowCount < 1) throw new Exception("no data"); //capture tag names once to speed parsing var pTag = Rules.Fields.GetName(FieldName.ProductId); if (string.IsNullOrEmpty(pTag)) //product id is the only required field throw new Exception("order-productId xml tag is undefined"); //set booleans to reduce lookups var nTag = Rules.Fields.GetName(FieldName.Name); var bName = !string.IsNullOrEmpty(nTag); var a1Tag = Rules.Fields.GetName(FieldName.Att1Id); var bAtt1 = !string.IsNullOrEmpty(a1Tag); var a2Tag = Rules.Fields.GetName(FieldName.Att2Id); var bAtt2 = !string.IsNullOrEmpty(a2Tag); var fTag = Rules.Fields.GetName(FieldName.Filter); var bFilt = !string.IsNullOrEmpty(fTag); var prTag = Rules.Fields.GetName(FieldName.Price); var bPrice = !string.IsNullOrEmpty(prTag); var sprTag = Rules.Fields.GetName(FieldName.SalePrice); var bSale = !string.IsNullOrEmpty(sprTag); var lprTag = Rules.Fields.GetName(FieldName.ListPrice); var bList = !string.IsNullOrEmpty(lprTag); var cprTag = Rules.Fields.GetName(FieldName.Cost); var bCost = !string.IsNullOrEmpty(cprTag); var lTag = Rules.Fields.GetName(FieldName.Link); var bLink = !string.IsNullOrEmpty(lTag); var imTag = Rules.Fields.GetName(FieldName.ImageLink); var bImageLink = !string.IsNullOrEmpty(imTag); var rTag = Rules.Fields.GetName(FieldName.Rating); var bRating = !string.IsNullOrEmpty(rTag); var scTag = Rules.Fields.GetName(FieldName.StandardCode); var bCode = !string.IsNullOrEmpty(scTag); var vTag = Rules.Fields.GetName(FieldName.Visible); var bVisible = !string.IsNullOrEmpty(vTag); var inTag = Rules.Fields.GetName(FieldName.Inventory); var bInventory = !string.IsNullOrEmpty(inTag); Products = new List<ProductRecord>(); //var errors = 0; try { //create product records foreach (var item in data) { var p = new ProductRecord { ProductId = Input.GetValue(item, pTag), Name = bName ? CleanUpTitle(Input.GetValue(item, nTag)) : "", Att1Id = bAtt1 && Rules.Fields.Att1Enabled ? Input.GetValue(item, a1Tag) : "", Att2Id = bAtt2 && Rules.Fields.Att2Enabled ? Input.GetValue(item, a2Tag) : "", Filter = bFilt ? Input.GetValue(item, fTag) : "", Price = bPrice ? Input.GetValue(item, prTag) : "", SalePrice = bSale ? Input.GetValue(item, sprTag) : "", ListPrice = bList ? Input.GetValue(item, lprTag) : "", Cost = bCost ? Input.GetValue(item, cprTag) : "", //TopPrice = bPriceT ? Input.GetValue(item, prtTag) : "", //TopSalePrice = bSaleT ? Input.GetValue(item, sprtTag) : "", //TopListPrice = bListT ? Input.GetValue(item, lprtTag) : "", //TopCost = bCostT ? Input.GetValue(item, cprtTag) : "", Link = bLink ? Input.GetValue(item, lTag) : "", ImageLink = bImageLink ? Input.GetValue(item, imTag) : "", Rating = bRating ? Input.GetValue(item, rTag) : "", StandardCode = bCode ? Input.GetValue(item, scTag) : "", Visible = bVisible ? Input.GetValue(item, vTag) : "", Inventory = bInventory ? Input.GetValue(item, inTag) : "", }; if (Rules.MapCategoriesToFilters) p.Filter = Input.GetValue(item, a1Tag); //note: xml feed extractor does not yet support parent/child relationships so no ParentProducts will exist var productX = item; //must make copy to pass by ref ApplyAltPricesAndLinks(ref p, ref productX); ApplyRules(ref p, productX); Products.Add(p); Progress.UpdateTask(Products.Count); //, -1, null, string.Format("{0} errors", errors)); } } catch (Exception e) { Debug.WriteLine(e.Message); } itemCount = Products.Count; Progress.EndTask(itemCount); if (_migrationSlave) return ""; //don't need to save catalog for migration slaves (MigrationMap created in AllpyRules) Progress.UpdateTable(itemCount, -1, "Writing table"); return TableAccess.Instance.WriteTable(Alias, CatalogFilename, Products); }