public override string WriteTable(out int itemCount)
        {
            //set column indices for each catalog field
            var iPId = _rules.Fields.GetHeaderIndex(FieldName.ProductId);
            var iPrice = _rules.Fields.GetHeaderIndex(FieldName.Price);
            var iSale = _rules.Fields.GetHeaderIndex(FieldName.SalePrice);
            var iList = _rules.Fields.GetHeaderIndex(FieldName.ListPrice);
            var iCost = _rules.Fields.GetHeaderIndex(FieldName.Cost);
            var iInv = _rules.Fields.GetHeaderIndex(FieldName.Inventory);
            var iRate = _rules.Fields.GetHeaderIndex(FieldName.Rating);
            var iName = _rules.Fields.GetHeaderIndex(FieldName.Name);
            var iAtt1 = _rules.UseDepartmentsAsCategories
                                        ? _rules.Fields.GetHeaderIndex(FieldName.Department)
                                        : _rules.Fields.GetHeaderIndex(FieldName.Att1Id);
            var iAtt2 = _rules.Fields.GetHeaderIndex(FieldName.Att2Id);
            var iFilt = _rules.Fields.GetHeaderIndex(FieldName.Filter);
            var iLink = _rules.Fields.GetHeaderIndex(FieldName.Link);
            var iImag = _rules.Fields.GetHeaderIndex(FieldName.ImageLink);
            var iCode = _rules.Fields.GetHeaderIndex(FieldName.StandardCode);
            var iVis = _rules.Fields.GetHeaderIndex(FieldName.Visible);
            var iDep = _rules.Fields.GetHeaderIndex(FieldName.Department);

            //set the image format
            string imageLinkFormat = null;
            if (!string.IsNullOrEmpty(_rules.ImageLinkFormat))
                imageLinkFormat = _rules.ImageLinkFormat;
            else if (!string.IsNullOrEmpty(_rules.ImageLinkBaseUrl)) //convert base url to a format
            {
                imageLinkFormat = _rules.ImageLinkBaseUrl + "/{0}"; //add the relative image address to the end
            }

            //parse the json data into product records
            _progress.StartTask("Parsing data", "items", null, Rows.Count);
            var errors = 0;
            var products = new List<ProductRecord>();
            for (var i = 0; i < Rows.Count; i++)
            {
                var cols = Rows[i];
                try
                {
            #if DEBUG
                    var breakHere = false;
                    if (TableAccess.Instance.DebugIds.Contains(cols[iPId]))
                        breakHere = true;
            #endif

                    var catIds = Input.GetColVal(cols, iAtt1);
                    if (!string.IsNullOrEmpty(_rules.CategorySeparator))
                        catIds = catIds.Replace(_rules.CategorySeparator, ",");

                    //assign values from the indexes created above
                    var p = new ProductRecord
                    {
                        ProductId = Input.GetColVal(cols, iPId),
                        Name = _cart.CleanUpTitle(Input.GetColVal(cols, iName)),
                        Att1Id = _rules.Fields.Att1Enabled ? catIds : "",
                        Att2Id = _rules.Fields.Att2Enabled ? Input.GetColVal(cols, iAtt2) : "",
                        Filter = Input.GetColVal(cols, iFilt),
                        Price = Input.GetColVal(cols, iPrice),
                        SalePrice = Input.GetColVal(cols, iSale),
                        ListPrice = Input.GetColVal(cols, iList),
                        Cost = Input.GetColVal(cols, iCost),
                        Link = Input.GetColVal(cols, iLink),
                        ImageLink = Input.GetColVal(cols, iImag),
                        Rating = Input.GetColVal(cols, iRate),
                        StandardCode = Input.GetColVal(cols, iCode),
                        Visible = Input.GetColVal(cols, iVis),
                        Inventory = Input.GetColVal(cols, iInv)
                    };

                    if (_rules.MapCategoriesToFilters) p.Filter = catIds;
                    if (p.Link.Length > 0) p.Link = p.Link.Replace("\\/", "/"); //unescape slashes
                    if (p.ImageLink.Length > 0)
                    {
                        p.ImageLink = p.ImageLink.Replace("\\/", "/"); //unescape slashes
                        var start = p.ImageLink.IndexOf("//");
                        if (start > 0) p.ImageLink = p.ImageLink.Substring(start); //remove protocol
                        if (imageLinkFormat != null)
                            p.ImageLink = string.Format(imageLinkFormat, p.ImageLink);
                    }

                    //create an XElement for ApplyAltPrices and ApplyRules
                    //var productX = new XElement("product");
                    //for (var j = 0; j < cols.Count; j++)
                    //{
                    //  try
                    //  {
                    //    var value = cols[j];
                    //    //convert department ids to names
                    //    if (_rules.ExportDepartmentNames && j == iDep && _cart.Departments.Any())
                    //    {
                    //      var idList = value.Split(new[] { ',' });
                    //      string name;
                    //      var nameList = idList.Select(x => _cart.Departments.TryGetValue(x, out name) ? name : x);
                    //      value = nameList.Aggregate((w, z) => string.Format("{0},{1}", w, z));
                    //    }

                    //    //productX will contain all fields retrieved, even if not part of standard list
                    //    //this is important to enable _rules to use other fields
                    //    productX.Add(new XElement(Header[j], value));
                    //  }
                    //  catch
                    //  {
                    //    errors++;
                    //  }
                    //}

                    //get parent inventory, price ranges and additional alternate prices
                    ApplyAltPricesAndLinks(ref p, ref cols);
                    if (_rules.MapStockToVisibility && !string.IsNullOrEmpty(p.Inventory) && p.Inventory.Equals("0"))
                        p.Visible = "0";

                    ApplyRules(ref p, cols);
                    products.Add(p);
                }
                catch
                {
                    errors++;
                }
                _progress.UpdateTask(products.Count, -1, null, string.Format("{0} errors", errors));
            }
            if (products.Count < 1)
                throw new Exception("Catalog feed has no data");
            itemCount = products.Count;
            _progress.EndTask(itemCount);
            if (_rules.ApiMinimumCatalogSize > itemCount)
                throw new Exception("Catalog size is less than the minimum required in the site _rules");

            if (_migrationSlave) return ""; //don't need to save catalog for migration slaves (MigrationMap created in Apply_rules)

            _progress.UpdateTable(itemCount, -1, "Writing table");
            return TableAccess.Instance.WriteTable(_rules.Alias, CartExtractor.CatalogFilename, products);
        }
        /// <summary>
        /// Check all site rules that pertain to catalog items (exclusions, filters ,etc).
        /// Note that an XElement is always used even if the input format is not xml.
        /// This allows all fields to be analyzed, even if they are not part of the standard list.
        /// Also note that for efficiency, exclusion tests are stopped as soon as one is found to be true.
        /// This means that each item will only show one exclusion cause even if it matches more than one rule.
        /// That could be changed in the future if it is deemed worth the performance trade-off.
        /// </summary>
        /// <param name="p"></param>
        /// <param name="product"></param>
        public void ApplyRules(ref ProductRecord p, List<string> data)
        {
            try
            {
                if (_rules.ReverseVisibleFlag)
                {
                    if (p.Visible.Equals("0")
                        || p.Visible.Equals("false", StringComparison.OrdinalIgnoreCase)
                        || p.Visible.Equals("no", StringComparison.OrdinalIgnoreCase))
                        p.Visible = "1";
                    else
                        p.Visible = "0";
                }
            #if DEBUG
                var breakHere = false;
                if (TableAccess.Instance.DebugIds.Contains(p.ProductId))
                    breakHere = true;
            #endif
                //first check for excluded categories
                var excluded = false;
                if (_rules.CategoryRules.AnyExcluded(p.Att1Id))
                {
                    _cart.Exclusions.Add(new ExclusionRecord(p.ProductId));
                    _cart.AddExclusionCause(p.ProductId, CartExtractor.ExcludedCategoryCause);
                    excluded = true;
                }

                //then check all other exclusion rules (note: item replacements are handled in GetReplacements()
                else if (_rules.ExclusionRules != null)
                {
                    foreach (var c in _rules.ExclusionRules.Where(c => c.Compare(Input.GetValue(Header, data, c.ResultField))))
                    {
                        _cart.Exclusions.Add(new ExclusionRecord(p.ProductId));
                        _cart.AddExclusionCause(p.ProductId, c.Name);
                        excluded = true;
                        break;
                    }
                }
                if (!excluded && _rules.ExclusionSet != null)
                {
                    List<string> matchingNames;
                    excluded = _rules.ExclusionSet.Evaluate(Header, data, out matchingNames);
                    if (excluded)
                        foreach (var name in matchingNames)
                            _cart.AddExclusionCause(p.ProductId, name);
                }

                //exclude items that do not have images if AllowMissingPhotos is false (which is the default)
                //this is checked last so that hidden and out-of-stock items don't count toward missing image count
                if (!excluded && String.IsNullOrEmpty(p.ImageLink) && !_rules.AllowMissingPhotos)
                {
                    //provides a means for clients to exclude items if image link empty
                    _cart.Exclusions.Add(new ExclusionRecord(p.ProductId));
                    _cart.AddExclusionCause(p.ProductId, CartExtractor.MissingImageCause);
                    excluded = true;
                }

                //remove ignored categories
                p.Att1Id = _rules.CategoryRules.RemoveIgnored(p.Att1Id);

                //apply filters
                if (_rules.FiltersOn)
                {
                    //check category rules
                    var filters = _rules.CategoryRules.AnyFiltered(p.Att1Id);
                    if (_rules.CategoryRules.AnyUniversal(p.Att1Id))
                        filters.Add(_rules.UniversalFilterName);

                    //check filter rules
                    if (_rules.FilterRules != null && _rules.FilterRules.Any())
                    {
                        var matches = _rules.FilterRules.Where(c => c.Compare(Input.GetValue(Header, data, c.ResultField)))
                                                                                            .Select(c => c.Name);
                        if (matches.Any())
                            filters.AddRange(matches);
                    }

                    //check filter parsing rules
                    if (_rules.FilterParsingRules != null)
                    {
                        foreach (var f in _rules.FilterParsingRules)
                        {
                            List<string> results;
                            if (f.ApplyRules(Header, data, out results))
                                filters.AddRange(results);
                        }
                    }

                    //combine any filters found
                    if (filters.Count > 0)
                    {
                        if (p.Filter.Length > 0) filters.AddRange(p.Filter.Split(new[] { ',' })); //combine first so we can remove duplicates
                        p.Filter = filters.Distinct().Aggregate((w, j) => String.Format("{0},{1}", w, j));
                    }
                    if (p.Filter.Length < 1) //if no matches then assume universal
                        p.Filter = _rules.UniversalFilterName;
                }

                //check for full catalog replacement
                ApplyReplacementRules(data);

                //check for Featured recs
                if (_rules.FeaturedCrossSellOn)
                    _cart.FeaturedCrossSells.AddRecords(p.ProductId, Header, data, _rules.FeaturedCrossSellRules);
                if (_rules.FeaturedUpSellOn)
                    _cart.FeaturedUpSells.AddRecords(p.ProductId, Header, data, _rules.FeaturedUpSellRules);

                //check for migration mapping
                if (_rules.MigrationRules != null)
                    _rules.MigrationRules.MapItem(p.ProductId, Header, data);
            }
            catch (Exception ex)
            {
                if (BoostLog.Instance != null)
                    BoostLog.Instance.WriteEntry(EventLogEntryType.Information, "Error applying rules", ex, _cart.Alias);
            }
        }
        protected override string GetCatalog(out int itemCount)
        {
            Progress.UpdateTable(-1, -1, "Extracting");
            itemCount = 0;

            //get product count
            var pXml = XDocument.Load(Rules.CatalogFeedUrl);
            //var pXml = XDocument.Load("C:\\\\ProgramData\\4-Tell2.0\\CookDrct\\upload\\cookstest_optimize.xml");

            if (pXml.Root == null) throw new Exception("Error retrieving the catalog feed.");
            var catalogXml = pXml.Root.Descendants("Offer");
            if (catalogXml == null) throw new Exception("The catalog feed does not conatin any products.");

            var pRows = catalogXml.Count();
            Progress.UpdateTable(pRows, 0, "Parsing data");

            Products = new List<ProductRecord>();
            try
            {
                //now get each page of products
                var errors = 0;
                foreach (var product in catalogXml)
                {
                    try
                    {
                        var p = new ProductRecord
                                            {
                                                ProductId = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.ProductId]),
                                                Name = CleanUpTitle(Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Name])),
                                                Att1Id = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Att1Id]),
                                                Att2Id = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Att2Id]),
                                                Price = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Price]),
                                                SalePrice = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.SalePrice]),
                                                ListPrice = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.ListPrice]),
                                                Cost = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Cost]),
                                                Filter = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Filter]),
                                                Rating = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Rating]),
                                                StandardCode = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.StandardCode]),
                                                Link = StripUriPrefix(Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Link])),
                                                ImageLink = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.ImageLink]),
                                                Visible = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Visible]),
                                                Inventory = Input.GetValue(product, Rules.FieldNames[(int)SiteRules.FieldName.Inventory])
                                                                            .Equals("In Stock", StringComparison.OrdinalIgnoreCase)
                                                                            ? "1"
                                                                            : "0"
                                            };

                        if (string.IsNullOrEmpty(p.Visible))
                            p.Visible = "1";
                        if (p.Inventory.Equals("0")) p.Visible = "0"; //make sure out-of-stock items are marked not visible

                        //check category conditions, exclusions, and filters
                        ApplyRules(ref p, product);

                        Products.Add(p);
                    }
                    catch (Exception ex)
                    {
                        Debug.WriteLine(ex.Message);
                        errors++;
                    }
                    Progress.UpdateTask(Products.Count, -1, null, string.Format("{0} errors", errors));
                }
            }
            catch (Exception ex)
            {
                Debug.WriteLine(ex.Message);
            }
            if (Products.Count < 1)
                throw new Exception("Unable to extract catalog");
            itemCount = Products.Count;
            Progress.EndTask(itemCount);

            if (_migrationSlave) return ""; //don't need to save catalog for migration slaves (MigrationMap created in AllpyRules)

            Progress.UpdateTable(itemCount, -1, "Writing table");
            return TableAccess.Instance.WriteTable(Alias, CatalogFilename, Products);
        }
        public void ApplyAltPricesAndLinks(ref ProductRecord p, ref List<string> data)
        {
            if (p == null) return;

            //setup list to record alternate prices
            var altPriceList = new List<string>();

            //map child price ranges gathered above to parents
            var id = p.ProductId;
            #if DEBUG
            var breakHere = false;
            if (TableAccess.Instance.DebugIds.Contains(id))
                breakHere = true;
            #endif
            ParentItem parent;
            if (_parentProducts.TryGetValue(id, out parent))
            {
                //if parent has no inventory, check accumulated inventory of children
                if (Input.SafeIntConvert(p.Inventory) < 1)
                {
                    p.Inventory = parent.Inventory.ToString("N");
                    var fieldName = _rules.Fields.GetName(FieldName.Inventory);
                    if (!string.IsNullOrEmpty(fieldName))
                    {
                        var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase));
                        if (index > -1 && index < data.Count)
                            data[index] = p.Inventory; //adjust for other later rules
                    }
                    //product.SetElementValue(_rules.Fields.GetName(FieldName.Inventory), p.Inventory);
                }

                //if parent has no rating, apply child ratings (leave blank for no ratings)
                if (string.IsNullOrEmpty(p.Rating))
                {
                    if (_rules.UseAverageChildRating)
                    {
                        if (parent.RatingSum > 0 && parent.RatingCount > 0)
                        {
                            var average = parent.RatingSum / (float)(parent.RatingCount);
                            p.Rating = average.ToString("F2");
                        }
                    }
                    else if (parent.TopRating > 0)
                        p.Rating = parent.TopRating.ToString("F2");
                }

                //standard prices contain low end of ranges from child prices
                bool showSalePrice = false;
                if (parent.Price > 0)
                {
                    p.Price = parent.Price.ToString("F2");
                    var fieldName = _rules.Fields.GetName(FieldName.Price);
                    if (!string.IsNullOrEmpty(fieldName))
                    {
                        var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase));
                        if (index > -1 && index < data.Count)
                            data[index] = p.Price; //adjust for other later rules
                    }
                    //product.SetElementValue(_rules.Fields.GetName(FieldName.Price), p.Price);

                    //always use parent sale price when using parent price
                    if (parent.SalePrice > 0)
                    {
                        if (parent.SalePrice < parent.Price)
                        {
                            p.SalePrice = parent.SalePrice.ToString("F2");
                            showSalePrice = true;
                        }
                        else p.SalePrice = _rules.HiddenSalePriceText;
                    }
                    else
                        p.SalePrice = "";

                    fieldName = _rules.Fields.GetName(FieldName.SalePrice);
                    if (!string.IsNullOrEmpty(fieldName))
                    {
                        var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase));
                        if (index > -1 && index < data.Count)
                            data[index] = p.SalePrice; //adjust for other later rules
                    }
                    //product.SetElementValue(_rules.Fields.GetName(FieldName.SalePrice), p.SalePrice);

                }
                if (parent.ListPrice > 0)
                {
                    p.ListPrice = parent.ListPrice.ToString("F2");
                    var fieldName = _rules.Fields.GetName(FieldName.ListPrice);
                    if (!string.IsNullOrEmpty(fieldName))
                    {
                        var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase));
                        if (index > -1 && index < data.Count)
                            data[index] = p.ListPrice; //adjust for other later rules
                    }
                    //product.SetElementValue(_rules.Fields.GetName(FieldName.ListPrice), p.ListPrice);
                }
                if (parent.Cost > 0)
                {
                    p.Cost = parent.Cost.ToString("F2");
                    var fieldName = _rules.Fields.GetName(FieldName.Cost);
                    if (!string.IsNullOrEmpty(fieldName))
                    {
                        var index = Header.FindIndex(x => x.Equals(fieldName, StringComparison.OrdinalIgnoreCase));
                        if (index > -1 && index < data.Count)
                            data[index] = p.Cost; //adjust for other later rules
                    }
                    //product.SetElementValue(_rules.Fields.GetName(FieldName.Cost), p.Cost);
                }

                //alternate prices contain top end of ranges
                //NOTE: Altprices are not added to the product XElement so they cannot be used in rules
                altPriceList.Add(parent.TopPrice > parent.Price ? parent.TopPrice.ToString("F2") : "");
                altPriceList.Add(showSalePrice && parent.TopSalePrice > parent.SalePrice ? parent.TopSalePrice.ToString("F2") : "");
                altPriceList.Add(parent.TopListPrice > parent.ListPrice ? parent.TopListPrice.ToString("F2") : "");
                altPriceList.Add(parent.TopCost > parent.Cost ? parent.TopCost.ToString("F2") : "");
            }

            //process any AltPrice fields
            //var productX = product; //must make copy to pass in lambda
            var d = data; //must make copy to pass in lambda
            var fields = _rules.Fields.GetAltFields(AltFieldGroup.AltPriceFields);
            if (fields != null && fields.Any())
            {
                while (altPriceList.Count < 4)
                    altPriceList.Add("");
                altPriceList.AddRange(fields.Select(t => CartExtractor.FormatPrice(Input.GetValue(Header, d, t))));
            }
            if (altPriceList.Any(x => !string.IsNullOrEmpty(x)))
                _cart.AltPrices.Add(p.ProductId, altPriceList);

            //process any AltPage fields
            fields = _rules.Fields.GetAltFields(AltFieldGroup.AltPageFields);
            if (fields != null && fields.Any())
            {
                var altPageList = new List<string>();
                altPageList.AddRange(fields.Select(t => Input.GetValue(Header, d, t)));
                if (altPageList.Any(x => !string.IsNullOrEmpty(x)))
                    _cart.AltPageLinks.Add(p.ProductId, altPageList);
            }

            //process any altImage fields
            fields = _rules.Fields.GetAltFields(AltFieldGroup.AltImageFields);
            if (fields != null && fields.Any())
            {
                var altImageList = new List<string>();
                altImageList.AddRange(fields.Select(t => Input.GetValue(Header, d, t)));
                if (altImageList.Any(x => !string.IsNullOrEmpty(x)))
                    _cart.AltImageLinks.Add(p.ProductId, altImageList);
            }

            //process any altTitle fields
            fields = _rules.Fields.GetAltFields(AltFieldGroup.AltTitleFields);
            if (fields != null && fields.Any())
            {
                var altTitleList = new List<string>();
                altTitleList.AddRange(fields.Select(t => Input.GetValue(Header, d, t)));
                if (altTitleList.Any(x => !string.IsNullOrEmpty(x)))
                    _cart.AltTitles.Add(p.ProductId, altTitleList);
            }
        }
        protected override string GetCatalog(out int itemCount)
        {
            //extra fields are any fields defined in ClientSettings fieldNames or rules that are not in the standard feed
            // check rules and combine with active fields
            var extraList = Rules.GetRuleFields().Union(Rules.Fields.GetActiveFields(DataGroup.Catalog))
                                                    .Except(Rules.Fields.GetStandardFields(DataGroup.Catalog), StringComparer.OrdinalIgnoreCase).ToList();
            //then convert into a comma-separated query string
            var extraFields = string.Empty;
            if (extraList.Count > 0)
                extraFields = extraList.Aggregate((w, j) => string.Format("{0},{1}", w, j));

            Progress.StartTask("Retrieving data", "items");
            itemCount = 0;
            XDocument xdoc;
            GetFeedData(out xdoc, DataGroup.Catalog, DateTime.Now, 1, 0, extraFields);
            if (xdoc == null)
                throw new Exception("no data");

            var tag = Rules.Fields.GetName(FieldName.ProductGroupId);
            if (string.IsNullOrEmpty(tag))
                throw new Exception("no product group xml tag defined");
            XName name = tag;
            var data = xdoc.Descendants(name);
            var rowCount = data.Count();
            if (rowCount < 1)
                throw new Exception("no data");

            //capture tag names once to speed parsing
            var pTag = Rules.Fields.GetName(FieldName.ProductId);
            if (string.IsNullOrEmpty(pTag)) //product id is the only required field
                throw new Exception("order-productId xml tag is undefined");

            //set booleans to reduce lookups
            var nTag = Rules.Fields.GetName(FieldName.Name);
            var bName = !string.IsNullOrEmpty(nTag);
            var a1Tag = Rules.Fields.GetName(FieldName.Att1Id);
            var bAtt1 = !string.IsNullOrEmpty(a1Tag);
            var a2Tag = Rules.Fields.GetName(FieldName.Att2Id);
            var bAtt2 = !string.IsNullOrEmpty(a2Tag);
            var fTag = Rules.Fields.GetName(FieldName.Filter);
            var bFilt = !string.IsNullOrEmpty(fTag);
            var prTag = Rules.Fields.GetName(FieldName.Price);
            var bPrice = !string.IsNullOrEmpty(prTag);
            var sprTag = Rules.Fields.GetName(FieldName.SalePrice);
            var bSale = !string.IsNullOrEmpty(sprTag);
            var lprTag = Rules.Fields.GetName(FieldName.ListPrice);
            var bList = !string.IsNullOrEmpty(lprTag);
            var cprTag = Rules.Fields.GetName(FieldName.Cost);
            var bCost = !string.IsNullOrEmpty(cprTag);
            var lTag = Rules.Fields.GetName(FieldName.Link);
            var bLink = !string.IsNullOrEmpty(lTag);
            var imTag = Rules.Fields.GetName(FieldName.ImageLink);
            var bImageLink = !string.IsNullOrEmpty(imTag);
            var rTag = Rules.Fields.GetName(FieldName.Rating);
            var bRating = !string.IsNullOrEmpty(rTag);
            var scTag = Rules.Fields.GetName(FieldName.StandardCode);
            var bCode = !string.IsNullOrEmpty(scTag);
            var vTag = Rules.Fields.GetName(FieldName.Visible);
            var bVisible = !string.IsNullOrEmpty(vTag);
            var inTag = Rules.Fields.GetName(FieldName.Inventory);
            var bInventory = !string.IsNullOrEmpty(inTag);

            Products = new List<ProductRecord>();
            //var errors = 0;
            try
            {
                //create product records
                foreach (var item in data)
                {
                    var p = new ProductRecord
                        {
                            ProductId = Input.GetValue(item, pTag),
                            Name = bName ? CleanUpTitle(Input.GetValue(item, nTag)) : "",
                            Att1Id = bAtt1 && Rules.Fields.Att1Enabled ? Input.GetValue(item, a1Tag) : "",
                            Att2Id = bAtt2 && Rules.Fields.Att2Enabled ? Input.GetValue(item, a2Tag) : "",
                            Filter = bFilt ? Input.GetValue(item, fTag) : "",
                            Price = bPrice ? Input.GetValue(item, prTag) : "",
                            SalePrice = bSale ? Input.GetValue(item, sprTag) : "",
                            ListPrice = bList ? Input.GetValue(item, lprTag) : "",
                            Cost = bCost ? Input.GetValue(item, cprTag) : "",
                            //TopPrice = bPriceT ? Input.GetValue(item, prtTag) : "",
                            //TopSalePrice = bSaleT ? Input.GetValue(item, sprtTag) : "",
                            //TopListPrice = bListT ? Input.GetValue(item, lprtTag) : "",
                            //TopCost = bCostT ? Input.GetValue(item, cprtTag) : "",
                            Link = bLink ? Input.GetValue(item, lTag) : "",
                            ImageLink = bImageLink ? Input.GetValue(item, imTag) : "",
                            Rating = bRating ? Input.GetValue(item, rTag) : "",
                            StandardCode = bCode ? Input.GetValue(item, scTag) : "",
                            Visible = bVisible ? Input.GetValue(item, vTag) : "",
                            Inventory = bInventory ? Input.GetValue(item, inTag) : "",
                        };

                    if (Rules.MapCategoriesToFilters) p.Filter = Input.GetValue(item, a1Tag);

                    //note: xml feed extractor does not yet support parent/child relationships so no ParentProducts will exist
                    var productX = item; //must make copy to pass by ref
                    ApplyAltPricesAndLinks(ref p, ref productX);

                    ApplyRules(ref p, productX);
                    Products.Add(p);
                    Progress.UpdateTask(Products.Count); //, -1, null, string.Format("{0} errors", errors));
                }
            }
            catch (Exception e)
            {
                Debug.WriteLine(e.Message);
            }
            itemCount = Products.Count;
            Progress.EndTask(itemCount);

            if (_migrationSlave) return ""; //don't need to save catalog for migration slaves (MigrationMap created in AllpyRules)

            Progress.UpdateTable(itemCount, -1, "Writing table");
            return TableAccess.Instance.WriteTable(Alias, CatalogFilename, Products);
        }