protected override string GetCatalog()
{
string result = "\n" + CatalogFilename + ": ";
ProgressText += result + "Extracting from web service...";
var stopWatch = new StopWatch(true);
var products = new List<ProductRecord>();
//Note: the following attempts at getting custom results did not return category ids
// only the generic request was able to get the ids
// downside is that the generic request does not get all fields, so we have to make a second request per item to get missing fields
//string query = "EDI_Name=Generic\\Products&SELECT_Columns=p.HideProduct,p.IsChildOfProductCode,p.ProductCode,p.ProductID,p.ProductName,p.ProductPopularity,p.StockStatus,p.ProductUrl,p.PhotoUrl,pe.Hide_When_OutOfStock,pe.ProductCategory,pe.ProductManufacturer,pe.PhotoURL_Small,pe.ProductPrice,pe.SalePrice,pe.UPC_code,pe.Google_Gender";
//string query = "EDI_Name=Generic\\Products&SELECT_Columns=*";
string query = null;
try
{
GetCatalogXml(query);
}
catch (Exception ex)
{
result += ex.Message;
if (ex.InnerException != null)
result += Environment.NewLine + ex.InnerException.Message;
return result;
}
int numItems = (m_catalogXml == null)? 0 : m_catalogXml.Count();
if (numItems == 0)
{
result += "There are no products in the catalog.";
return result;
}
ProgressText += string.Format("completed ({0}){1}Parsing Product Catalog ({2} rows)...",
stopWatch.Lap(), Environment.NewLine, numItems);
string tempDisplay = ProgressText;
//any extra fields not in the standard export will need to be requested from the API for each product
string extraFields = "p.Photos_Cloned_From,p.IsChildOfProductCode"; //special cases needed to find images and remove children
List<string> extraFieldList = GetRuleFields();
if (extraFieldList.Count > 0)
{
//use the first product to find standard attributes and remove them from the list
extraFieldList = extraFieldList.Where(x => (m_catalogXml.First().Descendants().Where(
y => y.Name.LocalName.Equals(x, StringComparison.CurrentCultureIgnoreCase)
).DefaultIfEmpty(null).Single() == null)).ToList<string>();
if (extraFieldList.Count > 0)
extraFields += ",pe." + extraFieldList.Aggregate((w, j) => string.Format("{0},pe.{1}", w, j));
//Note: Currently assuming that all fields are in pe table --true so far
//An option would be to compile local lists of fields in each Volusion table and check against the lists
//but then we may have to adjust for different version of Volusion
}
//create new replacement list in case child items are found
m_replacementList = new List<ReplacementRecord>();
int rows = 0;
foreach (var product in m_catalogXml)
{
var p = new ProductRecord
{
ProductId = Client.GetValue(product, "ProductCode"),
Name = Client.GetValue(product, "ProductName"),
Att2Id = string.Empty,
Price = Client.GetValue(product, "ProductPrice"),
SalePrice = Client.GetValue(product, "SalePrice"),
Filter = string.Empty
};
#if DEBUG
string[] testProducts = { "calerachard10-w", "varnerfoxglovechard10-w", "mountedenchardwolff08-w", "viticciobere2008-w" };
bool testFlag = false;
if (testProducts.Contains(p.ProductId))
testFlag = true;
#endif
if (m_secondAttEnabled)
p.Att2Id = Client.GetValue(product, "ProductManufacturer");
p.Link = string.Format("{0}/ProductDetails.asp?ProductCode={1}", "http://" + m_storeShortUrl, p.ProductId); //pdp link is never https
p.Rating = Client.GetValue(product, "ProductPopularity");
p.StandardCode = Client.GetValue(product, "UPC_code");
if (p.StandardCode.Length < 1)
p.StandardCode = p.ProductId;
var categories = product.Descendants().Where(x => x.Name.LocalName.Equals("Categories", StringComparison.CurrentCultureIgnoreCase)).SingleOrDefault();
if(categories != null)
p.Att1Id = categories.Descendants("Category").Aggregate(string.Empty, (current, cat) => current + string.Format("{0},", cat.Descendants().Where(x => x.Name.LocalName.Equals("CategoryID")).Select(x => x.Value).DefaultIfEmpty("").Single())).TrimEnd(',');
else
p.Att1Id = Client.GetValue(product, "CategoryIDs");
//If extra fields not in the current export are required, call api to get the missing fields
if (extraFields.Length > 0)
{
try
{
query = string.Format("API_Name=Generic\\Products&SELECT_Columns={0}&WHERE_Column=p.ProductCode&WHERE_Value={1}",
extraFields, p.ProductId);
XDocument extraXml = QueryVolusionApi(query);
if ((extraXml != null) && (extraXml.Root != null))
{
IEnumerable<XElement> ix = extraXml.Root.Elements("Products");
if (ix != null)
//remove any repetitive fields such as the product code
product.Add(ix.Descendants().Where(x => Client.GetValue(product, x.Name.ToString()).Length < 1));
}
}
catch
{ }
}
//check to make sure this isn't a child
string parentID = Client.GetValue(product, "IsChildOfProductCode");
if ((parentID.Length > 0) && !parentID.Equals(p.ProductId)) //child so add to replacements
m_replacementList.Add(new ReplacementRecord { OldId = p.ProductId, NewId = parentID });
//get the image link
p.ImageLink = string.Empty;
string imageID = Client.GetValue(product,"Photos_Cloned_From");
#if DEBUG
if (imageID.Length > 0) // cloned image found!
p.ImageLink = string.Empty; //just something to put a breakpoint on
#endif
if (imageID.Length < 1) //not using cloned image
{
//try to pull image from dtabase fields (these appear to never have the image)
imageID = p.ProductId;
p.ImageLink = Client.GetValue(product, "PhotoURL_Small");
if (p.ImageLink.Length < 1)
p.ImageLink = Client.GetValue(product, "PhotoURL");
}
if (p.ImageLink.Length < 1) //build the image link from the imageID
{
//replace illegal characters
string encodedID = imageID.Replace("/", "-fslash-").Replace(" ", "%20");
p.ImageLink = string.Format("{0}{1}-1.jpg", m_photoBaseUrl, encodedID);
}
//check category conditions, exclusions, and filters
ApplyRules(ref p, product);
products.Add(p);
ProgressText = string.Format("{0}{1} rows completed ({2})", tempDisplay, ++rows, stopWatch.Lap());
}
ProgressText = string.Format("{0}Completed ({1}){2}Uploading to server...",
tempDisplay, stopWatch.Lap(), Environment.NewLine);
result += m_boostService.WriteTable(m_alias, CatalogFilename, products);
ProgressText = string.Format("{0}({1})", result, stopWatch.Stop());
return result;
}
protected override string GetCatalog()
{
string result = "\n" + CatalogFilename + ": ";
string tempDisplay = ProgressText;
ProgressText += result + "Exporting catalog...";
StopWatch exportWatch = new StopWatch(true);
Mage_Api_Model_Server_V2_HandlerPortTypeClient Mclient = new Mage_Api_Model_Server_V2_HandlerPortTypeClient();
string MsessionId = "";
//---------------------CATALOG EXPORT----------------------------
//errors caught by calling function
//login
MsessionId = Mclient.login(m_apiUserName, m_apiKey);
//Set product attributes to fetch
catalogProductRequestAttributes prodAttributes = new catalogProductRequestAttributes();
string[] attributes = { "sku", "url_key", "price", "special_price", "special_from_date", "special_to_date", "parent_item_id" };
prodAttributes.attributes = attributes;
//loop through all products to build the list
var products = new List<ProductRecord>();
catalogProductEntity[] plist;
Mclient.catalogProductList(out plist, MsessionId, null, "");
int maxCid = 0;
foreach (catalogProductEntity product in plist)
{
var p = new ProductRecord
{
ProductId = product.product_id,
Name = product.name,
};
p.Att1Id = "";
bool first = true;
foreach (string cid in product.category_ids)
{
if (first) first = false;
else p.Att1Id += ",";
p.Att1Id += cid;
int id = Convert.ToInt32(cid);
if (id > maxCid) maxCid = id;
}
p.Att2Id = "";
catalogProductReturnEntity pinfo = Mclient.catalogProductInfo(MsessionId, p.ProductId, "", prodAttributes, "id");
p.Price = pinfo.price; ;
if ((pinfo.special_from_date != null) && (pinfo.special_to_date != null))
{
DateTime saleStart = DateTime.Parse(pinfo.special_from_date);
DateTime saleEnd = DateTime.Parse(pinfo.special_to_date);
DateTime now = DateTime.Now;
if (now >= saleStart && now <= saleEnd)
p.Price = pinfo.special_price;
}
p.Filter = "";
p.Link = pinfo.url_key;
p.ImageLink = "";
catalogProductImageEntity[] pimageinfo = null;
try
{
pimageinfo = Mclient.catalogProductAttributeMediaList(MsessionId, pinfo.sku, "default", null);
}
catch { }
if ((pimageinfo != null) && (pimageinfo.Length > 0))
p.ImageLink = pimageinfo[0].url;
else
{
p.ImageLink = string.Format(m_thumbnailFormat, pinfo.sku);
}
p.StandardCode = pinfo.sku;
}
ProgressText = tempDisplay + string.Format("Completed ({0}){1}Uploading to server...",
exportWatch.Lap(), Environment.NewLine);
var sb = new StringBuilder(CommonHeader + ProductRecord.Header());
foreach (var product in products)
{
sb.Append(string.Format("{0}\t", product.ProductId));
sb.Append(string.Format("{0}\t", product.Name));
sb.Append(string.Format("{0}\t", product.Att1Id));
sb.Append(string.Format("{0}\t", product.Att2Id));
sb.Append(string.Format("{0}\t", product.Price));
sb.Append(string.Format("{0}\t", product.SalePrice));
sb.Append(string.Format("{0}\t", product.Rating));
sb.Append(string.Format("{0}\t", product.Filter));
sb.Append(string.Format("{0}\t", product.Link));
sb.Append(string.Format("{0}\t", product.ImageLink));
sb.Append(string.Format("{0}\r\n", product.StandardCode));
}
result += m_boostService.WriteTable(m_alias, CatalogFilename, sb);
//get cat info
result += "\n" + Att1Filename + ": ";
ProgressText = tempDisplay + result + "Exporting category names...";
catalogCategoryInfo cinfo;
StringBuilder csb = new StringBuilder(CommonHeader + AttributeRecord.Header());
for (int cid = 1; cid <= maxCid; cid++)
{
try
{
cinfo = Mclient.catalogCategoryInfo(MsessionId, cid, "", null);
csb.Append(cid.ToString() + "\t" + cinfo.name + "\r\n");
}
catch
{
csb.Append(cid.ToString() + "\t" + cid.ToString() + "\r\n");
}
}
result += m_boostService.WriteTable(m_alias, Att1Filename, csb);
ProgressText = tempDisplay + result;
return result;
}
protected override string GetCatalog()
{
string result = "\n" + CatalogFilename + ": ";
ProgressText += result + "Rows to export...";
var pdWatch = new StopWatch(true);
//Query 3dCart for the data
//Note: Can't query for it all at once (times out on server)
//See how many rows there are
int targetRows = 0;
string sqlQuery = "SELECT COUNT(*) FROM products";
IEnumerable<XElement> queryResults = Get3dCartQuery(sqlQuery);
if (queryResults == null)
{
result += "(no data)";
ProgressText = result;
return result;
}
var child = queryResults.First<XElement>();
var rowsXml = child.Element("Expr1000") ?? child.Element("field0");
if (rowsXml != null)
targetRows = Convert.ToInt32(rowsXml.Value);
queryResults = null;
ProgressText += targetRows.ToString();
ProgressText += " (" + pdWatch.Lap() + ")";
string tempDisplay1 = ProgressText + "\nProduct/Category pairs...";
pdWatch.Start(); //restart to time cat export
//Get all product/category pairs
ProgressText += "\nRetrieving Product/Category pairs...";
string tempDisplay2 = ProgressText;
//need to step through category list in chunks since there may be a very large number
IEnumerable<XElement> catResults = null;
int catRows = 0;
string catID = "0";
while (true)
{
string categorySql = "SELECT TOP 5000 pc.catalogid, pc.categoryid\n"
+ "FROM product_category AS pc\n"
+ "WHERE pc.catalogid > " + catID + "\n"
+ "ORDER BY pc.catalogid";
queryResults = Get3dCartQuery(categorySql);
if (queryResults == null)
break; //no more data
int tempCount = queryResults.Count();
if (tempCount > 0)
{
catResults = catResults == null ? queryResults : catResults.Concat(queryResults);
//get the last cat ID from this query so next query can start there
XElement x = queryResults.ElementAt(tempCount - 1);
catID = Client.GetValue(x, "catalogid");
//update count
catRows = catResults.Count();
ProgressText = tempDisplay2 + catRows.ToString();
pdWatch.Lap();
ProgressText += " (" + pdWatch.TotalTime + ")";
}
if (tempCount < 5000)
break; //all rows found
}
ProgressText = tempDisplay1 + catRows.ToString();
ProgressText += " (" + pdWatch.TotalTime + ")";
tempDisplay1 = ProgressText + "\nRows Exported...";
ProgressText += "\nExporting...";
pdWatch.Start(); //restart to time row export
var products = new List<ProductRecord>();
int exportedRows = 0;
int categorizedRows = 0;
int lastId = 0;
int errors = 0;
bool hasResults = false;
char[] illegalChars = { '\r', '\n'}; //list any illegal characters for the product name here
//compile the list of fields that need to be extracted for each item in the catalog
var fields = new List<string>()
{ "catalogid",
"name",
"price",
"saleprice",
"review_average",
"onsale",
"eproduct_serial",
m_imageField
};
if (m_secondAttEnabled)
fields.Add(m_secondAttField);
List<string> extraFields = GetRuleFields();
if (extraFields.Count > 0)
fields = fields.Union(extraFields).Distinct().ToList();
//Loop through catalog grabbing 5000 items at a time from 3dCart
string sqlQueryStart = "SELECT TOP 5000 p." + fields.Aggregate((w, j) => string.Format("{0}, p.{1}", w, j));
sqlQueryStart += "\nFROM products AS p\n";
while (true)
{
sqlQuery = sqlQueryStart
+ "WHERE p.catalogid > " + lastId.ToString() + "\n"
+ "ORDER BY p.catalogid";
queryResults = Get3dCartQuery(sqlQuery);
if (queryResults == null)
break; //no more data
exportedRows += queryResults.Count();
pdWatch.Lap();
ProgressText = tempDisplay1 + exportedRows.ToString();
tempDisplay2 = ProgressText;
ProgressText += " (" + pdWatch.TotalTime + ")";
string productID = "";
foreach (XElement product in queryResults)
{
categorizedRows++;
productID = Client.GetValue(product, "catalogid");
string name = RemoveIllegalChars(Client.GetValue(product, "name"), illegalChars);
try
{
var p = new ProductRecord
{
Name = name,
ProductId = productID,
Link = "product.asp?itemid=" + productID,
ImageLink = string.Empty,
Att1Id = string.Empty,
Att2Id = string.Empty,
Price = Client.GetValue(product, "price"),
SalePrice = string.Empty,
Filter = string.Empty,
Rating = Client.GetValue(product, "review_average"),
StandardCode = Client.GetValue(product, "eproduct_serial"),
};
//create image link
string thumb = Client.GetValue(product, m_imageField);
if (thumb.Length > 0)
{
int doublehash = thumb.IndexOf("//");
if (doublehash >= 0) //thumb contains full url
p.ImageLink = thumb.Substring(doublehash); //drop http protocol if present
else //thumb contains asset location
{
p.ImageLink = "thumbnail.asp?file=";
if (!thumb.StartsWith("/")) p.ImageLink += "/";
p.ImageLink += thumb;
}
}
else //no thumbnail found so try using product id
p.ImageLink = "thumbnail.asp?file=/assets/images/" + p.ProductId + ".jpg";
//need to query product_category table for each product to get it's list of categories
//Note: these are all retrieved in advance in an XElement so we don't call 3dCart 30K times for this
if (catResults != null)
{
List<string> catList = catResults.Where(x => Client.GetValue(x, "catalogid").Equals(productID)).Select(y => Client.GetValue(y, "categoryid")).ToList();
p.Att1Id += catList.Aggregate((w, j) => string.Format("{0},{1}", w, j));
}
if (m_secondAttEnabled)
p.Att2Id = Client.GetValue(product, m_secondAttField);
bool onsale = Client.GetValue(product, "onsale").Equals("1");
if (onsale)
p.SalePrice = Client.GetValue(product, "saleprice");
//call base class to check category conditions, exclusions, and filters
ApplyRules(ref p, product);
products.Add(p);
}
catch { errors++; }
ProgressText = string.Format("{0} {1} completed, {2} errors ({3})", tempDisplay2, categorizedRows, errors, pdWatch.Lap());
}
lastId = Convert.ToInt32(productID);
hasResults = true;
if (categorizedRows == targetRows)
break; //read all products
}
if (products.Count < 1)
return result + "(no data)";
var pCount = string.Format("({0} items) ", products.Count);
ProgressText = string.Format("{0}{1}completed ({2}){3}Uploading to server...", tempDisplay2, pCount, pdWatch.Lap(), Environment.NewLine);
result += pCount + m_boostService.WriteTable(m_alias, CatalogFilename, products);
return result;
}
protected override string GetCatalog()
{
string result = "\n" + CatalogFilename + ": ";
ProgressText += result + "Extracting...";
var stopWatch = new StopWatch(true);
var products = new List<ProductRecord>();
IEnumerable<XElement> items = Catalog.Elements("Product_Add");
int numItems = (items == null) ? 0 : items.Count();
if (numItems == 0)
{
result += " (no data)";
ProgressText = result;
return result;
}
IEnumerable<XElement> customFields = Catalog.Elements("Product_CustomField");
IEnumerable<XElement> catAssignments = Catalog.Elements("CategoryProduct_Assign");
result += string.Format("({0} items) ", numItems);
ProgressText += result + string.Format("Extracting...completed ({0}){1}Parsing Product Catalog...",
stopWatch.Lap(), Environment.NewLine);
string tempDisplay = ProgressText;
int rows = 0, errors = 0;
foreach (var product in items)
{
try
{
var p = new ProductRecord
{
ProductId = Client.GetValue(product, "Code"),
Name = Client.GetValue(product, "Name"),
Att1Id = string.Empty,
Att2Id = string.Empty,
Price = Client.GetValue(product, "Price"),
SalePrice = string.Empty,
Filter = string.Empty,
ImageLink = Client.GetValue(product, "ThumbnailImage"),
};
if (p.ImageLink.Length < 1)
p.ImageLink = Client.GetValue(product, "FullSizeImage");
p.Link = string.Format("{0}/product/{1}.html", "http://" + m_storeShortUrl, p.ProductId); //pdp link is never https
p.Rating = string.Empty;
//parse custom fields
IEnumerable<XElement> cfSubset = customFields.Where(x => Client.GetAttribute(x, "product").Equals(p.ProductId));
p.StandardCode = cfSubset.Where(x => Client.GetAttribute(x, "field").Equals("UPC")).Select(x => x.Value).DefaultIfEmpty(string.Empty).Single();
if (p.StandardCode.Length < 1)
p.StandardCode = p.ProductId;
p.Att2Id = cfSubset.Where(x => Client.GetAttribute(x, "field").Equals("brand")).Select(x => x.Value).DefaultIfEmpty(string.Empty).Single();
//get category list
var categories = catAssignments.Where(x => Client.GetAttribute(x, "product_code").Equals(p.ProductId)).Select(x => Client.GetAttribute(x, "category_code"));
if (categories != null)
//{
// bool first = true;
// foreach (string id in categories)
// {
// if (first) first = false;
// else p.Attr1 += ",";
// p.Attr1 += id;
// }
//}
// p.Attr1 = categories.Aggregate(string.Empty, (current, cat) => current + string.Format("{0},", cat.Descendants().Where(x => x.Name.LocalName.Equals("CategoryID")).Select(x => x.Value).DefaultIfEmpty("").Single())).TrimEnd(',');
p.Att1Id = categories.Aggregate((w, j) => string.Format("{0},{1}", w, j));
//check category conditions, exclusions and filters
ApplyRules(ref p, product);
products.Add(p);
//This is now handled in ApplyRules
////Need to take care of exclusions here since we have access to all the product attributes
////if (Client.GetValue(product, "Active").Equals("No"))
//// m_catConditions.AddExcludedItem(p.ProductId);
//if (m_exclusions != null)
//{
// foreach (var c in m_exclusions)
// {
// string value = Client.GetValue(product, c.FieldName);
// if (value.Length < 1) //not found so check custom fields
// value = cfSubset.Where(x => Client.GetAttribute(x, "field").Equals(c.FieldName)).Select(x => x.Value).DefaultIfEmpty(string.Empty).Single();
// if (value.Length > 0)
// if (c.Compare(value))
// {
// m_catConditions.AddExcludedItem(p.ProductId);
// break;
// }
// }
//}
}
catch { errors++; }
ProgressText = string.Format("{0}{1} products, {2} errors ({3})", tempDisplay, ++rows, errors, stopWatch.Lap());
}
ProgressText = string.Format("{0}Completed ({1}){2}Uploading to server...",
tempDisplay, stopWatch.Lap(), Environment.NewLine);
result += m_boostService.WriteTable(m_alias, CatalogFilename, products);
stopWatch.Stop();
return result;
}
protected void ApplyRules(ref ProductRecord p, XElement product)
{
//first check for excluded categories
if (m_catConditions.AnyExcluded(p.Att1Id))
m_catConditions.AddExcludedItem(p.ProductId);
else //then check all other exclusion rules
{
if (m_exclusions != null)
{
if (m_exclusions.Any(c => c.Compare(Client.GetValue(product, c.FieldName))))
m_catConditions.AddExcludedItem(p.ProductId); //add to cat excluded items so it will be prepolulated for GetExclusions
}
}
//remove ignored categories
p.Att1Id = m_catConditions.RemoveIgnored(p.Att1Id);
//apply filters
if (m_filtersEnabled)
{
if (m_catConditions.AnyUniversal(p.Att1Id))
p.Filter = m_universalFilterName;
string filters = m_catConditions.AnyFiltered(p.Att1Id);
if (filters.Length > 0)
{
if (p.Filter.Length > 0) p.Filter += ",";
p.Filter += filters;
}
if (m_filters != null)
{
//bool firstItem = p.Filter.Length < 1;
//foreach (Condition c in m_filters.Where(c => c.Compare(Client.GetValue(product, c.FieldName))))
//{
// if (firstItem) firstItem = false;
// else p.Filter += ",";
// p.Filter += c.Name;
//}
p.Filter += m_filters.Where(
c => c.Compare(Client.GetValue(product, c.FieldName))
).Select(
c => c.Name
).Aggregate((w, j) => string.Format("{0},{1}", w, j));
}
if (p.Filter.Length < 1) //if no matches then assume universal
p.Filter = m_universalFilterName;
}
//check for full catalog replacement
if (m_replacements != null && m_replacements[0].Type.Equals(ReplacementCondition.RepType.Catalog))
{
var oldId = Client.GetValue(product, m_replacements[0].OldName);
var newId = Client.GetValue(product, m_replacements[0].NewName);
if (!m_repRecords.Any(r => r.OldId.Equals(oldId))) //can only have one replacement for each item
m_repRecords.Add(new ReplacementRecord { OldId = oldId, NewId = newId });
}
//check for manual recs
if (m_manualCrossSell.Enabled)
m_manualCrossSell.AddRecords(p.ProductId, product);
if (m_manualUpSell.Enabled)
m_manualUpSell.AddRecords(p.ProductId, product);
}
protected override string GetCatalog()
{
var stopWatch = new StopWatch(true);
var products = new List<ProductRecord>();
//get product count
var countXml = CallCartApi("products/count", "products");
var countVal = countXml.Elements("count").First().Value;
int pRows = Convert.ToInt16(countVal);
//setup progress display
var result = string.Format("{0}{1}: ({2} items) ", Environment.NewLine, CatalogFilename, countVal);
ProgressText += result + "Extracting...";
string tempDisplay = ProgressText;
//first get all the images because there can be many images for each product so record pages are not aligned
countXml = CallCartApi("products/images/count", "images");
countVal = countXml.Elements("count").First().Value;
int iRows = Convert.ToInt16(countVal);
int pages = (int)Math.Ceiling( (decimal)iRows / (decimal)250 ); //can only get 250 products at a time
var imagesXml = new List<XElement>(); //this will hold all images for entire catalog
for (int page = 1; page <= pages; page++)
{
string paging = string.Format("?page={0}&limit=250", page);
var pageXml = CallCartApi("products/images" + paging, "images");
if (pageXml == null)
break;
imagesXml.AddRange(pageXml.Elements("image"));
ProgressText = string.Format("{1}{0}{2} Images ({3})", Environment.NewLine, tempDisplay, imagesXml.Count, stopWatch.Lap());
}
tempDisplay = ProgressText;
#if DEBUG
if (imagesXml.Count() != iRows)
{
var errMsg = string.Format("Error reading BigCommerce images\nReported count = {0}, actual count = {1}",
iRows, imagesXml.Count());
m_log.WriteEntry(errMsg, EventLogEntryType.Information, m_alias);
}
#endif
//Next get all the custom fields so we don't have to make a separate call for each product
countXml = CallCartApi("products/customfields/count", "customfields");
countVal = countXml.Elements("count").First().Value;
iRows = Convert.ToInt16(countVal);
pages = (int)Math.Ceiling((decimal)iRows / (decimal)250); //can only get 250 products at a time
var customFieldXml = new List<XElement>(); //this will hold all images for entire catalog
for (int page = 1; page <= pages; page++)
{
string paging = string.Format("?page={0}&limit=250", page);
var pageXml = CallCartApi("products/customfields" + paging, "customfields");
if (pageXml == null)
break;
customFieldXml.AddRange(pageXml.Elements("customfield"));
ProgressText = string.Format("{1}{0}{2} Custom Fields ({3})", Environment.NewLine, tempDisplay, customFieldXml.Count, stopWatch.Lap());
}
tempDisplay = ProgressText;
//Next get all the Categories so we can construct the tree
countXml = CallCartApi("categories/count", "categories");
countVal = countXml.Elements("count").First().Value;
iRows = Convert.ToInt16(countVal);
pages = (int)Math.Ceiling((decimal)iRows / (decimal)250); //can only get 250 products at a time
var categoryXml = new List<XElement>(); //this will hold all images for entire catalog
for (int page = 1; page <= pages; page++)
{
string paging = string.Format("?page={0}&limit=250", page);
var pageXml = CallCartApi("categories" + paging, "categories");
if (pageXml == null)
break;
categoryXml.AddRange(pageXml.Elements("category"));
ProgressText = string.Format("{1}{0}{2} Categories ({3})", Environment.NewLine, tempDisplay, categoryXml.Count, stopWatch.Lap());
}
SetCategoryParents(categoryXml);
ProgressText += Environment.NewLine;
tempDisplay = ProgressText;
//now get each page of products
pages = (int)Math.Ceiling((decimal)pRows / (decimal)250); //can only get 250 products at a time
IEnumerable<XElement> productsXml = null; //this will only hold a single page of products
int errors = 0; pRows = 0;
for (int page = 1; page <= pages; page++)
{
string paging = string.Format("?page={0}&limit=250", page);
productsXml = CallCartApi("products" + paging, "products");
if (productsXml == null)
{
errors++; //not necessarily an error
break;
}
var productElements = productsXml.Elements("product");
foreach (var product in productElements)
{
try
{
var p = new ProductRecord
{
Name = Client.GetValue(product, "name"),
ProductId = Client.GetValue(product, "id"),
Att2Id = Client.GetValue(product, "brand_id"),
Price = Client.GetValue(product, "price"),
SalePrice = Client.GetValue(product, "sale_price"),
Filter = string.Empty,
Rating = Client.GetValue(product, "rating_total"),
StandardCode = Client.GetValue(product, "upc"),
Link = Client.GetValue(product, "custom_url"),
ImageLink = string.Empty
};
p.ImageLink = GetProductImage(imagesXml, p.ProductId);
p.Att1Id = GetProductCategories(product.Element("categories"));
if (p.StandardCode == null)
p.StandardCode = string.Empty;
#if DEBUG
if (p.ProductId.Equals("11348") || p.ProductId.Equals("11012"))
{
var test = CallCartApi(string.Format("products/{0}/customfields", p.ProductId), "customfields");
var s = "break here";
}
#endif
var customFields = GetProductCustomFields(customFieldXml, p.ProductId);
if (customFields.Any())
product.Add(customFields);
//check category conditions, exclusions, and filters
ApplyRules(ref p, product);
products.Add(p);
}
catch { errors++; }
ProgressText = string.Format("{0}{1} items completed, {2} errors ({3})", tempDisplay, ++pRows, errors, stopWatch.Lap());
}
}
if (products.Count < 1)
return result + "(no data)";
var pCount = string.Format("({0} items) ", products.Count);
ProgressText = string.Format("{0}{1} items completed ({2}){3}Uploading to server...", tempDisplay, pCount, stopWatch.Lap(), Environment.NewLine);
result += m_boostService.WriteTable(m_alias, CatalogFilename, products);
stopWatch.Stop();
return result;
}
/// <summary>
/// Bind catalog data to CatalogItem(s)
/// </summary>
/// <param name="Content"></param>
internal List<ProductRecord> SetCatalogData(Stream Content)
{
List<ProductRecord> products = new List<ProductRecord>();
StreamReader rdr = new StreamReader(Content);
string sContent = rdr.ReadToEnd();
/*
*
* ? sContent
"[[\"ProductID (Yes) \", \"Name (Yes) \", \"CategoryIDs (Yes) \", \"BrandID (No) \", \"Price (Yes) \", \"SalePrice (No) \", \"Link (Yes) \", \"ImageLink (Yes) \", \"StandardCode (No)\", \"ActiveFlag (Yes)\", \"StockLevel (No)\"],
* [\"product1\", \"product1\", \"\", \"\", \"5.00\", \"\", \"http:\\/\\/4-tell.coolcommerce.net\\/mm5\\/merchant.mvc\\u003fStore_code\\u003d4tell\\u0026amp\\u003bScreen\\u003dPROD\\u0026amp\\u003bProduct_Code\\u003dproduct1\", \"\", \"product1\", \"0\", \"\"], [\"product2\", \"product2\", \"test4,test5\", \"\", \"110.00\", \"\", \"http:\\/\\/4-tell.coolcommerce.net\\/mm5\\/merchant.mvc\\u003fStore_code\\u003d4tell\\u0026amp\\u003bScreen\\u003dPROD\\u0026amp\\u003bProduct_Code\\u003dproduct2\", \"\", \"product2\", \"1\", \"\"], [\"product3\", \"product3\", \"\", \"\", \"1.22\", \"\", \"http:\\/\\/4-tell.coolcommerce.net\\/mm5\\/merchant.mvc\\u003fStore_code\\u003d4tell\\u0026amp\\u003bScreen\\u003dPROD\\u0026amp\\u003bProduct_Code\\u003dproduct3\", \"\", \"product3\", \"1\", \"\"], [\"product4\", \"product4\", \"test1\", \"\", \"55.19\", \"\", \"http:\\/\
\/4-tell.coolcommerce.net\\/mm5\\/merchant.mvc\\u003fStore_code\\u003d4tell\\u0026amp\\u003bScreen\\u003dPROD\\u0026amp\\u003bProduct_Code\\u003dproduct4\", \"\", \"product4\", \"1\", \"\"]]
*
*/
string[] rows = sContent.Trim().Split(']');
string thisRow = String.Empty;
foreach (string row in rows)
{
if (row.Length > 0)
{
thisRow = row;
thisRow = thisRow.Replace("[", String.Empty);
thisRow = thisRow.Replace("]", String.Empty);
ProductRecord pr = new ProductRecord();
if (! row.StartsWith("[[") == true && ! row.EndsWith("]]") == true)
{
// normal row
string[] cols = thisRow.Split(',');
// \"ProductID\",
pr.ProductId = cols[1].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"Name\",
pr.Name = cols[2].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"CategoryIDs\",
pr.CategoryIDs = cols[3].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"BrandID\",
pr.BrandID = cols[4].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"Price\",
pr.Price = cols[5].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"SalePrice\",
pr.SalePrice = cols[6].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"Link\",
pr.Link = cols[7].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"ImageLink\",
pr.ImageLink = cols[8].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"StandardCode \",
pr.StandardCode = cols[9].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"ActiveFlag\",
pr.ActiveFlag = cols[10].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
// \"StockLevel\"],
pr.StockLevel = cols[11].Replace("\\", String.Empty).Replace("/", String.Empty).Replace('"', ' ').Trim();
products.Add(pr);
}
}
}
return products;
}
